diff --git a/.agents/skills/asap-code-review/SKILL.md b/.agents/skills/asap-code-review/SKILL.md index 5d6091fcc..bf1b353c1 100644 --- a/.agents/skills/asap-code-review/SKILL.md +++ b/.agents/skills/asap-code-review/SKILL.md @@ -72,7 +72,7 @@ a trade-off, give the author choices here too. For example: - `price` recomputes `query_reach(dag)`, a walk over the whole DAG, for every - MatAtQuery node: O(n · (n + e)) per candidate, times every candidate. At + `OneEvaluation` node: O(n · (n + e)) per candidate, times every candidate. At Example 3's 972 candidates, if each DAG has about 50 nodes, that is about 2.5·10⁶ node visits instead of 5·10⁴. Alternative: compute it once per candidate before the loop and pass it in. diff --git a/Cargo.lock b/Cargo.lock index bb23328cc..933b01efb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -434,12 +434,7 @@ dependencies = [ name = "asap-physical-asap-optimization" version = "0.1.0" dependencies = [ - "asap-accuracy", - "asap-cost", - "asap-logical-asap-optimization", "asap-types", - "rmp-serde", - "serde_json", "thiserror 2.0.18", ] @@ -449,14 +444,9 @@ version = "0.1.0" dependencies = [ "asap-accuracy", "asap-cost", - "asap-frontend-promql", - "asap-logical-asap-optimization", - "asap-physical-asap-optimization", "asap-types", - "asap_sketchlib", "good_lp", "highs", - "rmp-serde", "serde", "serde_json", "thiserror 2.0.18", @@ -476,7 +466,6 @@ dependencies = [ "asap-logical-asap-optimization", "asap-physical-asap-optimization", "asap-plan-selection", - "asap-planner", "asap-types", "futures", "serde", diff --git a/crates/accuracy/src/estimators/cms.rs b/crates/accuracy/src/estimators/cms.rs index 663749928..ebce49b97 100644 --- a/crates/accuracy/src/estimators/cms.rs +++ b/crates/accuracy/src/estimators/cms.rs @@ -42,14 +42,11 @@ mod tests { #[test] fn local_guarantee_inverts_frequency_sizing() { use crate::estimators::size_params; - use asap_types::ir::schema::{GroupingStrategy, SketchKind}; let c = asap_types::ir::operator::agg_intent::default_cardinality(); let params = size_params(SketchAlgorithm::Cms, &c, 0.01, 0.001); - let g = local_guarantee( - &FieldDataType::Sketch( - SketchKind::new(SketchAlgorithm::Cms, params), - GroupingStrategy::default(), - ), + let g = sketch_guarantee( + &SketchAlgorithm::Cms, + ¶ms, &SketchStatistic::PointCount { key: asap_types::ir::scalar::expr_ir::ColumnRef::SampleValue, value: None, @@ -68,17 +65,14 @@ mod tests { #[test] fn heap_evaluation_retains_frequency_metric() { - use asap_types::ir::schema::{GroupingStrategy, SketchKind}; let cms_heap = SketchParams::CmsWithHeap { width: 272, depth: 5, heap_size: 10, }; - let topk_frequency = local_guarantee( - &FieldDataType::Sketch( - SketchKind::new(SketchAlgorithm::CmsWithHeap, cms_heap), - GroupingStrategy::default(), - ), + let topk_frequency = sketch_guarantee( + &SketchAlgorithm::CmsWithHeap, + &cms_heap, &SketchStatistic::TopK { k: 10 }, ) .expect("heap sketch still provides per-key frequency intervals"); diff --git a/crates/accuracy/src/estimators/count_sketch.rs b/crates/accuracy/src/estimators/count_sketch.rs index dac391fe0..433803db3 100644 --- a/crates/accuracy/src/estimators/count_sketch.rs +++ b/crates/accuracy/src/estimators/count_sketch.rs @@ -157,14 +157,11 @@ mod tests { fn count_sketch_uses_an_l2_guarantee() { use crate::estimators::size_params; use asap_types::ir::operator::agg_intent::default_cardinality; - use asap_types::ir::schema::{GroupingStrategy, SketchKind}; let intent = default_cardinality(); let count_sketch = size_params(SketchAlgorithm::CountSketch, &intent, 0.01, 0.01); - let guarantee = local_guarantee( - &FieldDataType::Sketch( - SketchKind::new(SketchAlgorithm::CountSketch, count_sketch), - GroupingStrategy::default(), - ), + let guarantee = sketch_guarantee( + &SketchAlgorithm::CountSketch, + &count_sketch, &SketchStatistic::PointCount { key: asap_types::ir::scalar::expr_ir::ColumnRef::SampleValue, value: Some("1".into()), diff --git a/crates/accuracy/src/estimators/hll.rs b/crates/accuracy/src/estimators/hll.rs index 18d45db02..65ff8512f 100644 --- a/crates/accuracy/src/estimators/hll.rs +++ b/crates/accuracy/src/estimators/hll.rs @@ -81,6 +81,7 @@ pub fn parse_classic_failure_statistic(statistic: &str) -> Option<(u8, f64, &str /// distinct bound ([`classic_failure_statistic`]). The failure probability /// is per output row: each group's estimate fails with at most it. `None` /// for a precision outside the contract's. +#[allow(dead_code)] // Read again in #1207 PR 4, with θ from `LogicalProperties::parameters`. pub(crate) fn classic_guarantee( precision: u8, relative_error: Option, @@ -479,14 +480,11 @@ mod tests { fn generic_rse_sizing_does_not_certify_confidence() { use crate::estimators::size_params; use asap_types::ir::operator::agg_intent::default_cardinality; - use asap_types::ir::schema::{GroupingStrategy, SketchKind}; let c = default_cardinality(); let params = size_params(SketchAlgorithm::Hll, &c, 0.01, 0.01); - let g = local_guarantee( - &FieldDataType::Sketch( - SketchKind::new(SketchAlgorithm::Hll, params), - GroupingStrategy::default(), - ), + let g = sketch_guarantee( + &SketchAlgorithm::Hll, + ¶ms, &SketchStatistic::Cardinality, ) .unwrap(); diff --git a/crates/accuracy/src/estimators/kll.rs b/crates/accuracy/src/estimators/kll.rs index 92bee9d84..3225994e4 100644 --- a/crates/accuracy/src/estimators/kll.rs +++ b/crates/accuracy/src/estimators/kll.rs @@ -180,34 +180,6 @@ pub(crate) fn with_failure_probability(params: &SketchParams, delta: f64) -> Ske } } -/// The `k` of one KLL shared by two readers sized alone at `(k, δ)` and -/// `(k2, δ2)`: each `k` resized to the smaller failure probability at no -/// larger rank error ([`with_failure_probability`]), then the larger. The -/// shared build states its guarantee at that smaller δ, where a `k` sized -/// for a larger δ has a larger rank error, so the larger `k` alone could -/// miss its reader's ε. -pub(crate) fn shared_k( - k: u32, - failure_probability: Option, - k2: u32, - failure_probability2: Option, -) -> u32 { - let delta = failure_probability - .unwrap_or(KLL_CALIBRATED_FAILURE_PROBABILITY) - .min(failure_probability2.unwrap_or(KLL_CALIBRATED_FAILURE_PROBABILITY)); - let resized = |k, failure_probability| match with_failure_probability( - &SketchParams::Kll { - k, - failure_probability, - }, - delta, - ) { - SketchParams::Kll { k, .. } => k, - _ => k, - }; - resized(k, failure_probability).max(resized(k2, failure_probability2)) -} - #[cfg(test)] mod tests { use super::*; @@ -216,14 +188,11 @@ mod tests { fn local_guarantee_inverts_rank_sizing() { use crate::estimators::size_params; use asap_types::ir::operator::agg_intent::default_quantile; - use asap_types::ir::schema::{GroupingStrategy, SketchKind}; let q = default_quantile(0.99); let params = size_params(SketchAlgorithm::Kll, &q, 0.01, 0.01); - let g = local_guarantee( - &FieldDataType::Sketch( - SketchKind::new(SketchAlgorithm::Kll, params), - GroupingStrategy::default(), - ), + let g = sketch_guarantee( + &SketchAlgorithm::Kll, + ¶ms, &SketchStatistic::Quantile { q: 0.99 }, ) .unwrap(); diff --git a/crates/accuracy/src/estimators/mod.rs b/crates/accuracy/src/estimators/mod.rs index 6f5dd4f08..69a14f3a1 100644 --- a/crates/accuracy/src/estimators/mod.rs +++ b/crates/accuracy/src/estimators/mod.rs @@ -11,6 +11,7 @@ pub mod hll; pub(crate) mod kll; pub mod univmon; +#[allow(dead_code)] // Read again for the error bound of each listed θ (`parameter_list`), in #1207 PRs 3–4. pub(super) fn sketch_guarantee( algorithm: &SketchAlgorithm, params: &SketchParams, @@ -28,6 +29,9 @@ pub(super) fn sketch_guarantee( } SketchParams::Kmv { .. } => cardinality::guarantee(algorithm, params, query), SketchParams::UnivMon { .. } => univmon::guarantee(algorithm, params, query), + SketchParams::Hydra(hydra) => { + sketch_guarantee(algorithm, &hydra.per_subpopulation(), query) + } } } @@ -71,8 +75,10 @@ pub(crate) fn local_guarantee( FieldDataType::ExactAggregate(kind, _) => Some(AccuracyGuarantee::exact(format!( "ExactAggregate({kind:?})" ))), - FieldDataType::Sketch(kind, _) if FamilyCapability::of(family).answers(query) => { - sketch_guarantee(kind.algorithm(), kind.params(), query) + FieldDataType::Sketch(..) if FamilyCapability::of(family).answers(query) => { + todo!( + "#1207 PR 4: the guarantee of θ from the candidate's LogicalProperties::parameters" + ) } FieldDataType::Sketch(..) => None, // No error model is registered for these families. @@ -136,6 +142,7 @@ pub fn with_failure_probability(params: &SketchParams, delta: f64) -> SketchPara } } +#[allow(dead_code)] // Read again for the predefined θ lists (`parameter_list`), in #1207 PR 1. pub(crate) fn size_params( kind: SketchAlgorithm, intent: &AggIntent, @@ -240,162 +247,6 @@ pub(crate) fn binomial_upper_tail(n: u32, p: f64, k: u32) -> f64 { (tail * (1.0 + 1e-12)).min(1.0) } -/// The parameters of one state shared by consumers of the same algorithm, -/// each sized alone with one of `params` (Pass 2's summary-capability -/// rule): each parameter's largest value, except where a smaller value is -/// stricter, where it takes the smallest (a KLL's failure probability, -/// DDSketch's α). Not every bound is monotone in its parameters (a -/// UnivMon's L2 collision bound is tighter at a power-of-two width than one -/// column wider), so the shared state is not assumed to meet each -/// consumer's target: the summary-capability rule composes every guarantee -/// above it again, and plan selection checks each consumer against the -/// shared state. `None` for no parameters, -/// parameters of different algorithms, or a shared UnivMon the library -/// would not build (`univmon::hash_independent`). -/// -/// A KLL's `k` and a distinct-count UnivMon's heap are the exceptions to -/// the largest value: their error grows as their failure probability -/// shrinks, so each reader's `k` or heap is first resized to the shared -/// (smallest) failure probability at no larger error (`kll::shared_k`, -/// `univmon::shared_heap`), and the shared build meets every reader. -pub fn size_for_all_readers(params: &[&SketchParams]) -> Option { - use SketchParams as P; - let (first, rest) = params.split_first()?; - let shared = rest.iter().try_fold((*first).clone(), |a, b| { - Some(match (a, *b) { - ( - P::UnivMon { - heap_size, - sketch_rows, - sketch_cols, - layers, - distinct_failure_probability, - }, - P::UnivMon { - heap_size: h, - sketch_rows: r, - sketch_cols: c, - layers: l, - distinct_failure_probability: f, - }, - ) => P::UnivMon { - // Not the largest heap: a distinct count's error grows as - // its failure probability shrinks, so each reader's heap - // first grows to the shared one at no larger error. - heap_size: univmon::shared_heap(heap_size, distinct_failure_probability, *h, *f), - sketch_rows: sketch_rows.max(*r), - sketch_cols: sketch_cols.max(*c), - layers: layers.max(*l), - // `None`: not sized for a distinct count. - distinct_failure_probability: min_recorded(distinct_failure_probability, *f), - }, - ( - P::Kll { - k, - failure_probability, - }, - P::Kll { - k: k2, - failure_probability: f2, - }, - ) => P::Kll { - k: kll::shared_k(k, failure_probability, *k2, *f2), - // `None` is the calibration's δ, above any recorded one. - failure_probability: min_recorded(failure_probability, *f2), - }, - (P::Cms { width, depth }, P::Cms { width: w, depth: d }) => P::Cms { - width: width.max(*w), - depth: depth.max(*d), - }, - (P::CountSketch { width, depth }, P::CountSketch { width: w, depth: d }) => { - P::CountSketch { - width: width.max(*w), - depth: depth.max(*d), - } - } - ( - P::CmsWithHeap { - width, - depth, - heap_size, - }, - P::CmsWithHeap { - width: w, - depth: d, - heap_size: h, - }, - ) => P::CmsWithHeap { - width: width.max(*w), - depth: depth.max(*d), - heap_size: heap_size.max(*h), - }, - ( - P::CountSketchWithHeap { - width, - depth, - heap_size, - }, - P::CountSketchWithHeap { - width: w, - depth: d, - heap_size: h, - }, - ) => P::CountSketchWithHeap { - width: width.max(*w), - depth: depth.max(*d), - heap_size: heap_size.max(*h), - }, - ( - P::Hll { - precision, - relative_error, - }, - P::Hll { - precision: p, - relative_error: e, - }, - ) => P::Hll { - precision: precision.max(*p), - relative_error: min_recorded(relative_error, *e), - }, - (P::DDSketch { alpha }, P::DDSketch { alpha: a }) => P::DDSketch { - alpha: alpha.min(*a), - }, - ( - P::Kmv { k, relative_error }, - P::Kmv { - k: k2, - relative_error: e, - }, - ) => P::Kmv { - k: k.max(*k2), - relative_error: min_recorded(relative_error, *e), - }, - _ => return None, - }) - })?; - // The largest rows of one UnivMon with the largest columns of another - // can pass the hash budget `asap_sketchlib` builds within. - match shared { - P::UnivMon { - sketch_rows, - sketch_cols, - layers, - .. - } if !univmon::hash_independent(sketch_rows, sketch_cols, layers) => None, - shared => Some(shared), - } -} - -/// The smaller of two recorded failure probabilities; an unrecorded one -/// (`None`) yields to a recorded one. -fn min_recorded(a: Option, b: Option) -> Option { - match (a, b) { - (Some(a), Some(b)) => Some(a.min(b)), - (a, b) => a.or(b), - } -} - /// Accuracy-dependent candidate budget, not a completeness theorem. The /// membership model must still certify the selected set independently. pub(crate) fn topk_capacity(k: usize, eps: f64) -> u32 { @@ -413,182 +264,6 @@ pub(crate) fn saturating_ceil(x: f64, lo: u32, hi: u32) -> u32 { (x.ceil() as u32).clamp(lo, hi) } -#[cfg(test)] -mod shared_size_tests { - use super::*; - - /// The shared size takes each parameter's largest value, and the - /// smallest where smaller is stricter (KLL δ, DDSketch α); sizes of - /// different algorithms have none. - #[test] - fn shared_size_takes_each_parameters_strictest_value() { - assert_eq!( - size_for_all_readers(&[&univmon(256, 5, 1024, 16), &univmon(1, 7, 4096, 1)]), - Some(univmon(256, 7, 4096, 16)) - ); - // The distinct count's failure probability, where one was sized - // for, takes the smallest. - let distinct = |heap_size, delta| SketchParams::UnivMon { - heap_size, - sketch_rows: 1, - sketch_cols: 1, - layers: 19, - distinct_failure_probability: Some(delta), - }; - let Some(SketchParams::UnivMon { - heap_size, - distinct_failure_probability, - .. - }) = size_for_all_readers(&[ - &univmon(256, 5, 1024, 16), - &distinct(4096, 0.01), - &distinct(2048, 0.001), - ]) - else { - unreachable!() - }; - // The heap of 4096 sized at δ = 0.01 grows to the shared δ = 0.001 - // at its own relative error. - assert_eq!( - (heap_size, distinct_failure_probability), - (40_961, Some(0.001)) - ); - let kll = |k, failure_probability| SketchParams::Kll { - k, - failure_probability, - }; - let SketchParams::Kll { - k, - failure_probability, - } = size_for_all_readers(&[ - &kll(200, None), - &kll(100, Some(0.001)), - &kll(150, Some(0.005)), - ]) - .unwrap() - else { - unreachable!() - }; - // The `k = 200` reader, sized at the calibration's δ = 0.01, is - // resized to the shared δ = 0.001 at its own rank error. - assert_eq!((k, failure_probability), (258, Some(0.001))); - assert_eq!( - size_for_all_readers(&[ - &SketchParams::DDSketch { alpha: 0.01 }, - &SketchParams::DDSketch { alpha: 0.001 } - ]), - Some(SketchParams::DDSketch { alpha: 0.001 }) - ); - assert_eq!( - size_for_all_readers(&[ - &kll(200, None), - &SketchParams::Hll { - precision: 12, - relative_error: None - } - ]), - None - ); - assert_eq!(size_for_all_readers(&[]), None); - } - - /// A KLL shared by a reader at rank (0.01, δ = 0.01) and one at - /// (0.05, δ = 1e-4) meets both: each reader's `k` is resized to the - /// shared δ = 1e-4 at no larger rank error (A's 0.00997) before the - /// larger is taken (k = 412), not 269 stated at 1e-4 (rank error 0.0151). - #[test] - fn a_kll_shared_across_failure_probabilities_meets_each_reader() { - let readers = [(0.01, 0.01), (0.05, 1e-4)]; - let params = readers.map(|(eps, delta)| kll::size_params(eps, delta)); - let shared = size_for_all_readers(&[¶ms[0], ¶ms[1]]).unwrap(); - let quantile = SketchStatistic::Quantile { q: 0.99 }; - let guarantee = sketch_guarantee(&SketchAlgorithm::Kll, &shared, &quantile).unwrap(); - for (eps, delta) in readers { - assert!( - satisfies( - &guarantee, - &AccuracyTarget::max_error(eps) - .failure_probability(delta) - .accuracy_metric(AccuracyMetric::Rank) - .accuracy_scope(asap_types::ir::properties::AccuracyScope::PerValue) - ), - "{shared:?} misses ({eps}, {delta})" - ); - } - assert_eq!( - shared, - SketchParams::Kll { - k: 412, - failure_probability: Some(1e-4) - } - ); - } - - fn univmon(heap_size: u32, sketch_rows: u32, sketch_cols: u32, layers: u8) -> SketchParams { - SketchParams::UnivMon { - heap_size, - sketch_rows, - sketch_cols, - layers, - distinct_failure_probability: None, - } - } - - /// Each UnivMon fits `asap_sketchlib`'s hash budget, but the largest - /// rows of one with the largest columns of the other do not - /// (`12 · (⌈log2 2^16⌉ + 1) = 204 > 128`): the library would not build - /// the shared shape, so there is none. - #[test] - fn no_shared_univmon_past_the_hash_budget() { - assert!(univmon::hash_independent(12, 512, 1)); - assert!(univmon::hash_independent(5, 1 << 16, 1)); - assert_eq!( - size_for_all_readers(&[&univmon(1, 12, 512, 1), &univmon(1, 5, 1 << 16, 1)]), - None - ); - } - - /// A distinct-count UnivMon shared by a reader at relative (0.05, - /// δ = 0.01) and one at (0.2, δ = 1e-4) meets both: each reader's heap - /// grows to the shared δ = 1e-4 at no larger error before the larger is - /// taken (5 400 000), not B's 337 500 stated at 1e-4 (error 0.2). - #[test] - fn a_distinct_univmon_shared_across_failure_probabilities_meets_each_reader() { - use asap_types::ir::operator::agg_intent::default_cardinality; - let readers = [(0.05, 0.01), (0.2, 1e-4)]; - let params = readers.map(|(eps, delta)| { - size_params(SketchAlgorithm::UnivMon, &default_cardinality(), eps, delta) - }); - let shared = size_for_all_readers(&[¶ms[0], ¶ms[1]]).unwrap(); - let guarantee = univmon::distinct_guarantee(&shared, "m").unwrap(); - let error = guarantee.spec.error.evaluate().unwrap(); - // The top-layer term stays symbolic in the source's distinct bound; - // the Chebyshev term is the share of δ the heap was sized for. - let ProbabilityExpr::UnionBound { terms } = &guarantee.spec.failure_probability else { - panic!("{guarantee:?}") - }; - let chebyshev = terms[0].evaluate().unwrap(); - for (eps, delta) in readers { - assert!( - error <= eps && chebyshev <= delta, - "{shared:?} misses ({eps}, {delta}): error {error}" - ); - } - let SketchParams::UnivMon { - heap_size, - distinct_failure_probability, - .. - } = shared - else { - unreachable!() - }; - assert_eq!( - (heap_size, distinct_failure_probability), - (5_400_000, Some(1e-4)) - ); - } -} - #[cfg(test)] mod tests { use super::*; @@ -701,25 +376,13 @@ mod tests { #[test] fn local_guarantee_needs_a_statistic_the_family_answers() { use asap_types::ir::scalar::expr_ir::ColumnRef; - use asap_types::ir::schema::{ - DataType, ExactKind, ExactParams, GroupingStrategy, SketchKind, - }; - let hll = FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Hll, - SketchParams::Hll { - precision: 12, - relative_error: None, - }, - ), - GroupingStrategy::default(), - ); + use asap_types::ir::schema::{DataType, ExactKind, ExactParams, GroupingStrategy}; + let hll = FieldDataType::Sketch(SketchAlgorithm::Hll, GroupingStrategy::default()); let total = SketchStatistic::PointCount { key: ColumnRef::SampleValue, value: None, }; assert!(local_guarantee(&hll, &total).is_none()); - assert!(local_guarantee(&hll, &SketchStatistic::Cardinality).is_some()); for exact in [ FieldDataType::Plain(DataType::Float64), FieldDataType::ExactAggregate(ExactKind::Count, ExactParams::Count), diff --git a/crates/accuracy/src/estimators/univmon.rs b/crates/accuracy/src/estimators/univmon.rs index c9378c1d0..41b694d1a 100644 --- a/crates/accuracy/src/estimators/univmon.rs +++ b/crates/accuracy/src/estimators/univmon.rs @@ -213,6 +213,7 @@ fn distinct_heap(eps: f64, delta: f64) -> u32 { /// The statistic a UnivMon distinct count over `source` leaves its /// top-layer failure probability as. +#[allow(dead_code)] // Read again in #1207 PR 4, with θ from `LogicalProperties::parameters`. pub(crate) fn incomplete_top_layer_statistic(heap_size: u32, layers: u8, source: &str) -> String { format!("{INCOMPLETE_TOP_LAYER}[k={heap_size},layers={layers}]({MAX_DISTINCT}[{source}])") } @@ -257,6 +258,7 @@ pub fn incomplete_top_layer_probability(heap_size: u32, layers: u8, max_distinct /// `δ_c` plus the chance of an incomplete top layer, left symbolic in the /// source's distinct bound. `None` for a shape that was not sized for a /// distinct count, a heap under 8, or a shape the library would not build. +#[allow(dead_code)] // Read again in #1207 PR 4, with θ from `LogicalProperties::parameters`. pub(crate) fn distinct_guarantee(params: &SketchParams, source: &str) -> Option { let SketchParams::UnivMon { heap_size, @@ -368,45 +370,6 @@ pub(super) fn size_params(intent: &AggIntent, eps: f64, delta: f64) -> SketchPar } } -/// The heap of one UnivMon shared by two readers with heaps `heap_size` -/// and `heap_size2`, sized for a distinct count at the failure -/// probabilities they record (`None`: not sized for one): each distinct -/// reader's heap grown to the smaller failure probability at no larger -/// relative error ([`with_failure_probability`]), then the larger. The -/// shared build states its distinct guarantee at that smaller δ, where a -/// heap sized for a larger δ has a larger error, so the larger heap alone -/// could miss its reader's ε. A reader not sized for a distinct count keeps -/// its heap. -pub(crate) fn shared_heap( - heap_size: u32, - distinct_failure_probability: Option, - heap_size2: u32, - distinct_failure_probability2: Option, -) -> u32 { - let delta = match (distinct_failure_probability, distinct_failure_probability2) { - (Some(a), Some(b)) => a.min(b), - (a, b) => match a.or(b) { - Some(delta) => delta, - None => return heap_size.max(heap_size2), - }, - }; - let resized = |heap_size, distinct_failure_probability| match with_failure_probability( - &SketchParams::UnivMon { - heap_size, - sketch_rows: 1, - sketch_cols: 1, - layers: 1, - distinct_failure_probability, - }, - delta, - ) { - SketchParams::UnivMon { heap_size, .. } => heap_size, - _ => heap_size, - }; - resized(heap_size, distinct_failure_probability) - .max(resized(heap_size2, distinct_failure_probability2)) -} - #[cfg(test)] mod tests { use super::*; diff --git a/crates/accuracy/src/propagation.rs b/crates/accuracy/src/propagation.rs index ed33978df..11b5d8c02 100644 --- a/crates/accuracy/src/propagation.rs +++ b/crates/accuracy/src/propagation.rs @@ -7,7 +7,7 @@ use std::collections::{HashMap, HashSet}; use std::rc::Rc; use asap_types::ir::operator::operator_properties::{BinaryOpKind, JoinKind, Reduction, Source}; -use asap_types::ir::operator::tumbling_window_chain::{merged_chain, merged_window_build}; +use asap_types::ir::operator::tumbling_window_chain::merged_chain; use asap_types::ir::operator::AggIntent; use asap_types::ir::properties::guarantee::{ AccuracyGuarantee, AccuracyMetric, BoundExpr, CompositionOperator, GuaranteeSource, @@ -18,8 +18,8 @@ use asap_types::ir::scalar::{resolve_column_ref, ColumnRef, CompareOpKind, Scala use asap_types::ir::schema::{DataType, Schema}; use asap_types::ir::schema::{ ExactKind, FamilyCapability, FieldDataType, GroupingStrategy, MergedError, - NonNegativeWeightProof, SketchAlgorithm, SketchKind, SketchParams, SketchStatistic, - SummaryInputExpr, SummaryUpdate, WeightDomain, + NonNegativeWeightProof, SketchAlgorithm, SketchStatistic, SummaryInputExpr, SummaryUpdate, + WeightDomain, }; use asap_types::ir::{ ASAPOp, BinaryOperator, NonASAPOp, Operator, OperatorNode, ProjectItem, QueryRoot, ScalarExpr, @@ -31,7 +31,7 @@ use crate::composition::{ PRODUCT_OPERAND_DOMAINS, QUOTIENT_OPERAND_DOMAINS, }; use crate::estimators::local_guarantee; -use crate::estimators::{hll, univmon}; + use crate::function_rules::approximate_input_composition; use crate::PropagationStats; @@ -122,7 +122,7 @@ pub fn root_guarantee(root: &QueryRoot) -> Option { /// Bind known statistics in existing contracts, then recover compositions /// that previously required constant child bounds. Physical state contracts, -/// schemas, timing, materialization, and sharing are preserved. This does not +/// schemas and sharing are preserved. This does not /// infer facts: `bind` must leave unsupported statistics unknown. pub fn bind_guarantees( root: &QueryRoot, @@ -137,9 +137,7 @@ pub fn bind_guarantees( return done.clone(); } let operator = node.operator.map_children(|child| walk(child, bind, cache)); - let mut rebuilt = OperatorNode::with_schema(operator, node.schema.clone()) - .with_timing(node.timing) - .with_materialization(node.materialization); + let mut rebuilt = OperatorNode::with_schema(operator, node.schema.clone()); rebuilt.guarantee = node .guarantee .as_ref() @@ -159,90 +157,6 @@ pub fn bind_guarantees( root_guarantee(&bound) } -/// `roots` with every sketch summary state rebuilt with the parameters -/// `resize` gives it (same algorithm; `None` keeps them), and every node -/// above one rebuilt over its new inputs with its guarantee composed again -/// ([`composed_guarantee`]; a node that had none keeps none). Timing, -/// materialization and sharing are kept. Pass 1 sizes summaries -/// for a failure probability this way. -pub fn with_resized_summaries( - roots: &[QueryRoot], - resize: &dyn Fn(&OperatorNode, &SketchParams) -> Option, -) -> Result, SchemaDerivationError> { - fn walk( - node: &Rc, - resize: &dyn Fn(&OperatorNode, &SketchParams) -> Option, - cache: &mut NodeToRewrittenNodeCache, - ) -> Result, SchemaDerivationError> { - if let Some(done) = cache.get(&Rc::as_ptr(node)) { - return Ok(done.clone()); - } - for child in node.children() { - walk(child, resize, cache)?; - } - let mut changed = node - .children() - .iter() - .any(|child| !Rc::ptr_eq(child, &cache[&Rc::as_ptr(child)])); - let mut operator = node.operator.clone(); - if let Operator::ASAP( - ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), - .. - } - | ASAPOp::SummaryTumblingWindowChain { - family: FieldDataType::Sketch(kind, _), - .. - }, - ) = &mut operator - { - if let Some(params) = resize(node, kind.params()).filter(|p| p != kind.params()) { - *kind = SketchKind::new(*kind.algorithm(), params); - changed = true; - } - } - let rebuilt = if changed { - let template = OperatorNode { - operator, - ..(**node).clone() - }; - let mut rebuilt = - template.with_new_children(|child| cache[&Rc::as_ptr(child)].clone())?; - rebuilt.guarantee = node - .guarantee - .as_ref() - .and_then(|_| composed_guarantee(&rebuilt)); - Rc::new( - rebuilt - .with_timing(node.timing) - .with_materialization(node.materialization), - ) - } else { - node.clone() - }; - cache.insert(Rc::as_ptr(node), rebuilt.clone()); - Ok(rebuilt) - } - let mut cache = HashMap::new(); - roots - .iter() - .map(|root| { - Ok(match root { - QueryRoot::Operator(node) => QueryRoot::Operator(walk(node, resize, &mut cache)?), - QueryRoot::Scalar(expr) => { - let mut rebuilt = HashMap::new(); - for node in expr.operator_refs() { - rebuilt.insert(Rc::as_ptr(node), walk(node, resize, &mut cache)?); - } - QueryRoot::Scalar( - expr.map_operator_refs(&mut |node| rebuilt[&Rc::as_ptr(node)].clone()), - ) - } - }) - }) - .collect() -} - /// The one source a summary state counts the distinct items of its own /// rows of, as `source_distinct_guarantee` reads it: the source whose /// distinct bound gives its failure probability. `None` for a state over @@ -583,7 +497,8 @@ fn rolled_up_guarantee(node: &OperatorNode) -> Option> return Some(None); }; // Kept whole, a selected state keeps the guarantee it was built with. - let input = asap_types::ir::operator::asap::selected_from(input).unwrap_or(input); + let input = + asap_types::ir::operator::asap::selected_from(input).unwrap_or_else(|| Rc::clone(input)); let family = state.asap()?.produced_state()?; let merged = match node.asap()? { ASAPOp::SummaryEstimate { query, .. } => FamilyCapability::of(family).merged_error(query), @@ -596,44 +511,43 @@ fn rolled_up_guarantee(node: &OperatorNode) -> Option> ) { return Some(None); } - let reader = OperatorNode::with_schema( + let consumer = OperatorNode::with_schema( node.operator.map_children(|child| { if Rc::ptr_eq(child, state) { - Rc::clone(input) + Rc::clone(&input) } else { Rc::clone(child) } }), node.schema.clone(), ); - Some(composed_guarantee(&reader)) + Some(composed_guarantee(&consumer)) } /// The guarantee of an estimate or finalized value read from merged /// tumbling windows ([`merged_chain`]): the guarantee of the same read of one -/// build over the merged tumbling windows' time range ([`merged_window_build`]: the whole -/// chain's, or its newest tumbling windows') when the +/// build over the merged tumbling windows' time range (the whole chain's, or +/// its newest tumbling windows') when the /// family's merged error for that read is exact or the same as that build's /// (the family-capability table); none when it is approximate (evidence /// only, for plan selection to judge) or the family has no merge. `None` for /// any node that does not read merged tumbling windows. fn merged_tumbling_windows_guarantee(node: &OperatorNode) -> Option> { - let (state, merged) = match node.asap()? { + let merged = match node.asap()? { ASAPOp::SummaryEstimate { summary_input, query, } => { - let family = merged_chain(summary_input)?.asap()?.produced_state()?; - ( - summary_input, - FamilyCapability::of(family).merged_error(query), - ) + let chain = merged_chain(summary_input)?; + let family = chain.asap()?.produced_state()?; + FamilyCapability::of(family).merged_error(query) } ASAPOp::FinalizeExactAccumulator { child } => { - let family = merged_chain(child)?.asap()?.produced_state()?; + let chain = merged_chain(child)?; + let family = chain.asap()?.produced_state()?; // An exact accumulator's merged state is exact. let merged = FamilyCapability::of(family).merge_inputs; - (child, merged.map(|_| MergedError::Exact)) + merged.map(|_| MergedError::Exact) } _ => return None, }; @@ -643,33 +557,9 @@ fn merged_tumbling_windows_guarantee(node: &OperatorNode) -> Option, ) -> Option { - let FieldDataType::Sketch(kind, GroupingStrategy::PerSubpopulationInstance) = family else { + let FieldDataType::Sketch(algorithm, GroupingStrategy::PerSubpopulationInstance) = family + else { return None; }; if *query != SketchStatistic::Cardinality { return None; } source_scan(input)?; - match kind.params() { - SketchParams::Hll { - precision, - relative_error, - } => hll::classic_guarantee(*precision, *relative_error, &source(input)?), - params @ SketchParams::UnivMon { .. } => { - univmon::distinct_guarantee(params, &source(input)?) + match algorithm { + SketchAlgorithm::Hll | SketchAlgorithm::UnivMon => { + todo!( + "#1207 PR 4: the distinct-count guarantee of θ from LogicalProperties::parameters" + ) } _ => None, } @@ -779,7 +668,7 @@ pub(crate) fn estimates_raw_data_samples_of(node: &OperatorNode) -> Option match merged_chain(summary_input) - .unwrap_or(summary_input) + .unwrap_or_else(|| Rc::clone(summary_input)) .asap()? { ASAPOp::SummaryAgg { @@ -1219,10 +1108,10 @@ pub(crate) fn stream_norm(node: &Rc) -> Option { matches!( node.asap(), Some(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), + family: FieldDataType::Sketch(algorithm, _), .. }) if matches!( - kind.algorithm(), + algorithm, SketchAlgorithm::Cms | SketchAlgorithm::CmsWithHeap | SketchAlgorithm::CountSketch @@ -1598,8 +1487,8 @@ pub(crate) fn estimate_has_bound( ) -> bool { let count_min = matches!( family, - FieldDataType::Sketch(kind, _) - if matches!(kind.algorithm(), SketchAlgorithm::Cms | SketchAlgorithm::CmsWithHeap) + FieldDataType::Sketch(algorithm, _) + if matches!(algorithm, SketchAlgorithm::Cms | SketchAlgorithm::CmsWithHeap) ); !matches!(query, SketchStatistic::TopK { .. }) && !(count_min && !matches!(update.weight_domain, WeightDomain::NonNegative { .. })) @@ -1616,8 +1505,7 @@ fn interpolation_guarantee( input: &OperatorNode, mut guarantee: AccuracyGuarantee, ) -> Option { - let interpolates = matches!(family, FieldDataType::Sketch(kind, _) - if *kind.algorithm() == SketchAlgorithm::DDSketch) + let interpolates = matches!(family, FieldDataType::Sketch(SketchAlgorithm::DDSketch, _)) && matches!(query, SketchStatistic::Quantile { q } if *q > 0.0 && *q < 1.0); if !interpolates || matches!(update.weight_domain, WeightDomain::NonNegative { .. }) { return Some(guarantee); diff --git a/crates/accuracy/src/sizing.rs b/crates/accuracy/src/sizing.rs index 4960f3d14..cdda5845c 100644 --- a/crates/accuracy/src/sizing.rs +++ b/crates/accuracy/src/sizing.rs @@ -3,21 +3,11 @@ //! family sized for one equal share of that target, and the failure //! probability each approximate input needs over the values the target //! covers. -use std::rc::Rc; use asap_types::ir::operator::AggIntent; use asap_types::ir::properties::{AccuracyMetric, AccuracyTarget}; use asap_types::ir::scalar::ColumnRef; -use asap_types::ir::schema::{ - FieldDataType, GroupingStrategy, SketchAlgorithm, SketchKind, SketchParams, SketchStatistic, -}; -use asap_types::ir::{OperatorNode, QueryRoot, SchemaDerivationError}; -use asap_types::workload::bounds::{series_read, values_per_result}; -use asap_types::workload::DataWorkload; - -use crate::estimators::{hll, local_guarantee, with_failure_probability}; -use crate::per_input_target; -use crate::propagation::{approximate_inputs, distinct_bound_source, with_resized_summaries}; +use asap_types::ir::schema::{SketchAlgorithm, SketchParams, SketchStatistic}; /// Confidence δ assumed when the target carries only an ε /// (`AccuracyTarget::epsilon`): the (ε, δ)-parameterised sketches (CMS) need @@ -99,6 +89,7 @@ pub fn accuracy_budget(accuracy: &AccuracyTarget) -> (f64, f64) { /// smallest parameter satisfying the target, clamped to the family's sane /// range. A non-positive ε saturates to the clamp maximum (tightest /// allowed). +#[allow(dead_code)] // Read by `test_support` until #1207 PR 1 replaces it with `parameter_list`. pub(crate) fn default_size_params( kind: SketchAlgorithm, intent: &AggIntent, @@ -108,175 +99,6 @@ pub(crate) fn default_size_params( crate::estimators::size_params(kind, intent, eps, delta) } -/// `algorithm` sized for one of `share` equal shares of `intent`'s accuracy -/// requirement ([`per_input_target`]), split in the metric of the summary's own -/// bound: a relative error splits so products and quotients of the shares -/// meet it. -pub fn sized(algorithm: &SketchAlgorithm, intent: &AggIntent, share: usize) -> SketchKind { - let target = sizing_target(intent).unwrap_or_else(AccuracyTarget::exact); - let size = |target: &AccuracyTarget| { - let (epsilon, delta) = accuracy_budget(target); - SketchKind::new( - *algorithm, - default_size_params(*algorithm, intent, epsilon, delta), - ) - }; - let whole = size(&target); - if share <= 1 { - return whole; - } - let metric = sketch_statistic(intent) - .and_then(|query| { - local_guarantee( - &FieldDataType::Sketch(whole.clone(), GroupingStrategy::default()), - &query, - ) - }) - .map(|guarantee| guarantee.spec.accuracy_metric) - .or(target.accuracy_metric); - // The split needs a concrete metric. A summary without an error model - // whose intent's target sets none is not split (never a guessed - // metric): it stays sized for the whole target, and Stage 3 rejects the - // candidate if its composed output then misses the query's target. - let Some(metric) = metric else { - debug_assert!( - target.is_exact(), - "sizing {algorithm:?} for a share of an approximate target without a metric" - ); - return whole; - }; - per_input_target(&target, metric, share).map_or(whole, |share| size(&share)) -} - -/// The failure probability each of the `budget_parts` approximate inputs -/// of query `root`'s value is sized for, so that the value meets `target` -/// over every value the target covers: `δ / (n · V)`, the query's `δ` -/// shared equally by `n = budget_parts` inputs and union-bounded over the -/// `V` values its scope covers -/// ([`AccuracyScope::max_values_covered`](asap_types::ir::properties::AccuracyScope::max_values_covered): -/// 1 for a per-value target, at most `⌊1 / share⌋` for one over the groups -/// holding a record share, and `G` ([`values_per_result`]) for the whole -/// result). `None` for an exact target, one without a positive failure -/// probability, and a whole-result target whose `G` is unbound: its target -/// then stays unmet, and plan selection rejects the plan. -pub fn per_input_failure_probability( - target: &AccuracyTarget, - budget_parts: usize, - root: &QueryRoot, - data: &DataWorkload, -) -> Option { - let delta = target - .failure_probability - .filter(|delta| !target.is_exact() && *delta > 0.0)?; - let values = target.accuracy_scope.max_values_covered(values_per_result( - root, - series_read(root, data), - data, - ))?; - Some(delta / (budget_parts.max(1) as f64 * values as f64)) -} - -/// Sketch summary `state`'s `params` sized for failure probability -/// `failure_probability`: the parameters that depend on it enlarged -/// ([`with_failure_probability`]), and classic HLL's precision from the -/// distinct bound of the source it counts (keeping the relative error it -/// was sized for). Never smaller than `params`. `None` keeps `params`: an -/// HLL whose source declares no distinct bound. -fn sized_for_failure_probability( - state: &OperatorNode, - params: &SketchParams, - failure_probability: f64, - data: &DataWorkload, -) -> Option { - Some(match params { - SketchParams::Hll { - precision, - relative_error, - } => { - let source = distinct_bound_source(state)?; - let bound = data.sources.get(&source)?.max_distinct_per_evaluation?; - let relative_error = hll::classic_relative_error(*precision, *relative_error); - SketchParams::Hll { - precision: hll::min_precision( - relative_error, - failure_probability, - bound, - *precision, - ), - relative_error: Some(relative_error), - } - } - params => with_failure_probability(params, failure_probability), - }) -} - -/// `roots` with every approximate input of their values -/// ([`approximate_inputs`]) sized for `failure_probability` -/// (`sized_for_failure_probability`) and the guarantees above composed -/// again ([`with_resized_summaries`]). A summary whose output carries no -/// guarantee (e.g. a top-k's heap, which bounds item frequencies, not -/// membership) is no approximate input and keeps its size: its failure -/// probability bounds nothing the target checks. The same `Rc` where -/// nothing changes. -pub fn with_summaries_sized_for_failure_probability( - roots: &[QueryRoot], - failure_probability: f64, - data: &DataWorkload, -) -> Result, SchemaDerivationError> { - let inputs: std::collections::HashSet<*const OperatorNode> = roots - .iter() - .flat_map(approximate_inputs) - .map(|state| Rc::as_ptr(&state)) - .collect(); - if inputs.is_empty() { - return Ok(roots.to_vec()); - } - with_resized_summaries(roots, &|state, params| { - inputs - .contains(&std::ptr::from_ref(state)) - .then(|| sized_for_failure_probability(state, params, failure_probability, data)) - .flatten() - }) -} - -/// Whole-workload `roots` (query `i`'s root `roots[i]` with target -/// `targets[i]`), composed outside the memo, with their summaries sized as -/// Pass 1 sizes a memo candidate: each approximate input of query `i`'s -/// value, `n_i` in all ([`approximate_inputs`]), for -/// [`per_input_failure_probability`] `δ_i / (n_i · V_i)`, a summary several -/// roots read for the smallest among them, and the guarantees above -/// composed again. For plans built by hand (tests, the fuzzer's witnesses); -/// the planner sizes its candidates in Pass 1. -pub fn with_summaries_sized_for_targets( - roots: &[QueryRoot], - targets: &[AccuracyTarget], - data: &DataWorkload, -) -> Result, SchemaDerivationError> { - let mut required: std::collections::HashMap<*const OperatorNode, f64> = - std::collections::HashMap::new(); - for (root, target) in roots.iter().zip(targets) { - let inputs = approximate_inputs(root); - let Some(failure_probability) = - per_input_failure_probability(target, inputs.len(), root, data) - else { - continue; - }; - for state in &inputs { - let entry = required - .entry(Rc::as_ptr(state)) - .or_insert(failure_probability); - *entry = entry.min(failure_probability); - } - } - if required.is_empty() { - return Ok(roots.to_vec()); - } - with_resized_summaries(roots, &|state, params| { - let failure_probability = *required.get(&std::ptr::from_ref(state))?; - sized_for_failure_probability(state, params, failure_probability, data) - }) -} - #[cfg(test)] mod tests { use super::*; diff --git a/crates/cost/Cargo.toml b/crates/cost/Cargo.toml index 1436c8114..8a8e35fa9 100644 --- a/crates/cost/Cargo.toml +++ b/crates/cost/Cargo.toml @@ -22,4 +22,4 @@ asap-executor = { path = "../executor" } # by the stages above this crate; `tests/held_memory.rs` builds them with # the reference pricer's `optimize_physical_asap`. asap-logical-asap-optimization = { path = "../logical-asap-optimization", features = ["test-support"] } -asap-physical-asap-optimization = { path = "../physical-asap-optimization", features = ["reference-pricer"] } +asap-physical-asap-optimization = { path = "../physical-asap-optimization" } diff --git a/crates/cost/src/execution_demand.rs b/crates/cost/src/execution_demand.rs index 6a7e380bd..d966d03bd 100644 --- a/crates/cost/src/execution_demand.rs +++ b/crates/cost/src/execution_demand.rs @@ -3,74 +3,75 @@ //! This preserves the configured accounting policy, not an exact timestamp //! union: different fixed intervals add, unknown phases join a known phase, //! estimated rates add, and one-off/scheduled demand uses a batch maximum. -//! Call `schedule_groups` or `evaluation_rate` on the chosen reader subset +//! Call `schedule_groups` or `evaluation_rate` on the chosen query subset //! only; neither enumerates subsets. For retained query-time state pass `None` for each range evaluation //! to price once per request, as `computations` does. use super::analytical::AnalyticalCostError; use asap_types::ir::physical_export::{PhysicalASAPDAG, PhysicalASAPNodeId}; -use asap_types::physical::Materialization; +use asap_types::ir::properties::ExecutionTiming; +use asap_types::physical::Lifetime; use asap_types::workload::{QueryRecurrence, RangeEvaluation, RepeatedDemand}; use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; /// How a [`ScheduleGroup`]'s evaluations add to the others'. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum Schedule { - /// Repeating readers on one interval and phase: evaluated together, + /// Repeating queries on one interval and phase: evaluated together, /// once per distinct instant their steps fall on. FixedInterval, - /// One reader with an estimated rate: its rate times its steps. + /// One query with an estimated rate: its rate times its steps. EstimatedRate, - /// The one-off, scheduled and unknown readers, run as one batch: the + /// The one-off, scheduled and unknown queries, run as one batch: the /// most evaluations among them, spread over the horizon. NonRepeating, } -/// Readers of one node that are evaluated together, and how often the node +/// Queries of one node that are evaluated together, and how often the node /// is computed for them. Groups add ([`evaluation_rate`]). #[derive(Debug, Clone, PartialEq)] pub struct ScheduleGroup { - /// The readers in the group, as positions in the reader list given to + /// The queries in the group, as positions in the query list given to /// [`schedule_groups`], ascending. - pub readers: Vec, - /// How often the node is computed per second for these readers. + pub queries: Vec, + /// How often the node is computed per second for these queries. pub evaluations_per_second: f64, /// Which kind of group it is. pub schedule: Schedule, } -/// One fixed (interval, phase) schedule's readers: each step in ms they -/// use with the longest span in ms at that step, and the readers. +/// One fixed (interval, phase) schedule's queries: each step in ms they +/// use with the longest span in ms at that step, and the queries. type FixedSchedule = (BTreeMap, Vec); -/// The groups of `readers` evaluated together (each a recurrence with its +/// The groups of `queries` evaluated together (each a recurrence with its /// range evaluation), in the order [`evaluation_rate`] adds them: fixed /// intervals by (interval, phase), then each estimated rate, then the /// non-repeating batch. A request with a range evaluation is /// [`RangeEvaluation::evaluations_per_request`] evaluations, one without it -/// one. Repeating readers with equal intervals and equal phases form one +/// one. Repeating queries with equal intervals and equal phases form one /// group, every request ending at the same instant: a node they share is /// computed once per distinct instant their steps fall on, -/// `end − k · step_ms` for `k · step_ms ≤ span_ms`. A reader of unknown +/// `end − k · step_ms` for `k · step_ms ≤ span_ms`. A query of unknown /// phase joins the group of the first known phase on its interval, if any. -/// A reader evaluated once per request (no range, or one step) evaluates +/// A query evaluated once per request (no range, or one step) evaluates /// only at the end. Different intervals or phases are different groups (an /// over-estimate for intervals whose instants coincide, e.g. every 20 s and /// every 30 s meet every 60 s); each estimated rate is its own group. -/// One-off, scheduled and unknown readers form one batch group: the most +/// One-off, scheduled and unknown queries form one batch group: the most /// evaluations among them, spread over `non_repeating_query_horizon_seconds`. pub fn schedule_groups<'r>( - readers: impl IntoIterator)>, + queries: impl IntoIterator)>, non_repeating_query_horizon_seconds: f64, ) -> Result, AnalyticalCostError> { // Per interval in ms and phase in ms on it (`None`: unknown), each step - // in ms its readers use and the longest span in ms at that step, with - // the readers; once per request is the interval with span 0. + // in ms its queries use and the longest span in ms at that step, with + // the queries; once per request is the interval with span 0. let mut by_schedule: BTreeMap<(u32, Option), FixedSchedule> = BTreeMap::new(); let mut estimated = Vec::new(); let mut invocations = 0u64; let mut non_repeating = Vec::new(); - for (reader, (recurrence, range_evaluation)) in readers.into_iter().enumerate() { + for (query, (recurrence, range_evaluation)) in queries.into_iter().enumerate() { let steps = range_evaluation.map_or(1, |range| range.evaluations_per_request()); match recurrence { QueryRecurrence::Repeated( @@ -96,7 +97,7 @@ pub fn schedule_groups<'r>( by_schedule.entry((interval.0, phase_ms)).or_default(); let longest_span_ms = longest_span_ms_by_step_ms.entry(step_ms).or_default(); *longest_span_ms = (*longest_span_ms).max(span_ms); - members.push(reader); + members.push(query); } QueryRecurrence::Repeated(RepeatedDemand::EstimatedRate(estimate)) => { let rate = estimate.expected_rate.0; @@ -104,28 +105,28 @@ pub fn schedule_groups<'r>( return Err(AnalyticalCostError::InvalidRecurrence); } estimated.push(ScheduleGroup { - readers: vec![reader], + queries: vec![query], evaluations_per_second: rate * steps as f64, schedule: Schedule::EstimatedRate, }); } QueryRecurrence::Repeated(RepeatedDemand::Scheduled(times)) => { invocations = invocations.max(times.len() as u64 * steps); - non_repeating.push(reader); + non_repeating.push(query); } QueryRecurrence::OneTime { invocations: count, .. } => { invocations = invocations.max(*count * steps); - non_repeating.push(reader); + non_repeating.push(query); } QueryRecurrence::Unknown => { invocations = invocations.max(steps); - non_repeating.push(reader); + non_repeating.push(query); } } } - // Readers of unknown phase run with the first known phase on their + // Queries of unknown phase run with the first known phase on their // interval. let unknown_phase_intervals_ms: Vec = by_schedule .keys() @@ -152,8 +153,8 @@ pub fn schedule_groups<'r>( let mut groups: Vec = by_schedule .into_iter() .map( - |((interval_ms, _), (longest_span_ms_by_step_ms, readers))| ScheduleGroup { - readers, + |((interval_ms, _), (longest_span_ms_by_step_ms, queries))| ScheduleGroup { + queries, evaluations_per_second: 1_000.0 / f64::from(interval_ms) * distinct_instants_per_request(&longest_span_ms_by_step_ms) as f64, schedule: Schedule::FixedInterval, @@ -163,7 +164,7 @@ pub fn schedule_groups<'r>( groups.extend(estimated); if !non_repeating.is_empty() { groups.push(ScheduleGroup { - readers: non_repeating, + queries: non_repeating, evaluations_per_second: invocations as f64 / non_repeating_query_horizon_seconds, schedule: Schedule::NonRepeating, }); @@ -280,7 +281,7 @@ pub fn query_starts(dag: &PhysicalASAPDAG) -> Vec> { .collect() } -/// What computes a `NotMaterialized` node once per its own execution +/// What computes a not-materialized node once per its own execution /// ([`consumers`]). #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] pub enum Consumer { @@ -290,19 +291,19 @@ pub enum Consumer { Materialized(PhysicalASAPNodeId), } -/// The consumers of each `NotMaterialized` node of `dag` that any consumer +/// The consumers of each not-materialized node of `dag` that any consumer /// reaches ([`computations`]): the queries and the materialized nodes whose /// walk down their inputs reaches it before meeting a materialized node, /// each once; the queries first, in order, then the materialized nodes in /// `dag`'s node order. pub fn consumers(dag: &PhysicalASAPDAG) -> HashMap> { - let materialized: Vec = dag + let materialized_nodes: Vec = dag .nodes .iter() - .filter(|n| n.materialization != Materialization::NotMaterialized) + .filter(|n| n.lifetime != Lifetime::None) .map(|n| n.id) .collect(); - let is_materialized: HashSet = materialized.iter().copied().collect(); + let is_materialized: HashSet = materialized_nodes.iter().copied().collect(); let inputs = |id: PhysicalASAPNodeId| { dag.edges .iter() @@ -314,7 +315,7 @@ pub fn consumers(dag: &PhysicalASAPDAG) -> HashMap HashMap, ) -> Result, AnalyticalCostError> { let range_of = |query: usize| range_evaluations.get(query).copied().flatten(); - let materialized: HashMap<_, _> = dag + // Each materialized node's timing and lifetime. + let materialized_nodes: HashMap = dag .nodes .iter() - .filter(|n| n.materialization != Materialization::NotMaterialized) - .map(|n| (n.id, n.materialization)) + .filter(|node| node.lifetime != Lifetime::None) + .map(|node| (node.id, (node.output_state.timing, node.lifetime))) .collect(); let inputs = |id: PhysicalASAPNodeId| { dag.edges @@ -385,14 +387,12 @@ pub fn computations( } } let mut rate_of_materialized = HashMap::new(); - for (&id, materialization) in &materialized { - // A chain kept across evaluations is built once per request. - let per_request = matches!( - materialization, - Materialization::MatAtQueryKeptAcrossEvaluations(_) - ); - let rate = match materialization { - Materialization::MatAtIngestion(_) => 1.0, + for (&id, &(timing, lifetime)) in &materialized_nodes { + let at_ingestion = timing == ExecutionTiming::IngestionTime; + // A query-time chain materialized across evaluations is built once per request. + let per_request = !at_ingestion && matches!(lifetime, Lifetime::AcrossEvaluations(_)); + let rate = match at_ingestion { + true => 1.0, _ => evaluation_rate( reached.get(&id).into_iter().flatten().filter_map(|&q| { let range = if per_request { None } else { range_of(q) }; @@ -416,9 +416,9 @@ pub fn computations( } // The refresh window of each node refreshed at ingestion. let refresh_windows = dag.refresh_windows( - materialized + materialized_nodes .iter() - .filter(|(_, m)| matches!(m, Materialization::MatAtIngestion(_))) + .filter(|(_, (timing, _))| *timing == ExecutionTiming::IngestionTime) .map(|(&id, _)| id), ); let mut count: HashMap<_, (u64, f64)> = HashMap::new(); @@ -476,20 +476,20 @@ mod tests { // Unknown phases join the first known phase; separate known phases add. #[test] fn chosen_readers_preserve_phase_accounting() { - let readers = [ + let queries = [ fixed(20_000, None), fixed(20_000, Some(5_000)), fixed(20_000, Some(0)), ]; close( - evaluation_rate(readers[..2].iter().map(|r| (r, None)), 3600.0).unwrap(), + evaluation_rate(queries[..2].iter().map(|r| (r, None)), 3600.0).unwrap(), 1.0 / 20.0, ); close( - evaluation_rate(readers.iter().map(|r| (r, None)), 3600.0).unwrap(), + evaluation_rate(queries.iter().map(|r| (r, None)), 3600.0).unwrap(), 2.0 / 20.0, ); - // No global subset cache: each caller's selected readers determine the rate. + // No global subset cache: each caller's selected queries determine the rate. close(evaluation_rate(std::iter::empty(), 3600.0).unwrap(), 0.0); } @@ -521,9 +521,9 @@ mod tests { // Different fixed intervals remain additive even if timestamps can coincide. #[test] fn different_intervals_preserve_existing_conservative_rate() { - let readers = [fixed(20_000, Some(0)), fixed(30_000, Some(0))]; + let queries = [fixed(20_000, Some(0)), fixed(30_000, Some(0))]; close( - evaluation_rate(readers.iter().map(|r| (r, None)), 3600.0).unwrap(), + evaluation_rate(queries.iter().map(|r| (r, None)), 3600.0).unwrap(), 1.0 / 20.0 + 1.0 / 30.0, ); } diff --git a/crates/cost/src/lib.rs b/crates/cost/src/lib.rs index d381c59bb..27e7f5ade 100644 --- a/crates/cost/src/lib.rs +++ b/crates/cost/src/lib.rs @@ -41,7 +41,7 @@ pub use measured::{ pub use operator_statistics::EdgeStatistics; pub use ranking::{cheaper, worst_latency_ms, PlanRanking}; pub use usage::{ - family_name, kept_tumbling_window_chains, maintained_window_ms, node_map, node_usage, + family_name, maintained_window_ms, materialized_tumbling_window_chains, node_map, node_usage, plan_cost, query_reach, retained_tumbling_windows, trailing_window, tumbling_window_retention_ms, tumbling_window_retention_ms_of_plan, tumbling_windows_of, NodeMap, NodeUsage, ProducerOutput, @@ -65,7 +65,7 @@ pub const MIXED_COST_SOURCE: &str = pub(crate) const DEFAULT_NON_REPEATING_QUERY_HORIZON_SECONDS: f64 = 3_600.0; /// What one candidate uses, per second of wall time unless the name says -/// otherwise, without prices: its nodes' CPU, scans, kept memory and disk +/// otherwise, without prices: its nodes' CPU, scans, retained memory and disk /// added up, with the candidate's peak cores and per-query latency. Every [`CostModel`] reads /// it. #[derive(Debug, Clone, Default, PartialEq)] @@ -79,11 +79,11 @@ pub struct ResourceUsage { /// state while its evaluation runs). pub average_memory_bytes: f64, /// The most memory live at once, assuming every evaluation overlaps: - /// in-memory kept state plus each computation's working memory, once - /// per consumer that recomputes it, with the `MatAtIngestion` consumers + /// in-memory materialized state plus each computation's working memory, once + /// per consumer that recomputes it, with the consumers materialized at ingestion time /// of one refresh window counted once. pub peak_memory_bytes: u64, - /// Bytes kept on disk: state materialized on `Medium::Disk`, plus the + /// Bytes retained on disk: state materialized on `StorageMedium::Disk`, plus the /// raw data query-time scans read (the longest window of each source). pub disk_bytes: u64, /// CPU cores to provision for the peak: the ingestion work's cores plus @@ -115,7 +115,7 @@ pub struct ResourceUnitCost { impl ResourceUnitCost { /// The prices plan selection uses until a deployment is calibrated: - /// 1 ns of CPU per operation, 0.1 ns per scanned byte, 1 GB kept for a + /// 1 ns of CPU per operation, 0.1 ns per scanned byte, 1 GB retained for a /// second costs 1/8 vCPU (125 CPU-ms), held or provisioned alike; one /// provisioned core-second is 1,000 CPU-ms. Disk is not priced (0). pub fn illustrative() -> Self { @@ -375,7 +375,7 @@ pub struct NodeCost { /// node the query reads first. pub cpu_core_seconds_per_second: f64, /// The number of consumers it is computed for, with the - /// `MatAtIngestion` consumers of one refresh window counted once (1 + /// consumers materialized at ingestion time of one refresh window counted once (1 /// when materialized, [`execution_demand::computations`]). pub multiplier: u64, /// How it was measured, e.g. "scan 1000 samples; x 1.0000/s; keeps diff --git a/crates/cost/src/state_bytes.rs b/crates/cost/src/state_bytes.rs index a8d4c8fba..4b7d297e6 100644 --- a/crates/cost/src/state_bytes.rs +++ b/crates/cost/src/state_bytes.rs @@ -60,7 +60,10 @@ pub fn keeps_every_value(measure: &AggIntent) -> bool { /// 1 for every other sketch. pub fn summary_shape(family: &FieldDataType, input_rows: StateInputRows) -> (f64, u64) { match family { - FieldDataType::Sketch(kind, _) => sketch_shape(kind.params(), input_rows), + FieldDataType::Sketch(..) => { + let _ = input_rows; + todo!("#1207 PR 6: the state shape of θ from the candidate's LogicalProperties::parameters") + } FieldDataType::ExactAggregate(..) => (1.0, EXACT_ACCUMULATOR_STATE_BYTES), _ => (1.0, 8), } @@ -104,6 +107,7 @@ pub fn sketch_shape(params: &SketchParams, input_rows: StateInputRows) -> (f64, usize::try_from(rows.max(1)).unwrap_or(usize::MAX) }); match params { + SketchParams::Hydra(hydra) => sketch_shape(&hydra.per_subpopulation(), input_rows), SketchParams::Cms { width, depth } => ( f64::from(*depth), CountMinF64::estimated_memory_bytes(*depth as usize, *width as usize) as u64, @@ -270,7 +274,7 @@ const DDSKETCH_ASSUMED_VALUE_RANGE: f64 = 1e9; #[cfg(test)] mod tests { use super::*; - use asap_types::ir::schema::{GroupingStrategy, SketchAlgorithm, SketchKind}; + use asap_types::ir::schema::{GroupingStrategy, SketchAlgorithm}; /// An exact accumulator's state is priced at the executor's per-group /// state, 56 bytes for every kind (a counter's state alone is 48), so a @@ -353,121 +357,6 @@ mod tests { assert_eq!(priced(Some(10_000), false), priced(None, false)); } - /// DDSketch state grows as its relative accuracy α tightens. - #[test] - fn ddsketch_state_grows_as_alpha_shrinks() { - let bytes = |alpha| { - summary_shape( - &FieldDataType::Sketch( - SketchKind::new(SketchAlgorithm::DDSketch, SketchParams::DDSketch { alpha }), - GroupingStrategy::default(), - ), - StateInputRows::UNKNOWN, - ) - .1 - }; - assert!( - bytes(0.1) < bytes(0.01), - "{} vs {}", - bytes(0.1), - bytes(0.01) - ); - assert!( - bytes(0.01) < bytes(0.001), - "{} vs {}", - bytes(0.01), - bytes(0.001) - ); - } - - /// Every sketch family's state is sized from its parameters, as - /// `asap_sketchlib` allocates it: an HLL's registers, a KMV's `k` - /// hashes once it has seen `k` distinct items, and a UnivMon's layers - /// once keys of the assumed size have churned through every heap. - #[test] - fn summary_state_is_sized_from_sketch_params() { - use asap_sketchlib::DataInput; - let bytes = |algorithm, params| { - summary_shape( - &FieldDataType::Sketch( - SketchKind::new(algorithm, params), - GroupingStrategy::default(), - ), - StateInputRows::UNKNOWN, - ) - .1 - }; - assert_eq!( - bytes( - SketchAlgorithm::Hll, - SketchParams::Hll { - precision: 14, - relative_error: None - } - ), - asap_sketchlib::HyperLogLog::::new().memory_bytes() as u64 - ); - let mut kmv: asap_sketchlib::Kmv = asap_sketchlib::Kmv::new(1_000); - for i in 0..2_000u64 { - kmv.update(&DataInput::U64(i)); - } - assert_eq!( - bytes( - SketchAlgorithm::Kmv, - SketchParams::Kmv { - k: 1_000, - relative_error: None - } - ), - kmv.memory_bytes() as u64 - ); - let mut univmon = asap_sketchlib::UnivMon::new(8, 3, 64, 2); - for i in 0..10_000u64 { - let mut key = [0; ASSUMED_VALUE_KEY_BYTES]; - key[..8].copy_from_slice(&i.to_le_bytes()); - univmon.update_weighted(&DataInput::Bytes(&key), 1); - } - assert_eq!( - bytes( - SketchAlgorithm::UnivMon, - SketchParams::UnivMon { - heap_size: 8, - sketch_rows: 3, - sketch_cols: 64, - layers: 2, - distinct_failure_probability: None, - } - ), - univmon.memory_bytes() as u64 - ); - } - - /// KLL state whose samples are unknown is charged its fixed 744 bytes - /// and the item array at the capacity `asap_sketchlib` grows it to: - /// `compute_max_capacity(k, 8)` 8-byte items, with `k` capped at 26 602. - #[test] - fn kll_state_is_sized_as_asap_sketchlib_allocates_it() { - let bytes = |k| { - summary_shape( - &FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k, - failure_probability: None, - }, - ), - GroupingStrategy::default(), - ), - StateInputRows::UNKNOWN, - ) - .1 - }; - assert_eq!(bytes(269), 744 + 8 * 1_206); - assert_eq!(bytes(26_602), 744 + 8 * 80_118); - assert_eq!(bytes(30_000), 744 + 8 * 80_118); - } - /// A KLL state is charged what `asap_sketchlib` allocates for the /// samples it sees, not its full capacity: its fixed struct and level /// array, plus the item array the library starts empty and doubles from @@ -727,18 +616,19 @@ mod tests { assert_eq!(key.to_bytes().unwrap().len(), ASSUMED_HEAP_KEY_BYTES); for (algorithm, params, rows, priced_as, fields, full) in &families { for n in [1, 100, *full] { - let family = FieldDataType::Sketch( - SketchKind::new(*algorithm, params.clone()), - GroupingStrategy::default(), - ); + let family = FieldDataType::Sketch(*algorithm, GroupingStrategy::default()); let input = match rows { Rows::Values => &value, Rows::Labels => &labeled, Rows::Keyed => &keyed, }; - let mut updater = - create_planner_accumulator(&family, input, &GroupingStrategy::default()) - .unwrap(); + let mut updater = create_planner_accumulator( + &family, + Some(params), + input, + &GroupingStrategy::default(), + ) + .unwrap(); for i in 0..n { match rows { // Distinct magnitudes over the assumed range, in a @@ -802,34 +692,4 @@ mod tests { } } } - - /// A UnivMon row updates layers `0..=b`, where its bottom layer `b` is - /// geometric (`asap_sketchlib`'s `bottom_layer_for_hash`), so - /// `2 - 2^(1 - layers)` layers in expectation; each updates one counter - /// per sketch row and its heap. - #[test] - fn univmon_update_cost_is_expected_layers_times_rows_and_heap() { - let ops = |layers| { - summary_shape( - &FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::UnivMon, - SketchParams::UnivMon { - heap_size: 256, - sketch_rows: 5, - sketch_cols: 1_024, - layers, - distinct_failure_probability: None, - }, - ), - GroupingStrategy::default(), - ), - StateInputRows::UNKNOWN, - ) - .0 - }; - assert_eq!(ops(1), 6.0); - assert_eq!(ops(2), 1.5 * 6.0); - assert_eq!(ops(16), (2.0 - 2f64.powi(-15)) * 6.0); - } } diff --git a/crates/cost/src/usage.rs b/crates/cost/src/usage.rs index f811145d4..7601756e5 100644 --- a/crates/cost/src/usage.rs +++ b/crates/cost/src/usage.rs @@ -1,5 +1,5 @@ //! Costing a physical plan DAG node by node: each node's resource usage -//! ([`node_usage`]), then the plan's usage, latency, retention, kept tumbling window +//! ([`node_usage`]), then the plan's usage, latency, retention, retained tumbling window //! chains and cost ([`plan_cost`]). Moved from Stage 3 (`asap-plan-selection`, //! whose module doc, "Cost model", describes the model), so Stage 3 computes //! no cost of its own. @@ -30,7 +30,7 @@ use asap_types::ir::schema::{ FieldDataType, GroupingStrategy, Schema, SketchParams, SketchStatistic, }; use asap_types::ir::{ASAPOp, NonASAPOp, TimeRangeKind}; -use asap_types::physical::{Materialization, Medium}; +use asap_types::physical::{Lifetime, StorageMedium}; use asap_types::workload::{DataWorkload, QueryRecurrence, RangeEvaluation}; use asap_types::workload::bounds::{ @@ -44,17 +44,17 @@ use asap_types::workload::bounds::{ /// reached. A query-time node is measured per computation over the rows its /// time ranges keep, times how often it is computed per second /// ([`computations`]); an ingestion-time node over one second of arriving -/// rows. A materialized node's kept state is counted in byte-seconds per -/// second; a tumbling-window chain kept beyond one evaluation keeps the tumbling windows its -/// retention covers ([`tumbling_window_retention_ms`]). `nodes` and `kept_chains` are -/// `dag`'s ([`node_map`], [`kept_tumbling_window_chains`]); `recurrence[q]` and +/// rows. A materialized node's state is counted in byte-seconds per +/// second; a tumbling-window chain materialized beyond one evaluation keeps the tumbling windows its +/// retention covers ([`tumbling_window_retention_ms`]). `nodes` and `materialized_chains` are +/// `dag`'s ([`node_map`], [`materialized_tumbling_window_chains`]); `recurrence[q]` and /// `range_evaluations[q]` say how often query `q` runs and how many steps /// each run evaluates (one when missing). #[allow(clippy::too_many_arguments)] pub fn plan_cost( dag: &PhysicalASAPDAG, nodes: &NodeMap<'_>, - kept_chains: &HashSet, + materialized_chains: &HashSet, recurrence: &[QueryRecurrence], range_evaluations: &[Option], data: &DataWorkload, @@ -77,7 +77,7 @@ pub fn plan_cost( } = each_node_usage( dag, nodes, - kept_chains, + materialized_chains, recurrence, range_evaluations, data, @@ -94,13 +94,13 @@ pub fn plan_cost( // Each node's work in ms on one core: one computation at query time, // one second of arriving rows at ingestion time. let mut work_ms: HashMap = HashMap::new(); - // Query-time memory held only while one evaluation runs, a MatAtQuery - // state in memory or a NotMaterialized node's working memory: (bytes, + // Query-time memory held only while one evaluation runs, a OneEvaluation + // state in memory or a not-materialized node's working memory: (bytes, // per second, whether each consumer computes its own copy), // counted once the evaluations' latencies are known, in node order so // the sum is the same in every run. let mut held: BTreeMap = BTreeMap::new(); - // The raw data query-time scans read, kept on disk: per source, the + // The raw data query-time scans read, retained on disk: per source, the // largest scan's (bytes, node). let mut raw_data: HashMap<&Source, (u64, PhysicalASAPNodeId)> = HashMap::new(); // Each in-memory ingestion-time state's bytes per window (what it keeps @@ -246,13 +246,13 @@ pub fn plan_cost( for (latency, evaluation_ms) in latency.iter_mut().zip(evaluation_ms_per_query) { latency.query_ms += evaluation_ms; } - // Query-time memory is held for one evaluation: a MatAtQuery state from + // Query-time memory is held for one evaluation: a `OneEvaluation` state from // its build until the longest query reading it finishes, about that - // query's latency, once per execution. A NotMaterialized node's working + // query's latency, once per execution. A not-materialized node's working // memory is computed once per consumer ([`consumers`]), and each copy is // held while its consumer's evaluation runs: a query's for that query's // latency, at its rate; a materialized node's until the longest query - // reading that node finishes, at the node's rate, as its MatAtQuery + // reading that node finishes, at the node's rate, as its `OneEvaluation` // state is. let longest_reader_ms = |id: PhysicalASAPNodeId| { reach @@ -379,7 +379,7 @@ struct EachNodeUsage<'a> { fn each_node_usage<'a>( dag: &'a PhysicalASAPDAG, nodes: &NodeMap<'a>, - kept_chains: &HashSet, + materialized_chains: &HashSet, recurrence: &[QueryRecurrence], range_evaluations: &[Option], data: &DataWorkload, @@ -423,11 +423,11 @@ fn each_node_usage<'a>( .collect(); let scan_window = match (&node.payload, node.output_state.timing) { (Payload::NonASAP(NonASAPOp::Scan { .. }), ExecutionTiming::QueryTime) => { - match dag.scan_window(node.id, kept_chains) { + match dag.scan_window(node.id, materialized_chains) { // Priced as before #955 decides `@` anchors and - // unwindowed readers: the widest window read. + // unwindowed queries: the widest window read. ScanWindow::Unbounded => trailing_window( - widest_window_read_ms(dag, nodes, node.id, kept_chains) + widest_window_read_ms(dag, nodes, node.id, materialized_chains) .or(root_scan_lookback_ms), ), window => window, @@ -475,10 +475,10 @@ fn state_family<'a>( family, grouping, .. }, ) => match family { - FieldDataType::Sketch(kind, _) + FieldDataType::Sketch(..) if *grouping == GroupingStrategy::PerSubpopulationInstance => { - StateFamily::Sketch(kind.params()) + todo!("#1207 PR 6: the state family of θ from the candidate's LogicalProperties::parameters") } FieldDataType::ExactAggregate(kind, _) => StateFamily::Exact(kind), _ => StateFamily::None, @@ -572,8 +572,8 @@ pub fn node_usage<'a>( let mut held = None; let mut maintained = None; let mut raw_scan = None; - // A query-time scan's raw rows kept ([`ScanWindow::lookback_ms`]). - let mut raw_kept_rows = None; + // A query-time scan's raw rows retained ([`ScanWindow::lookback_ms`]). + let mut raw_retained_rows = None; let ingestion = node.output_state.timing == ExecutionTiming::IngestionTime; let inputs: Vec = producers.iter().map(|p| p.output).collect(); let sources: BTreeSet<&str> = match &node.payload { @@ -722,15 +722,15 @@ pub fn node_usage<'a>( (rows_per_ms * lookback as f64).round() as u64 }) }; - // The raw rows kept for a query-time scan: back to the + // The raw rows retained for a query-time scan: back to the // oldest row it reads, `end_offset + width` (an offset // window keeps more than it reads), at most those held. if !ingestion { - let kept_rows = match (by_rate(scan_window.lookback_ms()), rows_held) { + let retained_rows = match (by_rate(scan_window.lookback_ms()), rows_held) { (Some(by_rate), Some(rows_held)) => Some(by_rate.min(rows_held)), (by_rate, rows_held) => by_rate.or(rows_held), }; - raw_kept_rows = kept_rows.map(|rows| rows.max(1)); + raw_retained_rows = retained_rows.map(|rows| rows.max(1)); } let by_rate = by_rate(lookback); let rows = match (ingestion, by_rate, rows_held) { @@ -867,13 +867,13 @@ pub fn node_usage<'a>( // A range keeps its window's samples; an instant // selector the latest sample of each series. // Unknown rate or series: no fewer than its input. - let kept = match kind { + let retained = match kind { TimeRangeKind::Range => rows_per_ms.map_or(input.rows, |rows_per_ms| { (rows_per_ms * range.as_millis() as f64).round() as u64 }), TimeRangeKind::Instant => series.unwrap_or(input.rows), }; - let out = edge(input.rows.min(kept.max(1)))?; + let out = edge(input.rows.min(retained.max(1)))?; let out = EdgeStatistics { rows: out.rows, bytes: out.bytes.min(input.bytes), @@ -945,7 +945,7 @@ pub fn node_usage<'a>( tumbling_window_count, .. }) => { - // Each input row is kept by the one tumbling window its time falls in, + // Each input row is retained by the one tumbling window its time falls in, // as a time range keeps it (one operation), and updates // that tumbling window. let group_count = groups(reduction)?; @@ -954,16 +954,16 @@ pub fn node_usage<'a>( .ok_or(AnalyticalCostError::Overflow)?; // Each tumbling window's state sees its group's rows in one tumbling window: of // one second's rows at ingestion time, of the rows read at - // query time (all of them for a chain kept across + // query time (all of them for a chain materialized across // evaluations, whose scan Stage 2 bounds to its newest tumbling window, // the only one it builds). - let rows_per_tumbling_window = match (ingestion, node.materialization) { + let rows_per_tumbling_window = match (ingestion, node.lifetime) { (true, _) => u64::try_from( (u128::from(input.rows) * u128::from(*tumbling_window_width_ms)) .div_ceil(1_000), ) .map_err(|_| AnalyticalCostError::Overflow)?, - (false, Materialization::MatAtQueryKeptAcrossEvaluations(_)) => input.rows, + (false, Lifetime::AcrossEvaluations(_)) => input.rows, (false, _) => input .rows .div_ceil(u64::from(*tumbling_window_count).max(1)), @@ -1117,7 +1117,7 @@ pub fn node_usage<'a>( let mut peak_memory_bytes = 0u64; let mut disk_bytes = 0u64; let mut detail = format!("{detail}; x {per_second:.4}/s"); - // State kept between computations: at ingestion time, the window + // State retained between computations: at ingestion time, the window // being built and the completed ones queries read (a chain of N // tumbling windows keeps N + 1), every second. At query time the output is // reused only within one evaluation, so it is held only while that @@ -1127,23 +1127,27 @@ pub fn node_usage<'a>( // keeps nothing holds its working memory while it runs, once per // consumer that recomputes it; at query time, that is held for the // evaluation too (counted in [`plan_cost`], per consumer: a query's - // latency, or a materialized consumer's longest reader's, as a - // MatAtQuery state is). + // latency, or a materialized consumer's longest query's, as a + // OneEvaluation state is). let state = out.bytes.max(estimate.peak_memory_bytes()); - let (kept, medium) = match (node.materialization, &node.payload) { - (Materialization::NotMaterialized, _) => { + let (materialized_bytes, medium) = match (ingestion, node.lifetime, &node.payload) { + (_, Lifetime::None, _) => { let working_memory_bytes = estimate.peak_memory_bytes(); peak_memory_bytes = working_memory_bytes.saturating_mul(multiplier); if !ingestion && working_memory_bytes > 0 { held = Some((working_memory_bytes, per_second, true)); } - (0, Medium::Memory) + (0, StorageMedium::Memory) } - // A tumbling-window chain kept across evaluations holds the tumbling windows its + // A tumbling-window chain materialized across evaluations holds the tumbling windows its // retention covers ([`tumbling_window_retention_ms`]: N, plus one per tumbling window - // width of the readers' range-evaluation span), all but the + // width of the queries' range-evaluation span), all but the // newest between evaluations and all while one runs. - (Materialization::MatAtQueryKeptAcrossEvaluations(medium), _) => { + ( + false, + Lifetime::AcrossEvaluations(medium), + Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }), + ) => { let tumbling_windows = tumbling_windows_of(node); let retained = retained_tumbling_window_count; let between = (state / tumbling_windows) @@ -1154,7 +1158,7 @@ pub fn node_usage<'a>( ) .ok_or(AnalyticalCostError::Overflow)?; match medium { - Medium::Memory => { + StorageMedium::Memory => { average_memory_bytes = between as f64; peak_memory_bytes = match retained == tumbling_windows { true => state, @@ -1163,24 +1167,25 @@ pub fn node_usage<'a>( .ok_or(AnalyticalCostError::Overflow)?, }; } - Medium::Disk => disk_bytes = between, + StorageMedium::Disk => disk_bytes = between, } - detail += &format!("; keeps {between} bytes across evaluations"); + detail += &format!("; materializes {between} bytes across evaluations"); (0, medium) } - (Materialization::MatAtQuery(medium), _) => { + (_, Lifetime::OneEvaluation(medium), _) => { match medium { - Medium::Memory => { + StorageMedium::Memory => { held = Some((state, per_second, false)); peak_memory_bytes = state; } - Medium::Disk => disk_bytes = state, + StorageMedium::Disk => disk_bytes = state, } (0, medium) } // The retained tumbling windows and the one being built. ( - Materialization::MatAtIngestion(medium), + true, + Lifetime::AcrossEvaluations(medium), Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }), ) => { let tumbling_windows = tumbling_windows_of(node); @@ -1195,35 +1200,36 @@ pub fn node_usage<'a>( medium, ) } - (Materialization::MatAtIngestion(medium), _) => ( + // A whole state materialized across evaluations: it and the one being built. + (_, Lifetime::AcrossEvaluations(medium), _) => ( state.checked_mul(2).ok_or(AnalyticalCostError::Overflow)?, medium, ), }; - if node.materialization == Materialization::MatAtIngestion(Medium::Memory) { + if ingestion && node.lifetime == Lifetime::AcrossEvaluations(StorageMedium::Memory) { maintained = Some(state / tumbling_windows_of(node)); } - if kept > 0 { + if materialized_bytes > 0 { match medium { - Medium::Memory => { - average_memory_bytes = kept as f64; - peak_memory_bytes = kept; + StorageMedium::Memory => { + average_memory_bytes = materialized_bytes as f64; + peak_memory_bytes = materialized_bytes; } - Medium::Disk => disk_bytes = kept, + StorageMedium::Disk => disk_bytes = materialized_bytes, } - detail += &format!("; keeps {kept} bytes"); + detail += &format!("; keeps {materialized_bytes} bytes"); } if let Payload::NonASAP(NonASAPOp::Scan { source, .. }) = &node.payload { if !ingestion { - let kept = match raw_kept_rows { + let retained = match raw_retained_rows { Some(rows) => edge(rows)?.bytes, None => out.bytes, }; - raw_scan = Some((source, kept)); + raw_scan = Some((source, retained)); } } // One computation's CPU and scan price times its rate, then the - // kept state's. + // materialized state's. let cost = usage_prices.map(|prices| { (estimate.cpu_ops() * prices.cost_per_cpu_op + estimate.scan_bytes() as f64 * prices.cost_per_scanned_byte) @@ -1381,9 +1387,9 @@ fn ingest_workers( (ingest_ms_per_s / 1_000.0).ceil().max(1.0) as u64 } -/// For each tumbling-window chain of `dag` kept beyond one evaluation (at +/// For each tumbling-window chain of `dag` materialized beyond one evaluation (at /// ingestion time, or at query time across evaluations), how far back -/// from the newest evaluation its completed tumbling windows must be kept, in +/// from the newest evaluation its completed tumbling windows must be retained, in /// milliseconds: how long before the evaluation its newest tumbling window ends (the /// offset of its input, `chain_input_offset_ms`), plus its window /// `tumbling_window_count · tumbling_window_width_ms`, plus the longest @@ -1399,7 +1405,7 @@ pub fn tumbling_window_retention_ms( let nodes = node_map(dag); dag.nodes .iter() - .filter_map(|node| match (&node.payload, node.materialization) { + .filter_map(|node| match (&node.payload, node.lifetime) { ( Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { child, @@ -1407,8 +1413,7 @@ pub fn tumbling_window_retention_ms( tumbling_window_count, .. }), - Materialization::MatAtIngestion(_) - | Materialization::MatAtQueryKeptAcrossEvaluations(_), + Lifetime::AcrossEvaluations(_), ) => { let span_ms = reach .iter() @@ -1452,7 +1457,7 @@ fn chain_input_offset_ms( u64::try_from(offset_ms).unwrap_or(0) } -/// The tumbling-window retention of every kept tumbling-window chain of `dag` +/// The tumbling-window retention of every materialized tumbling-window chain of `dag` /// ([`tumbling_window_retention_ms`] over [`query_reach`]), for a plan whose query /// `q` has range evaluation `range_evaluations[q]` (none when missing): /// what Stage 2's realizer records for a realized plan. @@ -1594,21 +1599,22 @@ pub fn node_map(dag: &PhysicalASAPDAG) -> NodeMap<'_> { pub type NodeMap<'a> = HashMap; -/// The tumbling-window chains of `dag` kept across evaluations -/// (`MatAtQueryKeptAcrossEvaluations`, which Stage 2 offers only when the -/// deployment keeps query-time results and every reader builds one new +/// The query-time tumbling-window chains of `dag` materialized across evaluations +/// (lifetime `AcrossEvaluations` at query time, which Stage 2 offers only when the +/// deployment keeps query-time results and every query builds one new /// tumbling window per evaluation). Each evaluation builds the newest tumbling window, and the -/// older ones, which complete the readers' look-back, are kept from +/// older ones, which complete the queries' look-back, are retained from /// earlier evaluations. -pub fn kept_tumbling_window_chains(dag: &PhysicalASAPDAG) -> HashSet { +pub fn materialized_tumbling_window_chains(dag: &PhysicalASAPDAG) -> HashSet { dag.nodes .iter() .filter(|node| { matches!( - (&node.payload, node.materialization), + (&node.payload, node.output_state.timing, node.lifetime), ( Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }), - Materialization::MatAtQueryKeptAcrossEvaluations(_), + ExecutionTiming::QueryTime, + Lifetime::AcrossEvaluations(_), ) ) }) @@ -1617,16 +1623,16 @@ pub fn kept_tumbling_window_chains(dag: &PhysicalASAPDAG) -> HashSet, scan: PhysicalASAPNodeId, - kept_chains: &HashSet, + materialized_chains: &HashSet, ) -> Option { let mut widest = None; let mut stack = vec![scan]; @@ -1641,7 +1647,7 @@ fn widest_window_read_ms( Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { tumbling_window_width_ms, .. - }) if kept_chains.contains(&edge.consumer) => *tumbling_window_width_ms, + }) if materialized_chains.contains(&edge.consumer) => *tumbling_window_width_ms, Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { tumbling_window_width_ms, tumbling_window_count, @@ -1667,7 +1673,7 @@ pub fn family_name(family: &FieldDataType) -> String { FieldDataType::Sketch(_, GroupingStrategy::SharedMultiSubpopulation { kind, .. }) => { format!("{kind:?}") } - FieldDataType::Sketch(kind, _) => format!("{:?}", kind.algorithm()), + FieldDataType::Sketch(algorithm, _) => format!("{algorithm:?}"), FieldDataType::ExactAggregate(kind, _) => format!("exact {kind:?} accumulator"), other => format!("{other:?}"), } diff --git a/crates/cost/tests/held_memory.rs b/crates/cost/tests/held_memory.rs deleted file mode 100644 index 8bc3978c0..000000000 --- a/crates/cost/tests/held_memory.rs +++ /dev/null @@ -1,168 +0,0 @@ -//! Query-time working memory in [`plan_cost`]: how long a `NotMaterialized` -//! node's working memory is held depends on whether a materialized node -//! reads it. - -use asap_cost::execution_demand::evaluation_rate; -use asap_cost::ResourceUnitCost; -use asap_cost::{ - kept_tumbling_window_chains, node_map, plan_cost, CostInputs, CostModel, PlanCost, -}; -use asap_logical_asap_optimization::test_support::lower_promql; -use asap_physical_asap_optimization::optimize_physical_asap; -use asap_types::ir::physical_export::PhysicalASAPDAG; -use asap_types::ir::properties::AccuracyTarget; -use asap_types::ir::QueryRoot; -use asap_types::physical::{Materialization, Medium}; -use asap_types::workload::{ - DataArrival, DataWorkload, QueryRecurrence, RepeatedDemand, RepetitionInterval, SourceFacts, -}; - -/// Data arriving at 10 000 rows per second over 10 000 series of `m`. -fn data() -> DataWorkload { - DataWorkload { - arrival: DataArrival::Ingesting { - rows_per_second: 10_000.0, - }, - sources: [( - "m".to_string(), - SourceFacts { - series_count: Some(10_000), - ..SourceFacts::default() - }, - )] - .into_iter() - .collect(), - } -} - -/// Two queries repeating every 10 s and every 20 s. -fn recurrence() -> [QueryRecurrence; 2] { - [10_000, 20_000].map(|interval_ms| { - QueryRecurrence::Repeated(RepeatedDemand::FixedInterval(RepetitionInterval( - interval_ms, - ))) - }) -} - -/// `plan_cost` of `dag` for [`recurrence`], at `cost_per_memory_byte_second`. -fn cost(dag: &PhysicalASAPDAG, cost_per_memory_byte_second: f64) -> PlanCost { - let mut unit_cost = ResourceUnitCost::illustrative(); - unit_cost.cost_per_memory_byte_second = cost_per_memory_byte_second; - let settings = CostInputs { - cost_model: CostModel::UsageOverTime(unit_cost), - ..CostInputs::default() - }; - let nodes = node_map(dag); - plan_cost( - dag, - &nodes, - &kept_tumbling_window_chains(dag), - &recurrence(), - &[], - &data(), - &settings, - 1.0, - ) - .unwrap() - .0 -} - -/// Per node holding query-time memory, (its id, its held bytes, its -/// held-memory cost per second at unit memory cost 1). -fn held_memory_costs(dag: &PhysicalASAPDAG) -> Vec<(usize, f64, f64)> { - let (with, without) = (cost(dag, 1.0), cost(dag, 0.0)); - with.per_node - .iter() - .filter_map(|(id, node)| { - let held = node.detail.split("; holds ").nth(1)?; - let bytes: f64 = held.split(' ').next().unwrap().parse().unwrap(); - let term = - node.cost_per_second.unwrap() - without.per_node[id].cost_per_second.unwrap(); - Some((*id, bytes, term)) - }) - .collect() -} - -fn materialization_of(dag: &PhysicalASAPDAG, id: usize) -> Materialization { - dag.nodes - .iter() - .find(|n| n.id == id) - .unwrap() - .materialization -} - -fn close(a: f64, b: f64) -> bool { - (a - b).abs() <= 1e-12 * a.abs().max(b.abs()).max(1e-300) -} - -/// Query 0 is `sum(...)` over query 1's output, so it runs longer. Without -/// a materialized node, each query computes the shared nodes and holds -/// their working memory for its own latency: `bytes · Σ rate_q · latency_q`. -/// When query 1's output is `MatAtQuery`, the nodes below it are computed -/// once per execution of it, at its rate, and held until the -/// longest reader, query 0, finishes, as the `MatAtQuery` state itself is: -/// `bytes · rate · latency_0`, not the rate-weighted mean of the two -/// latencies. -#[test] -fn working_memory_read_by_a_materialized_node_is_held_for_the_longest_reader() { - let outer = lower_promql("sum(sum_over_time(m[1m]))", AccuracyTarget::exact()); - let inner = outer.children()[0].clone(); - let plans = optimize_physical_asap( - "L", - &[QueryRoot::Operator(outer), QueryRoot::Operator(inner)], - &[], - &data(), - false, - ) - .unwrap(); - let dag_with = |materialization: Materialization| { - plans - .iter() - .map(|plan| &plan.dag) - .find(|dag| materialization_of(dag, dag.roots[1]) == materialization) - .unwrap() - }; - let recomputed = dag_with(Materialization::NotMaterialized); - let shared = dag_with(Materialization::MatAtQuery(Medium::Memory)); - let latency_ms = |dag| { - let cost = cost(dag, 1.0); - [0, 1].map(|q| cost.usage.latency_per_query[q].total_ms()) - }; - - // Recomputed per query: the rate-weighted sum of both latencies. - let [outer_ms, inner_ms] = latency_ms(recomputed); - assert!(outer_ms > inner_ms, "{outer_ms} vs {inner_ms}"); - let below = |dag: &PhysicalASAPDAG| { - let materialized = dag.roots[1]; - held_memory_costs(dag) - .into_iter() - .filter(move |(id, _, _)| *id != materialized && *id != dag.roots[0]) - .collect::>() - }; - let checked = below(recomputed); - assert!(!checked.is_empty(), "the shared nodes hold memory"); - for (id, bytes, term) in checked { - let expected = bytes * (0.1 * outer_ms + 0.05 * inner_ms) / 1_000.0; - assert!(close(term, expected), "node {id}: {term} vs {expected}"); - } - - // Below a MatAtQuery node: the longest reader's latency at the - // materialized node's rate (its readers' schedules together). - let materialized_rate = evaluation_rate( - recurrence().iter().map(|r| (r, None)), - CostInputs::default().non_repeating_query_horizon_seconds, - ) - .unwrap(); - let [outer_ms, inner_ms] = latency_ms(shared); - assert!(outer_ms > inner_ms, "{outer_ms} vs {inner_ms}"); - let checked = below(shared); - assert!(!checked.is_empty(), "the nodes below it hold memory"); - for (id, bytes, term) in checked { - assert_eq!( - materialization_of(shared, id), - Materialization::NotMaterialized - ); - let expected = bytes * materialized_rate * outer_ms / 1_000.0; - assert!(close(term, expected), "node {id}: {term} vs {expected}"); - } -} diff --git a/crates/cost/tests/ingestion_refresh_runs.rs b/crates/cost/tests/ingestion_refresh_runs.rs deleted file mode 100644 index ebd33bab5..000000000 --- a/crates/cost/tests/ingestion_refresh_runs.rs +++ /dev/null @@ -1,289 +0,0 @@ -//! How often a `NotMaterialized` node is computed for the views refreshed -//! at ingestion above it ([`computations`]): once per refresh window, since -//! a runtime refreshes the views of one refresh window in one ingestion run -//! (issue #1151). - -use std::rc::Rc; -use std::time::Duration; - -use asap_cost::execution_demand::computations; -use asap_types::ir::operator::operator_properties::{Reduction, Source, TimeShift}; -use asap_types::ir::physical_export::{compile_physical_asap_workload, PhysicalASAPDAG}; -use asap_types::ir::scalar::ColumnRef; -use asap_types::ir::schema::{ - DataType, Field, FieldDataType, GroupingStrategy, Schema, SketchAlgorithm, SketchKind, - SketchParams, SummaryUpdate, -}; -use asap_types::ir::{ - apply_materialization_timings_to_roots, ASAPOp, MaterializationAssignment, NonASAPOp, Operator, - OperatorNode, QueryRoot, TimeRangeKind, -}; -use asap_types::physical::{Materialization, Medium}; -use asap_types::workload::{QueryRecurrence, RepeatedDemand, RepetitionInterval}; - -/// `m` with columns `ts`, `value` and `job`. -fn scan() -> Rc { - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Scan { - source: Source::TimeSeries { metric: "m".into() }, - predicates: vec![], - schema: Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("value", DataType::Float64, false), - Field::plain("job", DataType::Utf8, true), - ], - 0, - vec![], - ), - })) - .unwrap() -} - -/// A quantile sketch of `algorithm` (KLL k = 200 or DDSketch α = 0.01). -fn quantile_sketch(algorithm: SketchAlgorithm) -> FieldDataType { - let params = match algorithm { - SketchAlgorithm::Kll => SketchParams::Kll { - k: 200, - failure_probability: None, - }, - _ => SketchParams::DDSketch { alpha: 0.01 }, - }; - FieldDataType::Sketch(SketchKind::new(algorithm, params), Default::default()) -} - -/// `tumbling_window_count` tumbling windows of `tumbling_window_width_ms` of a quantile sketch of -/// `algorithm` per `job` over `input`. -fn chain( - input: &Rc, - algorithm: SketchAlgorithm, - tumbling_window_width_ms: u64, - tumbling_window_count: u32, -) -> Rc { - OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { - child: Rc::clone(input), - family: quantile_sketch(algorithm), - input: SummaryUpdate::column(ColumnRef::Named("value".into())), - reduction: Reduction::by(vec![2]), - grouping: GroupingStrategy::default(), - filter: None, - tumbling_window_width_ms, - tumbling_window_count, - })) - .unwrap() -} - -/// One query per summary of `summaries`, merging it, with each summary -/// materialized as `materialization`: the DAG Stage 3 costs, chains -/// unexpanded. -fn dag_of(summaries: &[Rc], materialization: Materialization) -> PhysicalASAPDAG { - let mut assignment = MaterializationAssignment::default(); - let roots: Vec = summaries - .iter() - .map(|summary| { - if materialization != Materialization::NotMaterialized { - assignment.set(summary, materialization); - } - QueryRoot::Operator( - OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryMerge { - children: vec![Rc::clone(summary)], - newest_tumbling_windows: None, - group_by: None, - })) - .unwrap(), - ) - }) - .collect(); - let timed = apply_materialization_timings_to_roots(&roots, &assignment).unwrap(); - compile_physical_asap_workload(&timed).unwrap() -} - -/// `(multiplier, per_second)` of `dag`'s `Scan`, each query every 10 s. -fn scan_computations(dag: &PhysicalASAPDAG) -> (u64, f64) { - computations_of(dag, |op| matches!(op, NonASAPOp::Scan { .. })) -} - -/// `(multiplier, per_second)` of the one node of `dag` whose operator -/// `is_node` picks, each query every 10 s. -fn computations_of( - dag: &PhysicalASAPDAG, - is_node: impl Fn(&NonASAPOp) -> bool, -) -> (u64, f64) { - let every_10_s = - QueryRecurrence::Repeated(RepeatedDemand::FixedInterval(RepetitionInterval(10_000))); - let recurrence = vec![every_10_s; dag.roots.len()]; - let times = computations(dag, &recurrence, &[], 3600.0, None).unwrap(); - let node = dag - .nodes - .iter() - .find(|node| matches!(&node.payload, Operator::NonASAP(op) if is_node(op))) - .unwrap(); - times[&node.id] -} - -/// `child` evaluated `offset_ms` earlier. -fn shifted(child: &Rc, offset_ms: i64) -> Rc { - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeShift { - shift: TimeShift { - offset_ms, - at: None, - }, - child: Rc::clone(child), - })) - .unwrap() -} - -const AT_INGESTION: Materialization = Materialization::MatAtIngestion(Medium::Memory); - -/// A KLL chain and a DDSketch chain of 10 s tumbling windows over one `Scan`, both -/// kept at ingestion, are refreshed in one ingestion run per tumbling window: the -/// `Scan` is computed once per second of arriving data, not twice. -#[test] -fn a_scan_under_two_chains_of_one_tumbling_window_width_is_computed_once() { - let scan = scan(); - let dag = dag_of( - &[ - chain(&scan, SketchAlgorithm::Kll, 10_000, 6), - chain(&scan, SketchAlgorithm::DDSketch, 10_000, 30), - ], - AT_INGESTION, - ); - assert_eq!(scan_computations(&dag), (1, 1.0)); -} - -/// With 10 s and 60 s tumbling windows the chains need different runs (the 10 s chain -/// needs five tumbling window ends the 60 s chain does not): the `Scan` is computed -/// once for each. -#[test] -fn a_scan_under_chains_of_two_tumbling_window_widths_is_computed_for_each() { - let scan = scan(); - let dag = dag_of( - &[ - chain(&scan, SketchAlgorithm::Kll, 10_000, 6), - chain(&scan, SketchAlgorithm::DDSketch, 60_000, 5), - ], - AT_INGESTION, - ); - assert_eq!(scan_computations(&dag), (2, 2.0)); -} - -/// Query consumers are unchanged: with both chains built at query time -/// (`NotMaterialized`), each query computes the `Scan` at its own rate. -#[test] -fn query_consumers_still_count_once_each() { - let scan = scan(); - let range = |width_ms| { - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeRange { - range: Duration::from_millis(width_ms), - kind: TimeRangeKind::Range, - child: Rc::clone(&scan), - })) - .unwrap() - }; - let build = |input: Rc, algorithm| { - OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryAgg { - child: input, - family: quantile_sketch(algorithm), - input: SummaryUpdate::column(ColumnRef::Named("value".into())), - reduction: Reduction::by(vec![2]), - grouping: GroupingStrategy::default(), - filter: None, - })) - .unwrap() - }; - let dag = dag_of( - &[ - build(range(60_000), SketchAlgorithm::Kll), - build(range(60_000), SketchAlgorithm::DDSketch), - ], - Materialization::NotMaterialized, - ); - assert_eq!(scan_computations(&dag), (2, 0.2)); -} - -/// Two 10 s chains and a 60 s chain over one `Scan`: two refresh windows, -/// two ingestion runs. -#[test] -fn a_scan_under_chains_of_10_10_and_60_s_is_computed_for_each_width() { - let scan = scan(); - let dag = dag_of( - &[ - chain(&scan, SketchAlgorithm::Kll, 10_000, 6), - chain(&scan, SketchAlgorithm::DDSketch, 10_000, 30), - chain(&scan, SketchAlgorithm::Kll, 60_000, 5), - ], - AT_INGESTION, - ); - assert_eq!(scan_computations(&dag), (2, 2.0)); -} - -/// A 10 s chain over the `Scan` and a 10 s chain over the `Scan` shifted by -/// 5 s end their tumbling windows on different grids (phases 0 and 5 s): two runs. -#[test] -fn chains_of_one_width_and_two_phases_are_computed_apart() { - let scan = scan(); - let dag = dag_of( - &[ - chain(&scan, SketchAlgorithm::Kll, 10_000, 6), - chain( - &shifted(&scan, 5_000), - SketchAlgorithm::DDSketch, - 10_000, - 30, - ), - ], - AT_INGESTION, - ); - assert_eq!(scan_computations(&dag), (2, 2.0)); -} - -/// Two 10 s chains over one `TimeShift(10 s)` of the `Scan` (phase 0 for -/// both): the shift and the `Scan` below it run once per run. -#[test] -fn nodes_below_a_shared_intermediate_node_are_computed_once_per_run() { - let shift = shifted(&scan(), 10_000); - let dag = dag_of( - &[ - chain(&shift, SketchAlgorithm::Kll, 10_000, 6), - chain(&shift, SketchAlgorithm::DDSketch, 10_000, 30), - ], - AT_INGESTION, - ); - assert_eq!(scan_computations(&dag), (1, 1.0)); - assert_eq!( - computations_of(&dag, |op| matches!(op, NonASAPOp::TimeShift { .. })), - (1, 1.0) - ); -} - -/// Two summaries kept at ingestion over rows with no time window -/// (`Values`) have no refresh window: each counts alone. -#[test] -fn consumers_of_unbounded_windows_count_alone() { - let rows = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Values { - rows: vec![], - schema: scan().schema.clone(), - })) - .unwrap(); - let build = |algorithm| { - OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryAgg { - child: Rc::clone(&rows), - family: quantile_sketch(algorithm), - input: SummaryUpdate::column(ColumnRef::Named("value".into())), - reduction: Reduction::by(vec![2]), - grouping: GroupingStrategy::default(), - filter: None, - })) - .unwrap() - }; - let dag = dag_of( - &[ - build(SketchAlgorithm::Kll), - build(SketchAlgorithm::DDSketch), - ], - AT_INGESTION, - ); - assert_eq!( - computations_of(&dag, |op| matches!(op, NonASAPOp::Values { .. })), - (2, 2.0) - ); -} diff --git a/crates/devtools/Cargo.toml b/crates/devtools/Cargo.toml index 763e654fe..bb24e585b 100644 --- a/crates/devtools/Cargo.toml +++ b/crates/devtools/Cargo.toml @@ -13,9 +13,9 @@ asap-frontend-promql = { path = "../frontend-promql" } asap-frontend-sql = { path = "../frontend-sql" } asap-accuracy = { path = "../accuracy" } asap-logical-asap-optimization = { path = "../logical-asap-optimization" } -asap-physical-asap-optimization = { path = "../physical-asap-optimization", features = ["reference-pricer"] } +asap-physical-asap-optimization = { path = "../physical-asap-optimization" } asap-cost = { path = "../cost" } -asap-plan-selection = { path = "../plan-selection", features = ["reference-pricer"] } +asap-plan-selection = { path = "../plan-selection" } # The installable plan of the selected candidate (`installable_plan_of`), # whose stored outputs `asap_planner` writes. asap-planner = { path = "../planner" } @@ -32,14 +32,9 @@ serde_json = "1" serde_yaml = "0.9" tokio = { version = "1", features = ["rt", "macros", "rt-multi-thread"] } -[features] -# Each Stage 3 selection MILP solve also reports where its LP relaxation -# loses its bound, with `ASAP_STAGE3_LP_DIAGNOSIS=` (measurement only). -lp-diagnosis = ["asap-plan-selection/lp-diagnosis"] - [dev-dependencies] asap-types = { path = "../types", features = ["test-support"] } -asap-physical-asap-optimization = { path = "../physical-asap-optimization", features = ["test-support"] } +asap-physical-asap-optimization = { path = "../physical-asap-optimization" } # Pass 1's target list (`per_computation_candidates`) for the tests. asap-logical-asap-optimization = { path = "../logical-asap-optimization", features = ["test-support"] } diff --git a/crates/devtools/src/bin/asap_planner.rs b/crates/devtools/src/bin/asap_planner.rs index 4e231d027..2e9d883b5 100644 --- a/crates/devtools/src/bin/asap_planner.rs +++ b/crates/devtools/src/bin/asap_planner.rs @@ -59,7 +59,7 @@ // - stage2_physical_asap_optimization (from `stage2`): `pruned_candidates`, // the physical candidates Stage 2 pruned (each with its memo `group`, // `operator_node`, `operator`, `implementation_root`, `timing`, -// `materialization` and `reason`); with `stage3`, `candidates` holds +// `lifetime` and `reason`); with `stage3`, `candidates` holds // `P1`, the selected plan as Stage 2 built it; // - stage3_plan_selection (from `stage3`): Stage 3's `Selection`: // `selected_dag` (`candidate_of_node`, `edges`, @@ -69,7 +69,7 @@ // `search_report`; `error` when no plan was selected. // `tumbling_window_retention_ms` maps each tumbling-window chain the plan // keeps beyond one evaluation (by node id in `P1`) to how far back its -// tumbling windows are kept. +// tumbling windows are retained. // `materialized_views` are the plan's materialized views as // `plan_installable` derives them, each with the node of `P1` it is // installed from (`stage2_node_id`), its position among that node's @@ -97,11 +97,10 @@ // A section of a stage after `--run-until` is null. use std::collections::btree_map::Entry; -use std::collections::{BTreeSet, HashMap, HashSet}; -use std::rc::Rc; +use std::collections::BTreeSet; use asap_cost::{CostInputs, ResourceUsage}; -use asap_devtools::candidate_labels::{computation_label, node_label}; +use asap_devtools::candidate_labels::node_label; use asap_devtools::planning_stages::{ lower_promql_roots, lower_sql_roots, run_stages_until, sql_catalog, PlanningStagesError, PlanningStagesRun, RunUntil, @@ -109,18 +108,15 @@ use asap_devtools::planning_stages::{ use asap_devtools::planning_stages_text::planning_stages_text; use asap_devtools::SqlCatalog; use asap_frontend_promql::{metric_type_warnings, promql_metric_names}; -use asap_logical_asap_optimization::{ChosenCandidatePerMemoGroup, DeploymentCapabilities}; +use asap_logical_asap_optimization::DeploymentCapabilities; use asap_plan_selection::{large_peak_memory_warning, AcceptedAccuracy, Selection}; -use asap_planner::{ - installable_plan_of, selected_chosen_candidates, PlanningModels, PlanningRun, SelectionError, -}; -use asap_types::ir::flat::{flatten, FlatDag, FlatNode, NodeId}; -use asap_types::ir::operator::node::OperatorNode; +use asap_planner::{installable_plan_of, PlanningModels, SelectionError}; +use asap_types::ir::flat::{flatten, FlatDag, FlatNode}; use asap_types::ir::properties::{default_metric_for_root, AccuracyTarget}; use asap_types::ir::QueryRoot; -use asap_types::stages::{queries_rooted_at, MemoGroupId}; +use asap_types::stages::PhysicalASAPMemoCandidates; +use asap_types::stages::{queries_rooted_at, MemoGroup, MemoGroupId}; use asap_types::stages::{Memo, OriginOfCandidateForMemoGroup}; -use asap_types::stages::{PerComputationLogicalMemoCandidates, PhysicalASAPMemoCandidates}; use asap_types::workload::{ DataArrival, DataWorkload, PlanningWorkload, QueryLanguage, QueryRecurrence, QueryWorkloadEntry, RepeatedDemand, @@ -348,7 +344,6 @@ fn document(queries: Vec, entries: &[QueryWorkloadEntry], run: &PlanningS let mut stage1 = json!({ "combinations": logical.len(), "capped": false, - "pruned_candidates": pass1_pruned_candidates_json(run.run.per_computation_candidates.as_ref()), "candidates": logical, }); // Pass 1 or Pass 2 failed, or the deployment's capabilities @@ -393,21 +388,15 @@ fn document(queries: Vec, entries: &[QueryWorkloadEntry], run: &PlanningS /// its materialized views. fn stage3_json(run: &PlanningStagesRun, entries: &[QueryWorkloadEntry]) -> Value { let mut stage3 = selection_json(&run.run.selection); - match (&run.run.selection, &run.run.selected_plan) { - (Ok(selection), Some(roots)) => { - let tumbling_window_retention_ms = run - .run - .selected_physical_plan - .as_ref() - .map(|realized| realized.tumbling_window_retention_ms.clone()) - .unwrap_or_default(); - stage3["tumbling_window_retention_ms"] = tumbling_window_retention_ms + match (&run.run.selection, &run.run.selected_physical_plan) { + (Ok(selection), Some(realized)) => { + stage3["tumbling_window_retention_ms"] = realized + .tumbling_window_retention_ms .iter() .map(|(node, ms)| (node.to_string(), json!(ms))) .collect::>() .into(); - match materialized_views_json(roots, selection, &tumbling_window_retention_ms, entries) - { + match materialized_views_json(realized, selection, entries) { Ok(views) => stage3["materialized_views"] = Value::Array(views), Err(error) => { stage3["materialized_views"] = json!([]); @@ -420,16 +409,15 @@ fn stage3_json(run: &PlanningStagesRun, entries: &[QueryWorkloadEntry]) -> Value stage3 } -/// The document's `memo` section: the memo after each step `run` ran, and -/// with Stage 3 the selected plan's choice over the last step's memo. +/// The document's `memo` section: the memo after each step `run` ran. fn memo_json(run: &PlanningStagesRun) -> Value { let stage0 = run .stage0_memo_groups .as_ref() .expect("run_stages_until builds Stage 0's memo"); let mut named = vec![("stage0", "Stage 0, the frontends' DAG hash-consed", stage0)]; - if let Some(pass1) = &run.run.per_computation_candidates { - named.push(("pass1", "Stage 1, Pass 1", &pass1.memo)); + if let Some(pass1) = &run.run.pass1_memo { + named.push(("pass1", "Stage 1, Pass 1", pass1)); } if let Some(pass2) = &run.pass2_local_window_composition { named.push(( @@ -452,33 +440,7 @@ fn memo_json(run: &PlanningStagesRun) -> Value { })); previous = Some(memo); } - let mut memo = json!({ "nodes": table.nodes, "steps": json_steps }); - if run.run_until == RunUntil::Stage3 { - match selected_choice(&run.run) { - Ok(choice) => memo["selected_choice"] = choice_json(&choice), - Err(reason) => { - memo["selected_choice"] = Value::Null; - memo["selected_choice_unavailable"] = json!(reason); - } - } - } - memo -} - -/// The selected plan's candidate for each group of the Pass 2 memo -/// (`run.logical_asap_candidates`), or why there is none. -fn selected_choice(run: &PlanningRun) -> Result { - match ( - &run.selection, - &run.logical_asap_candidates, - &run.physical_asap_candidates, - ) { - (Ok(selection), Some(logical), Some(physical)) => { - selected_chosen_candidates(&logical.memo, physical, &selection.selected_dag) - } - (Err(error), _, _) => Err(format!("no plan was selected: {error}")), - _ => Err("no plan was selected".into()), - } + json!({ "nodes": table.nodes, "steps": json_steps }) } /// The [`metric_type_warnings`] of each PromQL query of `described` (the @@ -645,21 +607,12 @@ fn arrival_together(earlier: DataArrival, later: DataArrival) -> Option, entries: &[QueryWorkloadEntry], ) -> Result, String> { - let (plan, report) = installable_plan_of( - roots, - Some(selection), - tumbling_window_retention_ms, - entries, - ) - .map_err(|e| e.to_string())?; + let (plan, report) = + installable_plan_of(realized, Some(selection), entries).map_err(|e| e.to_string())?; let views = plan.materialized_views().map_err(|e| e.to_string())?; let mut stage2_of = std::collections::HashMap::new(); for node in report @@ -689,54 +642,15 @@ fn materialized_views_json( .collect() } -/// The ways to compute a measure Pass 1 pruned -/// (`PerComputationLogicalMemoCandidates::pruned_candidates`), in pruning -/// order: each with its aggregate's memo `group`, the `candidate` and the -/// `reason` (`PruneReason`, e.g. `UnsupportedByDeployment`). Empty when -/// Pass 1 failed. -fn pass1_pruned_candidates_json(pass1: Option<&PerComputationLogicalMemoCandidates>) -> Vec { - pass1 - .map_or(&[][..], |pass1| &pass1.pruned_candidates) - .iter() - .map(|pruned| { - json!({ - "group": pruned.group.0, - "candidate": computation_label(&pruned.candidate), - "reason": format!("{:?}", pruned.reason), - }) - }) - .collect() -} - /// The physical candidates Stage 2 pruned -/// (`PhysicalASAPMemoCandidates::pruned_candidates`), in pruning order: -/// each with its memo `group`, its `operator_node` (index into Stage 2's -/// operator nodes), the node's `operator` label, whether it is an -/// `implementation_root`, its `timing` and `materialization`, and the -/// `reason` (`PruneReason`, e.g. -/// `RawDataNotRetained`). Empty without memo steps. +/// (`PhysicalASAPMemoCandidates::pruned_candidates`), in pruning order. +/// Empty without memo steps. fn stage2_pruned_candidates_json(stage2: Option<&PhysicalASAPMemoCandidates>) -> Vec { let Some(stage2) = stage2 else { return Vec::new(); }; - stage2 - .pruned_candidates - .iter() - .map(|pruned| { - let key = &pruned.candidate; - let node = &stage2.operator_nodes_of_candidates_for_memo_groups[key.operator_node.0]; - let flat = export(&[QueryRoot::Operator(node.physical_node.clone())]); - json!({ - "group": pruned.group.0, - "operator_node": key.operator_node.0, - "operator": flat.nodes.last().map(node_label).unwrap_or_default(), - "implementation_root": node.is_implementation_root, - "timing": key.timing, - "materialization": key.materialization, - "reason": format!("{:?}", pruned.reason), - }) - }) - .collect() + let _ = &stage2.pruned_candidates; + todo!("#1207 PR 9 (#1205): each pruned PhysicalCandidateKey with its reason") } /// One plan's cost: per second, its predicted CPU, unit and source, its @@ -816,7 +730,7 @@ fn selection_json(selection: &Result) -> Value { "edges": dag .edges .iter() - .map(|e| json!({ "producer": e.producer.0, "consumer": e.consumer.0, "input": e.input })) + .map(|e| json!({ "producer": e.producer.0, "consumer": e.consumer.0, "input": e.input_index_of_consumer })) .collect::>(), "node_of_output_reference": nodes(&dag.node_of_output_reference), }, @@ -910,69 +824,24 @@ fn with_default_metrics(mut queries: Vec, roots: &[QueryRoot]) -> Vec Value { - Value::Object( - choice - .iter() - .map(|(group, &candidate)| (group.0.to_string(), json!(candidate))) - .collect(), - ) -} - /// The `memo` section's node table (`memo.nodes`): each distinct node once, -/// children before parents, its children as indices into the table. Nodes -/// are matched by their serialized JSON (children included), since the -/// steps rebuild equal nodes as different `Rc`s. +/// children before parents, its children as indices into the table. #[derive(Default)] struct MemoNodes { nodes: Vec, - index_of: HashMap, -} - -impl MemoNodes { - /// Adds `roots` and every node below them. Returns each root's index - /// and, for each node of that DAG, the original node and its index. - fn add( - &mut self, - roots: &[Rc], - ) -> (Vec, Vec<(Rc, NodeId)>) { - let roots: Vec<_> = roots.iter().cloned().map(QueryRoot::Operator).collect(); - let (dag, originals) = flatten(&roots); - let mut index_in_table: Vec = Vec::with_capacity(dag.nodes.len()); - for mut node in dag.nodes { - node.operator = node.operator.map_children(|&child| index_in_table[child]); - let key = serde_json::to_string(&node).expect("a flat node serializes"); - let index = *self.index_of.entry(key).or_insert_with(|| { - self.nodes.push(node); - self.nodes.len() - 1 - }); - index_in_table.push(index); - } - let roots = dag - .roots - .iter() - .map(|root| match root { - QueryRoot::Operator(id) => index_in_table[*id], - QueryRoot::Scalar(_) => unreachable!("memo DAGs have operator roots"), - }) - .collect(); - (roots, originals.into_iter().zip(index_in_table).collect()) - } } -/// `memo`'s groups, each with its candidates, their -/// nodes added to `table`; a candidate is `added` when `previous` (the memo -/// at the step before) has no candidate of that group with the same node and -/// children. +/// `memo`'s groups, each with its candidates, their nodes added to +/// `table`; a candidate is `added` when `previous` (the memo at the step +/// before) has no such candidate. fn memo_groups_json(table: &mut MemoNodes, memo: &Memo, previous: Option<&Memo>) -> Vec { let ids = |groups: &[MemoGroupId]| groups.iter().map(|g| g.0).collect::>(); - memo.groups + memo.groups() .iter() - .zip(&memo.candidates) .enumerate() - .map(|(id, (group, candidates))| { - let node = table.add(std::slice::from_ref(&group.logical_node)).0[0]; + .map(|(id, group)| { + let candidates = &group.candidates; + let node = group_node(table, group); let mut by_origin = serde_json::Map::new(); for candidate in candidates { let count = by_origin @@ -980,51 +849,18 @@ fn memo_groups_json(table: &mut MemoNodes, memo: &Memo, previous: Option<&Memo>) .or_insert(json!(0)); *count = json!(count.as_u64().unwrap_or(0) + 1); } - let earlier = previous - .and_then(|memo| memo.candidates.get(id)) - .map_or(&[][..], Vec::as_slice); - let listed: Vec<_> = candidates - .iter() - .enumerate() - .map(|(index, candidate)| { - let (roots, originals) = table.add(std::slice::from_ref(&candidate.root_node)); - let group_of_node: serde_json::Map<_, _> = originals - .iter() - .filter_map(|(original, node)| { - candidate - .input_groups - .iter() - .find(|child| { - Rc::ptr_eq( - &memo.groups[child.0 as usize].logical_node, - original, - ) - }) - .map(|child| (node.to_string(), json!(child.0))) - }) - .collect(); - json!({ - "index": index, - "implements": std::iter::once(id as u32) - .chain(candidate.fused_groups.iter().map(|g| g.0)) - .collect::>(), - "children": ids(&candidate.input_groups), - "origin": origin_name(&candidate.origin), - "accuracy_budget_parts": candidate.accuracy_budget_parts, - "added": !earlier.iter().any(|before| { - Rc::ptr_eq(&before.root_node, &candidate.root_node) - && before.input_groups == candidate.input_groups - }), - "label": candidate_node_label(&table.nodes, roots[0], &group_of_node), - "node": roots[0], - "group_of_node": group_of_node, - }) - }) - .collect(); + let listed: Vec = if candidates.is_empty() { + Vec::new() + } else { + let _ = previous; + todo!( + "#1207 PR 9 (#1205): each candidate's expression, properties and requirements" + ) + }; json!({ "id": id, "label": node_label(&table.nodes[node]), - "children": ids(&group.child_groups), + "children": group.expression.children().into_iter().map(|g| g.0).collect::>(), "parents": ids(group.parent_groups()), "queries": group.reached_by_queries(), "roots_of": queries_rooted_at(&memo.query_roots, MemoGroupId(id as u32)), @@ -1038,12 +874,16 @@ fn memo_groups_json(table: &mut MemoNodes, memo: &Memo, previous: Option<&Memo>) .collect() } +/// The node of `group`'s own operator, added to `table`. +fn group_node(table: &mut MemoNodes, group: &MemoGroup) -> usize { + let _ = (table, group); + todo!("#1207 PR 9: a group's node from its expression") +} + /// `OriginOfCandidateForMemoGroup` in snake case. fn origin_name(origin: &OriginOfCandidateForMemoGroup) -> String { match origin { OriginOfCandidateForMemoGroup::Pass1 => "pass1", - OriginOfCandidateForMemoGroup::SharedBuild => "shared_build", - OriginOfCandidateForMemoGroup::SummaryResult => "summary_result", OriginOfCandidateForMemoGroup::SummaryInput => "summary_input", OriginOfCandidateForMemoGroup::TumblingWindows => "tumbling_windows", OriginOfCandidateForMemoGroup::Rollup => "rollup", @@ -1052,89 +892,3 @@ fn origin_name(origin: &OriginOfCandidateForMemoGroup) -> String { } .into() } - -/// A candidate's own nodes (those above its child groups' nodes), root -/// first, as `node_label`s joined by " ← ". -fn candidate_node_label( - nodes: &[FlatNode], - root: NodeId, - group_of_node: &serde_json::Map, -) -> String { - let mut labels = Vec::new(); - let mut pending = vec![root]; - let mut seen = HashSet::new(); - while let Some(id) = pending.pop() { - if group_of_node.contains_key(&id.to_string()) || !seen.insert(id) { - continue; - } - labels.push(node_label(&nodes[id])); - pending.extend(nodes[id].operator.children().into_iter().rev().copied()); - } - labels.join(" ← ") -} - -#[cfg(test)] -mod tests { - use super::*; - - /// A node two DAGs share is stored once in the memo's node table, even - /// when each DAG built it as its own `Rc`, and each DAG rebuilt from the - /// table equals its own `FlatDag` export. - #[test] - fn memo_node_table_stores_a_shared_node_once() { - let root = |query: &str| { - let workload: PlanningWorkload = serde_json::from_value(json!({ - "query_workload": {"language": "promql", "query_batch": [{ - "query": query, "invocations": 1, - "requirements": {"accuracy": {"max_error": 0.0, "failure_probability": 0.0}}, - "predictability": {"predictable": {"known_at": null}}, - }]}, - "data_workload": { - "arrival": {"ingesting": {"rows_per_second": 1.0}}, - "sources": {"m": {"series_count": 10, "sample_interval": 15000, "metric_type": "gauge"}}, - }, - })) - .unwrap(); - match lower_promql_roots(&workload).unwrap().remove(0) { - QueryRoot::Operator(node) => node, - QueryRoot::Scalar(_) => unreachable!(), - } - }; - let (sum, max) = (root("sum(m)"), root("max(m)")); - let mut table = MemoNodes::default(); - let (sum_root, _) = table.add(std::slice::from_ref(&sum)); - let after_sum = table.nodes.len(); - let (max_root, _) = table.add(std::slice::from_ref(&max)); - // `max(m)` adds only its own aggregate: the scan is already there. - assert_eq!(table.nodes.len(), after_sum + 1); - // The DAG below `root`, renumbered in table order. - let rebuilt = |root: NodeId| { - let mut reached = BTreeSet::new(); - let mut pending = vec![root]; - while let Some(id) = pending.pop() { - if reached.insert(id) { - pending.extend(table.nodes[id].operator.children().into_iter().copied()); - } - } - let local: HashMap<_, _> = reached.iter().enumerate().map(|(i, &id)| (id, i)).collect(); - let nodes: Vec<_> = reached - .iter() - .map(|&id| { - let mut node = table.nodes[id].clone(); - node.operator = node.operator.map_children(|child| local[child]); - node - }) - .collect(); - FlatDag { - nodes, - roots: vec![QueryRoot::Operator(local[&root])], - } - }; - for (node, root) in [(sum, sum_root[0]), (max, max_root[0])] { - assert_eq!( - serde_json::to_value(rebuilt(root)).unwrap(), - serde_json::to_value(export(&[QueryRoot::Operator(node)])).unwrap() - ); - } - } -} diff --git a/crates/devtools/src/bin/planner_fuzz.rs b/crates/devtools/src/bin/planner_fuzz.rs index ce59b86ba..7f0fc16ec 100644 --- a/crates/devtools/src/bin/planner_fuzz.rs +++ b/crates/devtools/src/bin/planner_fuzz.rs @@ -15,8 +15,8 @@ // is filed. // // `--relations` takes relation names or the letters a–o (default: all); -// `--search` is `candidate-connections` (the default search) or -// `exhaustive`; `--ignore-accuracy-guarantees` accepts every output +// `--search` is `candidate-connections` (the default search); +// `--ignore-accuracy-guarantees` accepts every output // regardless of its accuracy guarantee (guarantees are enforced by // default, as `plan_installable` does). A relation is violated when it misses by more // than max(`--relative-tolerance` (default 0.01) × cost, @@ -33,7 +33,7 @@ use asap_devtools::planner_fuzz::report::{run_campaign, serve, Campaign, WorkerR use asap_devtools::planner_fuzz::{read_seed_file, FuzzSettings, SearchKind}; const USAGE: &str = "usage: planner_fuzz --seed --cases --time-budget-s \ -[--relations ,...] [--search candidate-connections|exhaustive] \ +[--relations ,...] [--search candidate-connections] \ [--seed-workload ]... [--ignore-accuracy-guarantees] [--relative-tolerance ] \ [--absolute-tolerance ] [--max-shrink-attempts ] [--case-timeout-s ] --out "; @@ -94,7 +94,6 @@ fn run(args: Vec) -> Result<(), String> { "--search" => { settings.search = match value()?.as_str() { "candidate-connections" => SearchKind::CandidateConnections, - "exhaustive" => SearchKind::Exhaustive, other => return Err(format!("unknown search {other}")), } } diff --git a/crates/devtools/src/bin/sketch_coverage.rs b/crates/devtools/src/bin/sketch_coverage.rs index f511b3497..e11557d33 100644 --- a/crates/devtools/src/bin/sketch_coverage.rs +++ b/crates/devtools/src/bin/sketch_coverage.rs @@ -3,7 +3,7 @@ // Lowers every query in every corpus we have (mirrors `variant_coverage`'s // corpus list exactly, so the two reports are directly comparable) with an // *approximate* `AccuracyTarget`, runs Pass 1 of logical ASAP-aware -// optimization (`generate_per_computation_logical_memo_candidates`, every +// optimization (`pass1_memo`, every // summary supported, no accuracy pruning) over each query, and // reports which summary candidates Pass 1 offers for it, plus the fraction of // lowered queries with at least one. @@ -20,11 +20,11 @@ use asap_devtools::corpora::{ }; use asap_devtools::{lower_promql_with_sample_interval, SqlCatalog}; use asap_frontend_sql::lower_sql_dialect; -use asap_logical_asap_optimization::generate_dag_of_logical_memo_groups; -use asap_logical_asap_optimization::generate_per_computation_logical_memo_candidates; +use asap_logical_asap_optimization::memo_of_queries; +use asap_logical_asap_optimization::pass1_memo; use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::schema::FieldDataType; -use asap_types::ir::{ASAPOp, OperatorNode, QueryRoot}; +use asap_types::ir::{ASAPOp, Operator, OperatorNode, QueryRoot}; use asap_types::workload::{DataWorkload, SqlDialect}; use std::collections::BTreeSet; use std::rc::Rc; @@ -60,8 +60,12 @@ fn analyze_corpus( let mut rejected = 0; let mut summaries = Vec::new(); for (id, root) in roots { - let Ok(pass1) = generate_per_computation_logical_memo_candidates( - generate_dag_of_logical_memo_groups(&[QueryRoot::Operator(root)]), + let Ok(memo) = memo_of_queries(&[QueryRoot::Operator(root)]) else { + rejected += 1; + continue; + }; + let Ok(pass1) = pass1_memo( + memo, std::slice::from_ref(accuracy), &IgnoreAccuracyGuarantees, &NoAccuracyEvidence, @@ -76,9 +80,9 @@ fn analyze_corpus( // excluded. A way to compute a measure that Pass 1 offers but cannot // compose into a candidate is not counted. let mut offered = BTreeSet::new(); - for candidate in pass1.memo.candidates.iter().flatten() { + for candidate in pass1.groups().iter().flat_map(|group| &group.candidates) { if candidate.fused_groups.is_empty() { - summary_families(&candidate.root_node, &mut offered); + summary_family(&candidate.expression.operator, &mut offered); } } summaries.push((id, offered)); @@ -92,20 +96,16 @@ fn analyze_corpus( } } -/// Adds the family of every summary `node` and its descendants build to -/// `families`. -fn summary_families(node: &OperatorNode, families: &mut BTreeSet) { - if let Some( +/// Adds the family of the summary `operator` builds, if any, to `families`. +fn summary_family(operator: &Operator, families: &mut BTreeSet) { + if let Operator::ASAP( ASAPOp::SummaryAgg { family, .. } | ASAPOp::SummaryTumblingWindowChain { family, .. }, - ) = node.asap() + ) = operator { if let FieldDataType::Sketch(kind, _) = family { - families.insert(format!("{:?}", kind.algorithm())); + families.insert(format!("{:?}", kind)); } } - for child in node.children() { - summary_families(child, families); - } } fn covered(r: &CorpusCoverage) -> usize { diff --git a/crates/devtools/src/candidate_labels.rs b/crates/devtools/src/candidate_labels.rs index 87ef5d797..8cf029620 100644 --- a/crates/devtools/src/candidate_labels.rs +++ b/crates/devtools/src/candidate_labels.rs @@ -2,7 +2,7 @@ //! `asap_planner` viewer document and text //! only. Planning never reads them: they //! are pure functions of the candidates' planning data (the logical roots -//! and the materialization assignment). +//! and the physical properties assignment). use std::collections::BTreeMap; use std::rc::Rc; @@ -10,10 +10,11 @@ use std::rc::Rc; use serde_json::Value; use asap_types::ir::physical_export::{PhysicalASAPDAG, PhysicalASAPNodeId}; +use asap_types::ir::properties::ExecutionTiming; use asap_types::ir::schema::{FieldDataType, SketchAlgorithm}; use asap_types::ir::NonASAPOp; use asap_types::ir::{ASAPOp, Operator, OperatorNode, QueryRoot}; -use asap_types::physical::Materialization; +use asap_types::physical::Lifetime; use asap_types::physical::PhysicalWorkloadPlan; use asap_types::stages::HowToComputeAggregateMeasure; @@ -90,8 +91,8 @@ fn connected_candidate_label(roots: &[QueryRoot]) -> String { } let mut shared: Vec<_> = producers .values() - .filter(|(_, readers)| readers.len() > 1) - .map(|(name, readers)| format!("shared {name} ({})", query_names(readers))) + .filter(|(_, queries)| queries.len() > 1) + .map(|(name, queries)| format!("shared {name} ({})", query_names(queries))) .collect(); shared.sort(); labels.extend(shared); @@ -144,22 +145,19 @@ fn duration_name(ms: u64) -> String { /// just `logical` when nothing is. pub fn physical_label(logical: &str, candidate: &PhysicalWorkloadPlan) -> String { let mut parts: Vec<(&str, Vec)> = Vec::new(); - for (id, materialization) in &candidate.materialization { - let phase = match materialization { - Materialization::NotMaterialized => continue, - Materialization::MatAtIngestion(_) => "ingestion time", - Materialization::MatAtQuery(_) => "query time, shared", - Materialization::MatAtQueryKeptAcrossEvaluations(_) => "query time, kept", + for node in &candidate.dag.nodes { + let id = &node.id; + let phase = match (node.output_state.timing, node.lifetime) { + (_, Lifetime::None) => continue, + (ExecutionTiming::IngestionTime, _) => "ingestion time", + (ExecutionTiming::QueryTime, Lifetime::OneEvaluation(_)) => "query time, shared", + (ExecutionTiming::QueryTime, Lifetime::AcrossEvaluations(_)) => { + "query time, materialized across evaluations" + } }; - let node = candidate - .dag - .nodes - .iter() - .find(|n| n.id == *id) - .expect("an assigned node is in the DAG"); let label = match &node.payload { // Name the chain: chains of one family and tumbling-window count differ by - // their readers and tumbling-window width. + // their queries and tumbling-window width. Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { tumbling_window_width_ms, tumbling_window_count, @@ -212,7 +210,7 @@ fn state_label(payload: &Operator) -> String { Operator::ASAP( ASAPOp::SummaryAgg { family, .. } | ASAPOp::SummaryTumblingWindowChain { family, .. }, ) => match family { - FieldDataType::Sketch(kind, _) => sketch_name(kind.algorithm()), + FieldDataType::Sketch(algorithm, _) => sketch_name(algorithm), FieldDataType::ExactAggregate(kind, _) => format!("{kind:?} acc"), _ => payload.kind_name().to_string(), }, @@ -401,9 +399,10 @@ mod tests { fn connected_labels_describe_selected_states() { let workload = example1(); let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups( + &asap_logical_asap_optimization::memo_of_queries( &lower_promql_roots(&workload).unwrap(), - ), + ) + .unwrap(), ) .unwrap(); let choice = vec![0; inventory.targets.len()]; @@ -418,9 +417,10 @@ mod tests { fn connected_labels_name_actual_shared_state() { let workload = example1(); let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups( + &asap_logical_asap_optimization::memo_of_queries( &lower_promql_roots(&workload).unwrap(), - ), + ) + .unwrap(), ) .unwrap(); let roots = @@ -430,120 +430,6 @@ mod tests { assert_eq!(label.matches("shared Sum acc").count(), 1); } - /// Two physical candidates that each maintain a different query's tumbling window - /// chain at ingestion time get different labels, even when both chains - /// have the same family and tumbling-window count (6 × 10s for Q1, 6 × 30s for Q2). - #[test] - fn physical_labels_name_the_tumbling_window_chain_maintained_at_ingestion() { - use asap_logical_asap_optimization::test_support::{ - tumbling_window_candidates, window_composition_pass2_candidates, - }; - use asap_physical_asap_optimization::optimize_physical_asap; - use asap_types::workload::PlanningWorkload; - - let workload: PlanningWorkload = serde_json::from_str( - r#"{"query_workload": {"language": "promql", "query_batch": null, "repeating_queries": [ - {"query": "quantile_over_time(0.5, data[1m])", "demand": {"fixed_interval": 10000}, - "requirements": {"accuracy": {"max_error": 0.01, "failure_probability": 0.001}}, - "predictability": {"predictable": {"known_at": null}}}, - {"query": "quantile_over_time(0.5, data[3m])", "demand": {"fixed_interval": 30000}, - "requirements": {"accuracy": {"max_error": 0.01, "failure_probability": 0.001}}, - "predictability": {"predictable": {"known_at": null}}}]}, - "data_workload": {"arrival": {"ingesting": {"rows_per_second": 666.67}}, - "sources": {"data": {"series_count": 10000, "metric_type": "gauge", "sample_interval": 15000}}}}"#, - ) - .unwrap(); - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups( - &lower_promql_roots(&workload).unwrap(), - ), - ) - .unwrap(); - let choice: Vec = inventory - .targets - .iter() - .map(|target| { - target - .ways_to_compute - .iter() - .position(|c| { - *c == HowToComputeAggregateMeasure::Summary(SketchAlgorithm::DDSketch) - }) - .expect("a DDSketch candidate") - }) - .collect(); - let summary: Vec<_> = compose_logical_candidate(&inventory, &choice).unwrap(); - let entries: Vec<_> = workload.query_workload.entries().collect(); - let grids: Vec<_> = entries.iter().map(|e| e.evaluation_grid_ms()).collect(); - let tumbling_windows = tumbling_window_candidates(&summary, &grids); - let roots = window_composition_pass2_candidates(&summary, &tumbling_windows) - .unwrap() - .into_iter() - .find(|c| c.tumbling_window_candidates.len() == 2 && c.shared_chains.is_empty()) - .expect("each query's DDSketch in its own tumbling-window chain") - .roots - .unwrap(); - let plans = optimize_physical_asap( - "L1", - &roots, - &entries, - &workload.data_workload.clone().unwrap_or_default(), - false, - ) - .unwrap(); - // The plans that maintain exactly one tumbling-window chain at ingestion time. - let one_chain_at_ingestion: Vec = plans - .iter() - .filter(|plan| { - plan.materialization - .iter() - .filter(|(id, m)| { - matches!(m, Materialization::MatAtIngestion(_)) - && matches!( - plan.dag - .nodes - .iter() - .find(|n| n.id == **id) - .unwrap() - .payload, - Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) - ) - }) - .count() - == 1 - }) - .map(|plan| physical_label("L1", plan)) - .collect(); - let ingestion = |label: &String| { - label - .split(" · ") - .find(|p| p.starts_with("ingestion time")) - .map(str::to_string) - }; - assert!( - one_chain_at_ingestion - .iter() - .any(|l| ingestion(l).as_deref() - == Some("ingestion time: DDSketch for Q1, 6 × 10s tumbling windows")), - "{one_chain_at_ingestion:#?}" - ); - assert!( - one_chain_at_ingestion - .iter() - .any(|l| ingestion(l).as_deref() - == Some("ingestion time: DDSketch for Q2, 6 × 30s tumbling windows")), - "{one_chain_at_ingestion:#?}" - ); - let mut distinct = one_chain_at_ingestion.clone(); - distinct.sort(); - distinct.dedup(); - assert_eq!( - distinct.len(), - one_chain_at_ingestion.len(), - "{one_chain_at_ingestion:#?}" - ); - } - /// Raw logical operators have an exact label, rather than an empty choice label. #[test] fn connected_labels_name_exact_results() { diff --git a/crates/devtools/src/lib.rs b/crates/devtools/src/lib.rs index f3c546d6e..6fe2ec83f 100644 --- a/crates/devtools/src/lib.rs +++ b/crates/devtools/src/lib.rs @@ -23,7 +23,6 @@ pub mod candidate_labels; pub mod corpora; -pub mod memo_pricing; pub mod planner_fuzz; pub mod planner_layering_examples; pub mod planning_stages; diff --git a/crates/devtools/src/memo_pricing.rs b/crates/devtools/src/memo_pricing.rs deleted file mode 100644 index 128cc4911..000000000 --- a/crates/devtools/src/memo_pricing.rs +++ /dev/null @@ -1,401 +0,0 @@ -//! Price a memo's enumerated choices as the selected plan is priced: each -//! choice's composed DAG (`memo::enumeration::build_logical_asap_dags_from_chosen_candidates`) goes through Stage 2 -//! (`optimize_physical_asap`, every materialization assignment) and Stage 3 -//! (the reference pricer, `asap_plan_selection::reference_pricer`, under the -//! run's models), and the plan it selects for -//! it is installed as `plan_installable` installs one (`installable_plan_of`, -//! which also compiles it with the executor), so a choice cheaper than the -//! selected plan shows that the search missed a plan the memo stands for. - -use std::collections::HashMap; -use std::rc::Rc; - -use asap_logical_asap_optimization::{ - build_logical_asap_dags_from_chosen_candidates, ChosenCandidatePerMemoGroup, -}; -use asap_physical_asap_optimization::optimize_physical_asap; -use asap_plan_selection::reference_pricer::{select_among_plans, WholePlanSelection}; -use asap_plan_selection::{ErrorSelectingPlan, Selection}; -use asap_planner::{installable_plan_of, InstallablePlanError, PlanningModels, PlanningRun}; -use asap_types::ir::physical_export::PhysicalASAPOperatorPayload; -use asap_types::ir::properties::ExecutionTiming; -use asap_types::ir::{NonASAPOp, OperatorNode, QueryRoot}; -use asap_types::physical::PhysicalWorkloadPlan; -use asap_types::stages::Memo; -use asap_types::stages::MemoGroupId; -use asap_types::workload::{DataWorkload, QueryWorkloadEntry}; - -/// One choice's result: valid with its cost, or why it is not valid. -#[derive(Debug, Clone, PartialEq)] -pub struct PricedChoice { - /// Stage 2 implemented it, Stage 3 accepted one of its physical plans, - /// and the plan Stage 3 selected installs (a standalone scalar query, - /// which no choice changes, aside). - pub valid: bool, - /// Why it is not valid: the Stage 3 rejection cause of its first - /// physical plan in snake_case (e.g. `accuracy_target_missed`), or - /// `compose_failed`, `invalid_structure`, `stage2_failed`, - /// `stage3_failed`, `rejected` (no plan and no rejection listed), - /// `not_executable` (the executor cannot compile the selected plan's - /// view refresh or query run), or `not_installable` (installation - /// refuses the selected plan for another reason, e.g. an unsupported - /// raw-data window). - pub rejection: Option, - /// The rejection's one-line reason, with the numbers. - pub rejection_reason: Option, - /// The cheapest cost per second of its valid physical plans under the - /// run's cost model; `None` when not valid or under cost model 3. - pub cost_per_second: Option, - /// Valid and costs less than the selected plan (beyond rounding). - pub cheaper_than_selected: bool, -} - -/// What the choices are compared against: the run's selection. -#[derive(Debug, Clone, Copy, PartialEq)] -pub struct SelectedBaseline { - /// The selected plan's cost per second. - pub cost_per_second: Option, - /// The calibration factor the run priced with, frozen for every choice - /// so each is priced on the same scale as the selected plan. - pub calibration_factor: Option, -} - -impl SelectedBaseline { - /// The baseline of `selection`: its selected plan's cost. Stage 3 - /// reports no calibration factor, so a measured cost table without a - /// frozen factor is priced at each choice's own fit. - pub fn of(selection: &Selection) -> Self { - Self { - cost_per_second: selection.cost.cost_per_second, - calibration_factor: None, - } - } -} - -/// Costs within this fraction of each other are the same sum of charges -/// rounded differently (Stage 3's relative tie tolerance). -const TIE_TOLERANCE: f64 = 1e-9; - -/// Each of `choices` of `memo`, composed, implemented and priced. Query `i` -/// is `stage0_roots[i]` with workload entry `entries[i]`; its operator -/// references are replaced by their root groups' composed DAGs. -pub fn price_choices( - memo: &Memo, - choices: &[ChosenCandidatePerMemoGroup], - stage0_roots: &[QueryRoot], - entries: &[QueryWorkloadEntry], - data: &DataWorkload, - models: PlanningModels<'_>, - selected: Option, -) -> Vec { - let mut settings = models.cost_inputs.clone(); - if settings.frozen_calibration_factor.is_none() { - settings.frozen_calibration_factor = selected.and_then(|s| s.calibration_factor); - } - let models = models.with_cost_inputs(&settings); - let selected_cost = selected.and_then(|s| s.cost_per_second); - // Which root group each query's operator references name is the same - // for every choice. - let root_groups = root_groups_of_queries(memo, stage0_roots); - choices - .iter() - .enumerate() - .map(|(index, choice)| { - let roots = build_logical_asap_dags_from_chosen_candidates(memo, choice) - .map_err(|error| error.to_string()) - .and_then(|composed| query_roots(&composed, stage0_roots, &root_groups)); - let mut priced = price_choice(roots, index, entries, data, models); - priced.cheaper_than_selected = match (priced.cost_per_second, selected_cost) { - (Some(cost), Some(selected)) => cost < selected - TIE_TOLERANCE * selected.abs(), - _ => false, - }; - priced - }) - .collect() -} - -/// One choice, its query roots composed (or why they do not compose), -/// implemented and priced. -fn price_choice( - roots: Result, String>, - index: usize, - entries: &[QueryWorkloadEntry], - data: &DataWorkload, - models: PlanningModels<'_>, -) -> PricedChoice { - let invalid = |cause: &str, reason: String| PricedChoice { - valid: false, - rejection: Some(cause.into()), - rejection_reason: Some(reason), - cost_per_second: None, - cheaper_than_selected: false, - }; - let roots = match roots { - Ok(roots) => roots, - Err(error) => return invalid("compose_failed", error), - }; - if let Err(error) = roots.iter().try_for_each(QueryRoot::validate_structure) { - return invalid("invalid_structure", error.to_string()); - } - let physical = match physical_plans_of(&roots, index, entries, data, models) { - Ok(physical) => physical, - Err(error) => return invalid("stage2_failed", error), - }; - match reference_select(&physical, entries, data, models) { - Ok(selection) => { - match installable_plan_of( - &selection.plan, - None, - &selection.tumbling_window_retention_ms, - entries, - ) { - // A standalone scalar query, which a runtime cannot answer yet, - // does not depend on the choice. - Ok(_) | Err(InstallablePlanError::UnsupportedQueryRoot { .. }) => PricedChoice { - valid: true, - rejection: None, - rejection_reason: None, - cost_per_second: selection.costs[&selection.selected].cost_per_second, - cheaper_than_selected: false, - }, - Err( - error @ (InstallablePlanError::MaterializedViewNotExecutable { .. } - | InstallablePlanError::QueryRunNotExecutable { .. }), - ) => invalid("not_executable", error.to_string()), - Err(error) => invalid("not_installable", error.to_string()), - } - } - Err(ErrorSelectingPlan::NoValidCandidate(rejected)) => match rejected.first() { - Some(first) => invalid( - serde_json::to_value(first.cause) - .ok() - .and_then(|v| v.as_str().map(str::to_owned)) - .as_deref() - .unwrap_or("rejected"), - first.reason.clone(), - ), - None => invalid("rejected", "no valid physical plan".into()), - }, - Err(error) => invalid("stage3_failed", error.to_string()), - } -} - -/// Every physical plan Stage 2 offers for choice `index`'s composed `roots` -/// (each materialization assignment), named `E-`. -pub fn physical_plans_of( - roots: &[QueryRoot], - index: usize, - entries: &[QueryWorkloadEntry], - data: &DataWorkload, - models: PlanningModels<'_>, -) -> Result, String> { - let id = format!("E{}", index + 1); - let mut physical = optimize_physical_asap( - &id, - roots, - entries, - data, - models.deployment_capabilities.query_time_results_retained, - ) - .map_err(|error| error.to_string())?; - retain_deployable(&mut physical, models); - if physical.is_empty() { - return Err("no physical plan".into()); - } - for (k, plan) in physical.iter_mut().enumerate() { - plan.id = format!("{id}-{k}"); - plan.from_logical = id.clone(); - } - Ok(physical) -} - -/// Drops the plans of `physical` that Stage 2 would not offer under -/// `models.deployment_capabilities`: without raw data retained, a plan that -/// scans raw data at query time (Stage 2's `PruneReason::RawDataNotRetained`; -/// `optimize_physical_asap` enumerates every assignment and leaves this -/// rule to Stage 2's catalog, and the reference pricer no longer checks it). -pub fn retain_deployable(physical: &mut Vec, models: PlanningModels<'_>) { - if models.deployment_capabilities.raw_data_retained { - return; - } - physical.retain(|plan| { - plan.dag.nodes.iter().all(|node| { - !matches!( - node.payload, - PhysicalASAPOperatorPayload::NonASAP(NonASAPOp::Scan { .. }) - ) || node.output_state.timing == ExecutionTiming::IngestionTime - }) - }); -} - -/// The reference pricer over `physical` under `models`: every plan checked, -/// priced and ranked, every candidate kept. -pub fn reference_select( - physical: &[PhysicalWorkloadPlan], - entries: &[QueryWorkloadEntry], - data: &DataWorkload, - models: PlanningModels<'_>, -) -> Result { - select_among_plans( - physical, - entries, - data, - &models.accuracy_check_inputs(), - models.cost_inputs, - ) -} - -/// One enumerated memo choice of a reference enumeration: its composed -/// logical roots (or why they do not compose), under id `E`. -#[derive(Debug)] -pub struct EnumeratedChoice { - pub id: String, - pub roots: Result, String>, -} - -/// The reference enumeration: the first `max_choices` choices of the -/// Pass 2 memo (`run.logical_asap_candidates`), each composed, -/// implemented by Stage 2 with every materialization assignment, and all -/// their physical plans priced together by the reference pricer. -#[derive(Debug)] -pub struct ReferenceEnumeration { - pub choices: Vec, - pub physical: Vec, - /// Why a composed choice has no physical plan, by choice id. - pub stage2_failed: Vec<(String, String)>, - /// Whether the memo has more choices than `max_choices`. - pub capped: bool, - pub selection: Result, -} - -/// The reference enumeration of `run` ([`ReferenceEnumeration`]); `None` -/// without a memo. -pub fn reference_enumeration( - run: &PlanningRun, - stage0_roots: &[QueryRoot], - entries: &[QueryWorkloadEntry], - data: &DataWorkload, - models: PlanningModels<'_>, - max_choices: usize, -) -> Option { - use asap_logical_asap_optimization::enumerate; - let memo = &run.logical_asap_candidates.as_ref()?.memo; - let enumeration = enumerate(memo, max_choices.saturating_add(1)); - let capped = !enumeration.complete || enumeration.all_chosen_candidates.len() > max_choices; - let root_groups = root_groups_of_queries(memo, stage0_roots); - let mut choices = Vec::new(); - let mut physical = Vec::new(); - let mut stage2_failed = Vec::new(); - for (index, choice) in enumeration - .all_chosen_candidates - .iter() - .take(max_choices) - .enumerate() - { - let id = format!("E{}", index + 1); - let roots = build_logical_asap_dags_from_chosen_candidates(memo, choice) - .map_err(|error| error.to_string()) - .and_then(|composed| query_roots(&composed, stage0_roots, &root_groups)) - .and_then(|roots| { - roots - .iter() - .try_for_each(QueryRoot::validate_structure) - .map_err(|error| error.to_string())?; - Ok(roots) - }); - if let Ok(roots) = &roots { - match physical_plans_of(roots, index, entries, data, models) { - Ok(plans) => physical.extend(plans), - Err(error) => stage2_failed.push((id.clone(), error)), - } - } - choices.push(EnumeratedChoice { id, roots }); - } - let selection = reference_select(&physical, entries, data, models); - Some(ReferenceEnumeration { - choices, - physical, - stage2_failed, - capped, - selection, - }) -} - -/// For each query, the root group of each of its operator references, in -/// `operator_refs` order: the one group the query's root (in the memo's -/// `query_roots`) reads when its root is one operator, else the group -/// whose node is the reference. -fn root_groups_of_queries( - memo: &Memo, - stage0_roots: &[QueryRoot], -) -> Result>, String> { - stage0_roots - .iter() - .enumerate() - .map(|(query, root)| { - let groups: Vec<_> = memo - .query_roots - .get(query) - .map(|root| { - root.operator_refs() - .into_iter() - .copied() - .collect::>() - }) - .unwrap_or_default() - .into_iter() - .collect(); - match root { - QueryRoot::Operator(_) if groups.len() == 1 => Ok(groups), - QueryRoot::Operator(node) => Ok(vec![group_of(memo, &groups, node, query)?]), - QueryRoot::Scalar(expression) => expression - .operator_refs() - .into_iter() - .map(|node| group_of(memo, &groups, node, query)) - .collect(), - } - }) - .collect() -} - -/// The group among `groups` whose node is `node`: the same node, else an -/// equal one (the memo's group nodes are the frontends' nodes with common -/// sub-DAGs shared). -fn group_of( - memo: &Memo, - groups: &[MemoGroupId], - node: &Rc, - query: usize, -) -> Result { - let node_of = |g: &&MemoGroupId| &memo.groups[g.0 as usize].logical_node; - groups - .iter() - .find(|g| Rc::ptr_eq(node_of(g), node)) - .or_else(|| groups.iter().find(|g| **node_of(g) == **node)) - .copied() - .ok_or_else(|| format!("query {query}'s root has no root group")) -} - -/// Each query's root with every operator reference replaced by the composed -/// DAG of its root group (`root_groups`). -fn query_roots( - composed: &[(MemoGroupId, Rc)], - stage0_roots: &[QueryRoot], - root_groups: &Result>, String>, -) -> Result, String> { - let root_groups = root_groups.as_ref().map_err(String::clone)?; - let composed: HashMap<_, _> = composed.iter().cloned().collect(); - stage0_roots - .iter() - .zip(root_groups) - .map(|(root, groups)| { - let mut next = groups.iter().map(|group| Rc::clone(&composed[group])); - Ok(match root { - QueryRoot::Operator(_) => QueryRoot::Operator(next.next().expect("one group")), - QueryRoot::Scalar(expression) => QueryRoot::Scalar( - expression.map_operator_refs(&mut |_| next.next().expect("a group per ref")), - ), - }) - }) - .collect() -} diff --git a/crates/devtools/src/planner_fuzz/README.md b/crates/devtools/src/planner_fuzz/README.md index 3370fa96f..044dbf232 100644 --- a/crates/devtools/src/planner_fuzz/README.md +++ b/crates/devtools/src/planner_fuzz/README.md @@ -106,7 +106,7 @@ derive_cost = cost(witness) − cost(W's plan) never from a guess: 1. Stage 2 and Stage 3 implement and price the witness the same way they price - any candidate. Each materialization assignment goes through + any candidate. Each physical properties assignment goes through `optimize_physical_asap` and `select_plan`, using the same cost model, deployment capabilities and accuracy checks as the search. The cheapest assignment is kept. 2. `W`'s selected logical plan is priced the same way. This gives the witness diff --git a/crates/devtools/src/planner_fuzz/mod.rs b/crates/devtools/src/planner_fuzz/mod.rs index e37b85e75..c40a2ef4a 100644 --- a/crates/devtools/src/planner_fuzz/mod.rs +++ b/crates/devtools/src/planner_fuzz/mod.rs @@ -8,8 +8,7 @@ //! target, a filter on a group key, a coarser grouping, ...), it builds a //! *witness*: `W`'s selected logical plan plus `q'` derived from `q`'s //! node by one operator. Stage 2 and Stage 3 implement and price the -//! witness exactly as they price any candidate -//! ([`optimize_physical_asap`], the reference pricer), and the relation +//! witness exactly as they price any candidate, and the relation //! //! ```text //! cost(W ∪ {q'}) − cost(W) ≤ derive_cost + tolerance, @@ -35,7 +34,7 @@ pub mod relations; pub mod report; pub mod shrink; -use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::collections::HashMap; use std::rc::Rc; use serde::{Deserialize, Serialize}; @@ -43,7 +42,6 @@ use serde_json::{json, Value}; use asap_cost::CostInputs; use asap_logical_asap_optimization::DeploymentCapabilities; -use asap_physical_asap_optimization::optimize_physical_asap; use asap_planner::{PlanSearchSettings, PlanningModels}; use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::schema::sketch::SketchAlgorithm; @@ -53,9 +51,7 @@ use asap_types::workload::{ SqlDialect, }; -use crate::planning_stages::{ - lower_promql_roots, lower_sql_roots, run_planning_stages_over, run_search_over, sql_catalog, -}; +use crate::planning_stages::{lower_promql_roots, lower_sql_roots, run_search_over, sql_catalog}; use generate::QuerySpec; /// The language of a fuzz workload: one per workload, as one @@ -219,17 +215,9 @@ impl FuzzWorkload { /// The `asap_planner` invocation (a bash command) that plans this /// workload, saved as `workload_file`, with `settings`' accuracy - /// guarantees. `asap_planner` has no exhaustive search, so under - /// `--search exhaustive` the command starts with a comment saying it - /// may not reproduce. Its output document is `.out.json`. + /// guarantees. Its output document is `.out.json`. pub fn asap_planner_command(&self, workload_file: &str, settings: &FuzzSettings) -> String { let mut command = String::new(); - if matches!(settings.search, SearchKind::Exhaustive) { - command.push_str( - "# asap_planner runs the candidate-connection search, not the \ - exhaustive one; it may not reproduce this.\n", - ); - } command.push_str(&format!( "cargo run -p asap-devtools --bin asap_planner -- --workload {workload_file} \ --run-until stage3 --format json" @@ -314,14 +302,8 @@ pub fn read_seed_file(path: &std::path::Path) -> Result { pub enum SearchKind { /// Joint candidate-connection planning, the facade's search. CandidateConnections, - /// Every generated candidate (`--exhaustive`), up to - /// [`EXHAUSTIVE_MAX_COMBINATIONS`]. - Exhaustive, } -/// `--exhaustive`'s default evaluated-assignment budget. -pub const EXHAUSTIVE_MAX_COMBINATIONS: usize = 10_000; - /// How the tester plans and judges. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct FuzzSettings { @@ -368,7 +350,7 @@ pub struct Planned { } /// A plan priced by Stage 2 and Stage 3 outside the search: the cheapest -/// materialization assignment of given logical roots. +/// physical properties assignment of given logical roots. #[derive(Debug, Clone, PartialEq, Serialize)] pub struct Priced { pub selected: String, @@ -443,70 +425,30 @@ impl Planner { .with_deployment_capabilities(&deployment) .with_ignore_accuracy_guarantees(!self.settings.enforce_accuracy_guarantees); let run = match self.settings.search { - SearchKind::Exhaustive => run_planning_stages_over( - roots.clone(), - &entries, - &data, - Some(EXHAUSTIVE_MAX_COMBINATIONS), - models, - ), SearchKind::CandidateConnections => run_search_over( roots.clone(), &entries, &data, models, PlanSearchSettings::default(), - None, ), } .map_err(|e| e.to_string())?; - match &run.reference { - // The reference pricer's selection among the enumerated plans. - Some(reference) => { - let selection = reference - .selection - .as_ref() - .map_err(|e| format!("selection: {e}"))?; - let cost_per_second = selection.costs[&selection.selected] - .cost_per_second - .ok_or("no cost per second (cost model 3)")?; - let from_logical = reference - .physical - .iter() - .find(|p| p.id == selection.selected) - .map(|p| p.from_logical.clone()); - let logical_roots = from_logical.and_then(|id| { - reference - .choices - .iter() - .find(|c| c.id == id) - .and_then(|c| c.roots.as_ref().ok().cloned()) - }); - Ok(Planned { - selected: selection.selected.clone(), - cost_per_second, - stage0_roots: roots, - logical_roots, - }) - } - None => { - let selection = run - .run - .selection - .as_ref() - .map_err(|e| format!("selection: {e}"))?; - let cost_per_second = selection - .cost - .cost_per_second - .ok_or("no cost per second (cost model 3)")?; - Ok(Planned { - selected: "selected".into(), - cost_per_second, - stage0_roots: roots, - logical_roots: run.run.selected_plan.clone(), - }) - } - } + let selection = run + .run + .selection + .as_ref() + .map_err(|e| format!("selection: {e}"))?; + let cost_per_second = selection + .cost + .cost_per_second + .ok_or("no cost per second (cost model 3)")?; + Ok(Planned { + selected: "selected".into(), + cost_per_second, + stage0_roots: roots, + logical_roots: run.run.selected_plan.as_ref().cloned(), + }) } /// Stage 2 and Stage 3 over the logical plan `roots` of `workload`'s @@ -514,7 +456,7 @@ impl Planner { /// assignment implemented and priced, the cheapest selected, with the /// same models as [`plan`](Self::plan). Guarantees are composed again /// first, so a derived node carries the guarantee of what it reads, and - /// summaries are sized for their readers as Pass 1 sizes them. + /// summaries are sized for their consumers as Pass 1 sizes them. /// With `last_root_is_key_filter`, the last root is a `Filter` on /// group-key columns only and keeps its input's guarantee (whole groups /// selected by an exact key keep each remaining row's bound), which @@ -526,93 +468,8 @@ impl Planner { roots: &[QueryRoot], last_root_is_key_filter: bool, ) -> Result { - for root in roots { - root.validate_structure() - .map_err(|e| format!("witness validation: {e}"))?; - } - let mut roots = asap_accuracy::propagation::recompose_guarantees(roots) - .map_err(|e| format!("witness guarantees: {e}"))?; - let planning = workload.planning_workload(); - let entries: Vec<_> = planning.query_workload.entries().collect(); - // A witness is built outside the memo: size its summaries as Pass 1 - // sizes a candidate for these readers, which Stage 3 only checks. - let targets: Vec<_> = entries - .iter() - .map(|entry| entry.requirements.accuracy.clone()) - .collect(); - roots = asap_accuracy::sizing::with_summaries_sized_for_targets( - &roots, - &targets, - &workload.data, - ) - .map_err(|e| format!("witness sizing: {e}"))?; - if let (true, Some(last)) = (last_root_is_key_filter, roots.last_mut()) { - let filter = last.as_operator().ok_or("a scalar root")?; - let input = filter.operator.children()[0].guarantee.clone(); - *last = QueryRoot::Operator(Rc::new((**filter).clone().with_guarantee(input))); - } - let cost_inputs = self.cost_inputs(); - let deployment = workload.deployment(); - let models = PlanningModels::builtin() - .with_cost_inputs(&cost_inputs) - .with_deployment_capabilities(&deployment) - .with_ignore_accuracy_guarantees(!self.settings.enforce_accuracy_guarantees); - let mut candidates = optimize_physical_asap( - "witness", - &roots, - &entries, - &workload.data, - deployment.query_time_results_retained, - ) - .map_err(|e| format!("witness Stage 2: {e}"))?; - for (index, candidate) in candidates.iter_mut().enumerate() { - candidate.id = format!("witness-m{index}"); - } - let selection = - crate::memo_pricing::reference_select(&candidates, &entries, &workload.data, models) - .map_err(|e| format!("witness Stage 3: {e}"))?; - let cost = &selection.costs[&selection.selected]; - let cost_per_second = cost - .cost_per_second - .ok_or("no cost per second (cost model 3)")?; - let candidate = candidates - .iter() - .find(|c| c.id == selection.selected) - .expect("the selected candidate is offered"); - let dag = &candidate.dag; - let mut inputs: BTreeMap<_, Vec<_>> = BTreeMap::new(); - for edge in &dag.edges { - inputs.entry(edge.consumer).or_default().push(edge.producer); - } - let reached = |root| { - let mut seen = BTreeSet::new(); - let mut stack = vec![root]; - while let Some(node) = stack.pop() { - if seen.insert(node) { - stack.extend(inputs.get(&node).into_iter().flatten().copied()); - } - } - seen - }; - let reached_by_root: Vec> = dag.roots.iter().map(|r| reached(*r)).collect(); - let (last, others) = reached_by_root.split_last().ok_or("no roots")?; - let private: Vec<_> = last - .iter() - .filter(|node| others.iter().all(|set| !set.contains(node))) - .filter_map(|node| cost.per_node.get(node)) - .collect(); - let derived_root = PrivateNodes { - cost_per_second: private.iter().filter_map(|n| n.cost_per_second).sum(), - operators: private - .iter() - .map(|n| format!("{:?}", n.operator)) - .collect(), - }; - Ok(Priced { - selected: selection.selected.clone(), - cost_per_second, - derived_root, - }) + let _ = (workload, roots, last_root_is_key_filter); + todo!("#1207 PR 6: price a witness plan through the new Stage 2 and Stage 3") } /// [`price`](Self::price) of `workload`'s selected logical plan diff --git a/crates/devtools/src/planner_fuzz/relations.rs b/crates/devtools/src/planner_fuzz/relations.rs index 1b72e8cc9..3bc274793 100644 --- a/crates/devtools/src/planner_fuzz/relations.rs +++ b/crates/devtools/src/planner_fuzz/relations.rs @@ -987,11 +987,11 @@ fn with_root_estimate( } /// The family of the build whose state `state` carries. -fn state_family(state: &OperatorNode) -> Result { +fn state_family(state: &Rc) -> Result { match state.summary_build() { Some( ASAPOp::SummaryAgg { family, .. } | ASAPOp::SummaryTumblingWindowChain { family, .. }, - ) => Ok(family.clone()), + ) => Ok(family), _ => Err("q's estimate reads no single summary build".into()), } } @@ -1226,7 +1226,7 @@ pub fn witness_root( }) } Witness::OtherStatistic { statistic } => { - let answers = |state: &OperatorNode| -> Result<(), String> { + let answers = |state: &Rc| -> Result<(), String> { let family = state_family(state)?; if FamilyCapability::of(&family).answers(statistic) { Ok(()) diff --git a/crates/devtools/src/planning_stages.rs b/crates/devtools/src/planning_stages.rs index b40f36dea..9efdd4fd9 100644 --- a/crates/devtools/src/planning_stages.rs +++ b/crates/devtools/src/planning_stages.rs @@ -4,9 +4,8 @@ use asap_frontend_sql::SqlCatalog; use asap_logical_asap_optimization::{ - generate_dag_of_logical_memo_groups, generate_per_computation_logical_memo_candidates, - pass2_after_local_window_composition, pass2_local_window_composition, - PerComputationLogicalMemoCandidatesError, + memo_of_queries, pass1_memo, pass2_after_local_window_composition, + pass2_local_window_composition, Pass1Error, }; use asap_physical_asap_optimization::{ generate_physical_asap_memo_candidates, Stage2AssignmentRealizer, @@ -17,20 +16,15 @@ use asap_planner::{ }; use asap_types::ir::schema::schema_support::with_promql_series_identity; use asap_types::ir::schema::{DataType, Field, Schema}; -use asap_types::ir::QueryRoot; +use asap_types::ir::{OperatorNode, QueryRoot}; use asap_types::physical::PhysicalWorkloadPlan; -use asap_types::stages::{AssignmentRealizer, DagOfLogicalMemoGroups, Memo}; +use asap_types::stages::{AssignmentRealizer, Memo}; use asap_types::workload::bounds::with_confirmed_weights_at; use asap_types::workload::{DataWorkload, PlanningWorkload, QueryWorkloadEntry}; use serde_json::Value; use std::collections::HashSet; use std::rc::Rc; -use crate::memo_pricing::{reference_enumeration, ReferenceEnumeration}; - -/// The memo choices the reference enumeration prices when no limit is given. -pub const DEFAULT_REFERENCE_CHOICES: usize = 10_000; - /// The last stage a run runs (`asap_planner --run-until`); the run's /// fields for later stages are left empty. #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] @@ -87,9 +81,6 @@ pub struct PlanningStagesRun { pub pass2_local_window_composition: Option, /// The stages' outputs: Pass 1, Pass 2, Stage 2 and the selection. pub run: PlanningRun, - /// With a reference enumeration: Pass 2's memo choices, each implemented - /// by Stage 2 and all priced by the reference pricer. - pub reference: Option, } /// A logical candidate the document shows: its id and composed roots, or @@ -102,47 +93,30 @@ pub struct ShownLogicalCandidate { impl PlanningStagesRun { /// The logical candidates the document shows: the selected plan (`L1`; - /// its roots as Stage 2 built them, with Stage 3's weight proofs), then - /// with a reference enumeration, its's choices (`E`). + /// its roots as Stage 2 built them, with Stage 3's weight proofs). pub fn logical_candidates(&self) -> Vec { - let selected = self - .run + self.run .selected_plan .iter() - .map(|roots| ShownLogicalCandidate { + .map(|plan| ShownLogicalCandidate { id: "L1".into(), - roots: Ok(roots.clone()), - }); - let enumerated = self.reference.iter().flat_map(|reference| { - reference - .choices - .iter() - .map(|choice| ShownLogicalCandidate { - id: choice.id.clone(), - roots: choice.roots.clone(), - }) - }); - selected.chain(enumerated).collect() + roots: Ok(plan.clone()), + }) + .collect() } /// The physical candidates the document shows: the selected plan as - /// Stage 2 built it (`P1`, from `L1`), then with a reference enumeration every - /// plan the reference pricer priced (`E-`). + /// Stage 2 built it (`P1`, from `L1`). pub fn physical_candidates(&self) -> Vec { - let selected = - self.run - .selected_physical_plan - .iter() - .map(|realized| PhysicalWorkloadPlan { - id: "P1".into(), - from_logical: "L1".into(), - ..realized.plan.clone() - }); - let enumerated = self - .reference + self.run + .selected_physical_plan .iter() - .flat_map(|reference| reference.physical.iter().cloned()); - selected.chain(enumerated).collect() + .map(|realized| PhysicalWorkloadPlan { + id: "P1".into(), + from_logical: "L1".into(), + ..realized.plan.clone() + }) + .collect() } } @@ -215,23 +189,14 @@ impl From for String { } } -/// The planning stages over `workload`, with the reference enumeration of -/// at most `max_combinations` memo choices (default -/// [`DEFAULT_REFERENCE_CHOICES`]). +/// The planning stages over `workload`. pub fn run_planning_stages( workload: &PlanningWorkload, roots: Vec, - max_combinations: Option, ) -> Result { let entries: Vec<_> = workload.query_workload.entries().collect(); let data = workload.data_workload.clone().unwrap_or_default(); - run_planning_stages_over( - roots, - &entries, - &data, - max_combinations, - PlanningModels::builtin(), - ) + run_planning_stages_over(roots, &entries, &data, PlanningModels::builtin()) } /// [`run_planning_stages`] over roots lowered from any mix of query @@ -244,7 +209,6 @@ pub fn run_planning_stages_over( roots: Vec, entries: &[QueryWorkloadEntry], data: &DataWorkload, - max_combinations: Option, models: PlanningModels<'_>, ) -> Result { run_search_over( @@ -252,14 +216,11 @@ pub fn run_planning_stages_over( entries, data, models, - // The reference enumeration reads Pass 2's memo. PlanSearchSettings { inspect: true }, - Some(max_combinations.unwrap_or(DEFAULT_REFERENCE_CHOICES)), ) } -/// The facade's planning, as `plan_installable` runs it, without a -/// reference enumeration. Use the executor's summary capabilities in +/// The facade's planning, as `plan_installable` runs it. Use the executor's summary capabilities in /// `models` when comparing with installation behavior. pub fn run_default_search_over( roots: Vec, @@ -267,53 +228,32 @@ pub fn run_default_search_over( data: &DataWorkload, models: PlanningModels<'_>, ) -> Result { - run_search_over( - roots, - entries, - data, - models, - PlanSearchSettings::default(), - None, - ) + run_search_over(roots, entries, data, models, PlanSearchSettings::default()) } -/// The facade's planning with `settings`, and with `reference_choices` -/// the reference enumeration of that many memo choices. +/// The facade's planning with `settings`. pub fn run_search_over( roots: Vec, entries: &[QueryWorkloadEntry], data: &DataWorkload, models: PlanningModels<'_>, settings: PlanSearchSettings, - reference_choices: Option, ) -> Result { validate(&roots)?; let run = run_planning(roots.clone(), entries, data, models, settings) .map_err(|e| PlanningStagesError::Pass1(e.to_string()))?; - if let Some(roots) = &run.selected_plan { - validate(roots)?; + if let Some(plan) = &run.selected_plan { + validate(plan)?; } - let reference = reference_choices - .and_then(|max| reference_enumeration(&run, &roots, entries, data, models, max)); Ok(PlanningStagesRun { run_until: RunUntil::Stage3, stage0_memo_groups: None, pass2_local_window_composition: None, stage0_query_language_frontends: roots, run, - reference, }) } -/// Stage 0's memo groups, `dag`, without candidates. -fn stage0_memo_groups(dag: DagOfLogicalMemoGroups) -> Memo { - Memo { - candidates: vec![Vec::new(); dag.groups.len()], - groups: dag.groups, - query_roots: dag.query_roots, - } -} - /// The stages over `roots` up to and including `until`, as the facade runs /// them. A stage's failure is reported in the run, as the facade does. // ponytail: copies run_planning's stage order, which has no stop point; @@ -326,14 +266,15 @@ pub fn run_stages_until( until: RunUntil, ) -> Result { validate(&roots)?; - let stage0_dag = generate_dag_of_logical_memo_groups(&roots); + let stage0_dag = + memo_of_queries(&roots).map_err(|e| PlanningStagesError::Validation(e.to_string()))?; let mut stages = PlanningStagesRun { run_until: until, - stage0_memo_groups: Some(stage0_memo_groups(stage0_dag.clone())), + stage0_memo_groups: Some(stage0_dag.clone()), pass2_local_window_composition: None, stage0_query_language_frontends: roots.clone(), run: PlanningRun { - per_computation_candidates: None, + pass1_memo: None, logical_asap_candidates: None, physical_asap_candidates: None, selection: Err(SelectionError::NotRun), @@ -341,7 +282,6 @@ pub fn run_stages_until( selected_plan: None, report: PlanningReport::default(), }, - reference: None, }; if until == RunUntil::Stage0 { return Ok(stages); @@ -352,7 +292,7 @@ pub fn run_stages_until( .map(|entry| entry.requirements.accuracy.clone()) .collect(); let accuracy = models.accuracy_check_inputs(); - let pass1 = generate_per_computation_logical_memo_candidates( + let pass1 = pass1_memo( stage0_dag, &accuracy_targets, accuracy.model, @@ -365,7 +305,7 @@ pub fn run_stages_until( ); // As `run_planning`: a query Pass 1 cannot plan is reported before // contradictory deployment capabilities. - if let Err(PerComputationLogicalMemoCandidatesError::PerComputation(error)) = &pass1 { + if let Err(Pass1Error::PerComputation(error)) = &pass1 { return Err(PlanningStagesError::Pass1(error.to_string())); } if let Err(error) = models.deployment_capabilities.validate(data) { @@ -379,11 +319,17 @@ pub fn run_stages_until( return Ok(stages); } }; - run.per_computation_candidates = Some(pass1.clone()); + run.pass1_memo = Some(pass1.clone()); if until == RunUntil::Pass1 { return Ok(stages); } - let composed = pass2_local_window_composition(pass1, entries); + let composed = match pass2_local_window_composition(pass1, entries) { + Ok(composed) => composed, + Err(error) => { + run.selection = Err(SelectionError::LogicalASAPCandidates(error.to_string())); + return Ok(stages); + } + }; stages.pass2_local_window_composition = Some(composed.clone()); let logical = match pass2_after_local_window_composition(composed) { Ok(logical) => logical, @@ -434,7 +380,7 @@ pub fn run_stages_until( Ok(realized) => { // As the facade does: the plan to run carries the weight // proofs Stage 3 made. - let proven: HashSet<*const asap_types::ir::OperatorNode> = selection + let proven: HashSet<*const OperatorNode> = selection .nodes_with_proven_non_negative_weights .iter() .filter_map(|node| realized.operator_node_of_selected_node.get(node.0)) diff --git a/crates/devtools/src/planning_stages_text.rs b/crates/devtools/src/planning_stages_text.rs index f68332018..e136e4e9c 100644 --- a/crates/devtools/src/planning_stages_text.rs +++ b/crates/devtools/src/planning_stages_text.rs @@ -1,6 +1,6 @@ //! The planner stages' output as plain text: `asap_planner --format text`. //! It prints, up to the run's last stage, the frontends' roots and their -//! memo groups (Stage 0), Pass 1's `PerComputationLogicalMemoCandidates`, +//! memo groups (Stage 0), Pass 1's `Memo`, //! the memo after Pass 2's local window composition and Pass 2's //! `LogicalASAPMemoCandidates` (Stage 2's input), Stage 2's //! `PhysicalASAPMemoCandidates` (Stage 3's input), and Stage 3's @@ -20,14 +20,11 @@ //! [1] TumblingWindows new FinalizeExactAccumulator ← SummaryMerge ← SummaryTumblingWindowChain tumbling_window_count=6 tumbling_window_width_ms=10000 inputs=[G0] //! ``` -use std::collections::HashSet; use std::fmt::Write; -use std::rc::Rc; -use asap_planner::selected_chosen_candidates; use asap_types::ir::flat::flatten; -use asap_types::ir::{OperatorNode, QueryRoot}; -use asap_types::stages::{Memo, MemoGroupId, PhysicalASAPMemoCandidates}; +use asap_types::ir::QueryRoot; +use asap_types::stages::{Memo, MemoGroup, PhysicalASAPMemoCandidates}; use crate::candidate_labels::node_label; use crate::planning_stages::{PlanningStagesRun, RunUntil}; @@ -52,7 +49,7 @@ pub fn planning_stages_text(stages: &PlanningStagesRun) -> String { write_dag(&mut out, &stages.stage0_query_language_frontends); section( &mut out, - "Stage 0 → Stage 1: DagOfLogicalMemoGroups (the roots hash-consed)", + "Stage 0 → Stage 1: Memo (the roots hash-consed, without candidates)", ); if let Some(memo) = &stages.stage0_memo_groups { write_memo(&mut out, memo, None, false); @@ -61,32 +58,12 @@ pub fn planning_stages_text(stages: &PlanningStagesRun) -> String { return out; } - let Some(pass1) = &run.per_computation_candidates else { + let Some(pass1) = &run.pass1_memo else { failure(&mut out); return out; }; - section( - &mut out, - "Stage 1 Pass 1 output: PerComputationLogicalMemoCandidates", - ); - write_memo(&mut out, &pass1.memo, None, true); - let _ = writeln!( - out, - "pruned_candidates ({}):", - pass1.pruned_candidates.len() - ); - let rows = pass1 - .pruned_candidates - .iter() - .map(|pruned| { - vec![ - format!("G{}:", pruned.group.0), - format!("{:?}", pruned.candidate), - format!("{:?}", pruned.reason), - ] - }) - .collect(); - write_rows(&mut out, " ", rows); + section(&mut out, "Stage 1 Pass 1 output: Memo"); + write_memo(&mut out, pass1, None, true); if until == RunUntil::Pass1 { return out; } @@ -102,7 +79,7 @@ pub fn planning_stages_text(stages: &PlanningStagesRun) -> String { &mut out, "Stage 1 Pass 2, after local window composition (tumbling-window candidates)", ); - write_memo(&mut out, pass2, Some(&pass1.memo), true); + write_memo(&mut out, pass2, Some(pass1), true); section( &mut out, "Stage 1 Pass 2 output = Stage 2 input: LogicalASAPMemoCandidates", @@ -126,20 +103,6 @@ pub fn planning_stages_text(stages: &PlanningStagesRun) -> String { } section(&mut out, "Stage 3 output: Selection"); - let chosen = match &run.selection { - Ok(selection) => { - selected_chosen_candidates(&logical.memo, physical, &selection.selected_dag) - } - Err(_) => Err("no plan was selected".into()), - }; - match &chosen { - Ok(choice) => { - let _ = writeln!(out, "selected choice: {}", choice_text(choice)); - } - Err(reason) => { - let _ = writeln!(out, "selected choice unavailable: {reason}"); - } - } write_selection(&mut out, stages); out } @@ -215,47 +178,24 @@ fn reads(inputs: &[String]) -> String { } } -/// `G0, G1`. -fn group_list(groups: &[MemoGroupId]) -> String { - groups - .iter() - .map(|group| format!("G{}", group.0)) - .collect::>() - .join(", ") -} - -/// `G2:c0 G3:c0`: each chosen group's candidate. -fn choice_text(choice: &asap_logical_asap_optimization::ChosenCandidatePerMemoGroup) -> String { - choice - .iter() - .map(|(group, candidate)| format!("G{}:c{candidate}", group.0)) - .collect::>() - .join(" ") -} - /// `memo`'s groups, one line each with the child groups it reads, and with -/// `with_candidates` each group's candidates below it: index, origin, -/// `new` when `previous` (the memo at the step before) has no candidate of -/// that group with the same node and inputs, chain and input groups (fused -/// groups, and accuracy budget parts when more than one). +/// `with_candidates` each group's candidates below it, marked `new` when +/// `previous` (the memo at the step before) has no such candidate. fn write_memo(out: &mut String, memo: &Memo, previous: Option<&Memo>, with_candidates: bool) { // The group lines share one set of column widths, with each group's // candidates printed under its line. let group_rows: Vec> = memo - .groups + .groups() .iter() .enumerate() .map(|(id, group)| { let children: Vec = group - .child_groups - .iter() + .expression + .children() + .into_iter() .map(|child| format!("G{}", child.0)) .collect(); - vec![ - format!("G{id}:"), - root_label(&group.logical_node), - reads(&children), - ] + vec![format!("G{id}:"), group_label(group), reads(&children)] }) .collect(); let mut group_lines = String::new(); @@ -265,43 +205,8 @@ fn write_memo(out: &mut String, memo: &Memo, previous: Option<&Memo>, with_candi if !with_candidates { continue; } - let earlier = previous - .and_then(|memo| memo.candidates.get(id)) - .map_or(&[][..], Vec::as_slice); - let rows = memo.candidates[id] - .iter() - .enumerate() - .map(|(index, candidate)| { - let added = previous.is_some() - && !earlier.iter().any(|before| { - Rc::ptr_eq(&before.root_node, &candidate.root_node) - && before.input_groups == candidate.input_groups - }); - let mut inputs = format!("inputs=[{}]", group_list(&candidate.input_groups)); - if !candidate.fused_groups.is_empty() { - let _ = write!(inputs, " fuses=[{}]", group_list(&candidate.fused_groups)); - } - if candidate.accuracy_budget_parts != 1 { - let _ = write!( - inputs, - " accuracy_budget_parts={}", - candidate.accuracy_budget_parts - ); - } - vec![ - format!("[{index}]"), - format!("{:?}", candidate.origin), - if added { - "new".to_owned() - } else { - String::new() - }, - chain_label(memo, &candidate.root_node, &candidate.input_groups), - inputs, - ] - }) - .collect(); - write_rows(out, " ", rows); + let _ = (previous, &memo.groups()[id].candidates); + todo!("#1207 PR 9: print each candidate's expression, logical properties and input requirements") } let roots: Vec = memo .query_roots @@ -315,112 +220,8 @@ fn write_memo(out: &mut String, memo: &Memo, previous: Option<&Memo>, with_candi } fn write_physical(out: &mut String, physical: &PhysicalASAPMemoCandidates) { - let nodes = &physical.operator_nodes_of_candidates_for_memo_groups; - let _ = writeln!( - out, - "operator_nodes_of_candidates_for_memo_groups ({}):", - nodes.len() - ); - let rows = nodes - .iter() - .enumerate() - .map(|(id, node)| { - let implements: Vec = node - .candidates_for_memo_group - .iter() - .map(|candidate| format!("G{}[{}]", candidate.group.0, candidate.index)) - .collect(); - let mut role = if node.is_implementation_root { - "implementation root".to_owned() - } else { - "private".to_owned() - }; - if node.accuracy_budget_parts != 1 { - let _ = write!( - role, - " accuracy_budget_parts={}", - node.accuracy_budget_parts - ); - } - vec![ - format!("O{id}:"), - root_label(&node.physical_node), - format!("implements=[{}]", implements.join(", ")), - role, - ] - }) - .collect(); - write_rows(out, " ", rows); - - let _ = writeln!(out, "candidates ({}):", physical.candidates.len()); - let rows = physical - .candidates - .iter() - .enumerate() - .map(|(id, candidate)| { - // One `{...}` per input: the candidates that may produce it. - let producers: Vec = candidate - .producer_candidates_per_input - .iter() - .map(|producers| { - let ids: Vec = producers.iter().map(|p| format!("C{}", p.0)).collect(); - format!("{{{}}}", ids.join(", ")) - }) - .collect(); - vec![ - format!("C{id}:"), - format!("O{}", candidate.operator_node.0), - format!("{:?}", candidate.timing), - format!("{:?}", candidate.materialization), - // It may produce a query's output. - if candidate.may_produce_query_output { - "output".to_owned() - } else { - String::new() - }, - reads(&producers), - ] - }) - .collect(); - write_rows(out, " ", rows); - - let output_groups: Vec = physical - .query_roots - .iter() - .map(|root| match root { - QueryRoot::Operator(group) => format!("G{}", group.0), - scalar => format!("{scalar:?}"), - }) - .collect(); - let _ = writeln!(out, "query_roots: [{}]", output_groups.join(", ")); - let _ = writeln!( - out, - "reader_constraints ({}):", - physical.reader_constraints.len() - ); - for constraint in &physical.reader_constraints { - let _ = writeln!(out, " {constraint:?}"); - } - let _ = writeln!( - out, - "pruned_candidates ({}):", - physical.pruned_candidates.len() - ); - let rows = physical - .pruned_candidates - .iter() - .map(|pruned| { - let key = &pruned.candidate; - vec![ - format!("G{}", pruned.group.0), - format!("O{}", key.operator_node.0), - format!("{:?}", key.timing), - format!("{:?}", key.materialization), - format!("{:?}", pruned.reason), - ] - }) - .collect(); - write_rows(out, " ", rows); + let _ = (out, physical); + todo!("#1207 PR 9: print each physical candidate's properties and its edges with α") } /// Stage 3's selection: the search's outcome, the selected DAG (each node's @@ -428,193 +229,15 @@ fn write_physical(out: &mut String, physical: &PhysicalASAPMemoCandidates) { /// node's charge, and each query's accepted accuracy; or why none was /// selected. fn write_selection(out: &mut String, stages: &PlanningStagesRun) { - let run = &stages.run; - let selection = match &run.selection { - Ok(selection) => selection, - Err(error) => { - let _ = writeln!(out, "error: {error}"); - return; - } - }; - let report = &selection.search_report; - let _ = writeln!( - out, - "search: {:?}, {} MILP variables, {} constraints, {} ms solving", - report.status, report.milp_variables, report.milp_constraints, report.solver_ms - ); - let dag = &selection.selected_dag; - let rows = dag - .candidate_of_node - .iter() - .enumerate() - .map(|(node, candidate)| { - let mut inputs: Vec<_> = dag - .edges - .iter() - .filter(|edge| edge.consumer.0 == node) - .collect(); - inputs.sort_by_key(|edge| edge.input); - let producers: Vec = inputs - .iter() - .map(|edge| format!("node{}", edge.producer.0)) - .collect(); - // The operator the candidate runs. - let operator = - run.physical_asap_candidates - .as_ref() - .map_or(String::new(), |physical| { - let node = physical.candidates[candidate.0].operator_node; - root_label( - &physical.operator_nodes_of_candidates_for_memo_groups[node.0] - .physical_node, - ) - }); - vec![ - format!("node{node}:"), - format!("C{}", candidate.0), - operator, - reads(&producers), - ] - }) - .collect(); - write_rows(out, "", rows); - let outputs: Vec = dag - .node_of_output_reference - .iter() - .map(|node| format!("node{}", node.0)) - .collect(); - let _ = writeln!(out, "node_of_output_reference: [{}]", outputs.join(", ")); - let cost = &selection.cost; - let _ = writeln!( - out, - "cost_per_second: {}", - cost.cost_per_second - .map_or("none".to_owned(), |cost| format!("{cost:.6}")) - ); - // Keyed by the selected plan's node ids, as Stage 2 built it. - let rows = cost - .per_node - .iter() - .map(|(node, charge)| { - vec![ - format!("#{node}"), - format!("{:?}", charge.operator), - format!("{:?}", charge.timing), - charge - .cost_per_second - .map_or("none".to_owned(), |cost| format!("{cost:.6}")), - charge.detail.clone(), - ] - }) - .collect(); - write_rows(out, " ", rows); - let _ = writeln!(out, "accepted accuracy:"); - for (query, accepted) in selection.accepted_accuracy_per_query.iter().enumerate() { - let _ = writeln!(out, " q{}: {accepted}", query + 1); - } + let _ = (out, stages); + todo!("#1207 PR 9: print the selected DAG over the new physical candidates") } -/// The label of `node`'s own operator, without its children. -fn root_label(node: &Rc) -> String { - let (flat, _) = flatten(&[QueryRoot::Operator(node.clone())]); - flat.nodes.last().map(node_label).unwrap_or_default() -} - -/// A candidate's own nodes, from `root` down to its `input_groups`' nodes -/// (excluded), root first, joined by " ← ". -fn chain_label(memo: &Memo, root: &Rc, input_groups: &[MemoGroupId]) -> String { - let (flat, originals) = flatten(&[QueryRoot::Operator(root.clone())]); - let is_input = |id: usize| { - input_groups - .iter() - .any(|group| Rc::ptr_eq(&memo.groups[group.0 as usize].logical_node, &originals[id])) - }; - let mut labels = Vec::new(); - let mut pending = vec![flat.nodes.len() - 1]; - let mut seen = HashSet::new(); - while let Some(id) = pending.pop() { - if is_input(id) || !seen.insert(id) { - continue; - } - labels.push(node_label(&flat.nodes[id])); - pending.extend( - flat.nodes[id] - .operator - .children() - .into_iter() - .rev() - .copied(), - ); - } - labels.join(" ← ") +/// The label of `group`'s own operator, without its input groups. +fn group_label(group: &MemoGroup) -> String { + let _ = group; + todo!("#1207 PR 9: label a group by its expression") } #[cfg(test)] -mod tests { - use super::*; - use crate::planner_layering_examples::example1; - use crate::planning_stages::{lower_promql_roots, run_stages_until}; - use asap_planner::PlanningModels; - use asap_types::workload::Query; - - /// One repeating `sum(rate(...[1m]))` query: every section is printed, - /// Pass 2 adds tumbling-window candidates with their layout, Stage 2 offers - /// the tumbling-window chain at ingestion time, and Stage 3 names the selected choice. - #[test] - fn prints_every_stage_of_a_repeating_rate_query() { - let mut workload = example1(); - let queries = workload.query_workload.repeating_queries.as_mut().unwrap(); - queries.truncate(1); - queries[0].query = Query("sum by (job) (rate(http_requests_total[1m]))".into()); - let roots = lower_promql_roots(&workload).unwrap(); - let entries: Vec<_> = workload.query_workload.entries().collect(); - let stages = run_stages_until( - roots, - &entries, - &workload.data_workload.clone().unwrap(), - PlanningModels::builtin(), - RunUntil::Stage3, - ) - .unwrap(); - - let text = planning_stages_text(&stages); - - for heading in [ - "Stage 0 output", - "Stage 0 → Stage 1: DagOfLogicalMemoGroups", - "Stage 1 Pass 1 output: PerComputationLogicalMemoCandidates", - "Stage 1 Pass 2, after local window composition", - "Stage 2 input: LogicalASAPMemoCandidates", - "Stage 3 input: PhysicalASAPMemoCandidates", - "Stage 3 output: Selection", - ] { - assert!(text.contains(heading), "missing {heading}:\n{text}"); - } - // A group line names its operator and the groups it reads. - assert!(text.contains("G1: TimeRange 60s <- G0\n"), "{text}"); - // The line containing every one of `parts`, columns aside. - let has_line = |parts: &[&str]| { - text.lines() - .any(|line| parts.iter().all(|part| line.contains(part))) - }; - // Pass 2 adds a tumbling-window candidate reading the scan, its layout in its chain. - assert!( - has_line(&[ - "TumblingWindows new FinalizeExactAccumulator ← SummaryMerge ← SummaryTumblingWindowChain tumbling_window_count=6 tumbling_window_width_ms=10000", - "inputs=[G0]", - ]), - "{text}" - ); - // Stage 2 offers the tumbling-window chain at ingestion time, read from the - // ingestion-time scan. - assert!( - has_line(&["IngestionTime MatAtIngestion(Memory)", "<- {C1}"]), - "{text}" - ); - assert!( - text.contains("selected choice: G0:c0 G1:c0 G2:c0 G3:c0\n"), - "{text}" - ); - assert!(text.contains("accepted accuracy:\n q1: "), "{text}"); - } -} +mod tests {} diff --git a/crates/devtools/tests/asap_planner.rs b/crates/devtools/tests/asap_planner.rs index 5f655d3b2..c09f75a24 100644 --- a/crates/devtools/tests/asap_planner.rs +++ b/crates/devtools/tests/asap_planner.rs @@ -5,7 +5,6 @@ use std::collections::HashSet; use std::path::{Path, PathBuf}; use std::process::{Command, Output}; -use asap_devtools::planning_stages::{lower_promql_roots, run_stages_until, RunUntil}; use asap_types::ir::flat::FlatDag; use asap_types::ir::physical_export::PhysicalASAPDAG; use asap_types::ir::QueryRoot; @@ -1058,134 +1057,6 @@ fn milp_lower_bound_is_consistent_with_the_selected_cost() { } } -/// A dashboard panel (`max_over_time(m[5m])` every minute over 1M series -/// sampled every 1.5 s) answered as the last hour at 1-minute steps -/// (`range_evaluation`) is priced as 61 evaluations per request: the -/// selected plan costs no more than the tumbling windows kept across evaluations, and -/// the document reports their retention, 1 h + 5 min. The workload section -/// shows the range. An invalid range is refused. -#[test] -fn workload_file_with_a_range_evaluation_is_planned() { - let workload = json!({ - "query_workload": { - "language": "promql", - "query_batch": null, - "repeating_queries": [{ - "query": "max_over_time(m[5m])", - "demand": {"fixed_interval": 60000}, - "requirements": {}, - "predictability": {"predictable": {"known_at": null}}, - "range_evaluation": {"step_ms": 60000, "span_ms": 3600000}, - }], - }, - "data_workload": { - "arrival": {"ingesting": {"rows_per_second": 666666.67}}, - "sources": {"m": {"series_count": 1000000, "sample_interval": 1500}}, - }, - }); - let capabilities = TempFile::new(r#"{"query_time_results_retained": true}"#); - let document = document( - &workload, - &[ - "--run-until", - "stage3", - "--ignore-accuracy-guarantees", - "--deployment-capabilities", - capabilities.path(), - ], - ); - assert_eq!( - document["workload"]["queries"][0]["requirements"]["range_evaluation"], - json!({"step_ms": 60000, "span_ms": 3600000}) - ); - let selection = &document["stage3_plan_selection"]; - let cost = selection["cost"]["cost_per_second"].as_f64().unwrap(); - // The reference: the window merged from tumbling windows kept across evaluations, - // priced under the same models. Its absolute cost is pinned in - // integration-tests/tests/range_evaluation.rs. - let kept_tumbling_windows = { - use asap_accuracy::propagation::recompose_guarantees; - use asap_logical_asap_optimization::test_support::{ - compose_logical_candidate, generate_per_computation_candidates, - }; - use asap_logical_asap_optimization::test_support::{ - tumbling_window_candidates, window_composition_pass2_candidates, - }; - use asap_types::ir::physical_export::PhysicalASAPOperatorPayload; - use asap_types::ir::ASAPOp; - use asap_types::physical::{Materialization, Medium}; - let parsed: PlanningWorkload = serde_json::from_value(workload.clone()).unwrap(); - let entries: Vec<_> = parsed.query_workload.entries().collect(); - let data = parsed.data_workload.clone().unwrap(); - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups( - &lower_promql_roots(&parsed).unwrap(), - ), - ) - .unwrap(); - let exact: Vec<_> = compose_logical_candidate(&inventory, &[0]).unwrap(); - let grids: Vec<_> = entries.iter().map(|e| e.evaluation_grid_ms()).collect(); - let tumbling_windows = tumbling_window_candidates(&exact, &grids); - let windowed = window_composition_pass2_candidates(&exact, &tumbling_windows) - .unwrap() - .remove(0) - .roots - .unwrap(); - let deployment = asap_planner::DeploymentCapabilities { - query_time_results_retained: true, - ..Default::default() - }; - let kept = asap_physical_asap_optimization::optimize_physical_asap( - "kept-tumbling-windows", - &recompose_guarantees(&windowed).unwrap(), - &entries, - &data, - deployment.query_time_results_retained, - ) - .unwrap() - .into_iter() - .find(|plan| { - plan.dag.nodes.iter().any(|n| { - matches!( - n.payload, - PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) - ) && n.materialization - == Materialization::MatAtQueryKeptAcrossEvaluations(Medium::Memory) - }) - }) - .expect("a plan keeps its tumbling windows across evaluations"); - let settings = asap_cost::CostInputs::default(); - let priced = asap_devtools::memo_pricing::reference_select( - std::slice::from_ref(&kept), - &entries, - &data, - asap_planner::PlanningModels::builtin() - .with_cost_inputs(&settings) - .with_deployment_capabilities(&deployment), - ) - .unwrap(); - priced.costs[&priced.selected].cost_per_second.unwrap() - }; - assert!( - cost <= kept_tumbling_windows * (1.0 + 1e-9), - "selected {SELECTED} costs {cost}, more than the kept tumbling windows' {kept_tumbling_windows}" - ); - assert!(selection["tumbling_window_retention_ms"] - .as_object() - .unwrap() - .values() - .any(|retention| retention.as_u64() == Some(3_900_000))); - - let mut invalid = workload.clone(); - invalid["query_workload"]["repeating_queries"][0]["range_evaluation"]["step_ms"] = json!(0); - let invalid = TempFile::json(&invalid); - let stderr = refused(&["--workload", invalid.path(), "--run-until", "stage0"]); - assert!( - stderr.contains("step_ms must be greater than zero"), - "{stderr}" - ); -} - /// The default is to enforce accuracy guarantees: Pass 1 prunes every /// sketch candidate of an average of p99s, which has no guarantee /// (`NoErrorBoundForQueryTarget`). With `--ignore-accuracy-guarantees`, the @@ -1428,7 +1299,7 @@ fn undeclared_supported_summaries_default_to_the_executors_families() { /// Two windowed quantiles over one metric (Q1 over 1 m, Q2 over 5 m, both /// every 10 s): with a shared scan, Stage 2 also offers the scan built once /// per evaluation, which reads the 5 m window once (≈0.1364/s) instead of -/// once per reader (0.2203/s); the selected plan costs at most 0.143/s. +/// once per query (0.2203/s); the selected plan costs at most 0.143/s. #[test] fn a_scan_two_queries_share_is_offered_built_once_per_evaluation() { let workload = json!({ @@ -1459,230 +1330,6 @@ fn a_scan_two_queries_share_is_offered_built_once_per_evaluation() { assert!(cost <= 0.143, "{cost}"); } -/// A scan and a time range that already have every label of their metric -/// are not widened, so no memo group's candidate reads its own group, and -/// Stage 2's candidates, whose producers are its input groups' candidates, -/// read no candidate that reads them back. -#[test] -fn widened_scans_keep_memo_groups_and_stage2_candidates_acyclic() { - use asap_logical_asap_optimization::DeploymentCapabilities; - let workload: PlanningWorkload = serde_json::from_value(json!({ - "query_workload": { - "language": "promql", - "query_batch": null, - "repeating_queries": [ - every_10s_within_1_percent("sum by (job, region) (rate(data[5m]))"), - every_10s_within_1_percent("sum by (job) (rate(data[5m]))"), - every_10s_within_1_percent("sum by (job, region) (data)"), - every_10s_within_1_percent("sum by (job) (data)"), - ], - }, - "data_workload": { - "arrival": {"ingesting": {"rows_per_second": 666.67}}, - "sources": {"data": {"series_count": 10000, "metric_type": "counter", "sample_interval": 15000}}, - }, - })) - .unwrap(); - let entries: Vec<_> = workload.query_workload.entries().collect(); - let deployment = DeploymentCapabilities { - raw_data_retained: true, - ..DeploymentCapabilities::default() - }; - let run = run_stages_until( - lower_promql_roots(&workload).unwrap(), - &entries, - workload.data_workload.as_ref().unwrap(), - asap_planner::PlanningModels::builtin().with_deployment_capabilities(&deployment), - RunUntil::Stage2, - ) - .unwrap() - .run; - let memo = &run.logical_asap_candidates.as_ref().unwrap().memo; - assert!(memo - .candidates - .iter() - .flatten() - .any(|candidate| candidate.origin - == asap_types::stages::OriginOfCandidateForMemoGroup::WidenedScans)); - // Whether `to` is reachable from `from` along `edges`. - fn reaches(edges: &dyn Fn(usize) -> Vec, from: usize, to: usize) -> bool { - let mut seen = HashSet::new(); - let mut pending = edges(from); - while let Some(next) = pending.pop() { - if next == to { - return true; - } - if seen.insert(next) { - pending.extend(edges(next)); - } - } - false - } - let group_reads = |group: usize| -> Vec { - memo.candidates[group] - .iter() - .flat_map(|candidate| candidate.input_groups.iter().map(|g| g.0 as usize)) - .collect() - }; - for group in 0..memo.groups.len() { - assert!( - !reaches(&group_reads, group, group), - "memo group {group} reads itself" - ); - } - let physical = run.physical_asap_candidates.as_ref().unwrap(); - let candidate_reads = |candidate: usize| -> Vec { - physical.candidates[candidate] - .producer_candidates_per_input - .iter() - .flat_map(|producers| producers.iter().map(|id| id.0)) - .collect() - }; - for candidate in 0..physical.candidates.len() { - assert!( - !reaches(&candidate_reads, candidate, candidate), - "Stage 2 candidate {candidate} reads itself" - ); - } -} - -/// #840's workload: Q1 and Q2 window `data` (1 m and 5 m), Q3 and Q4 read -/// its latest samples by (host, job) and by job, so the four scans project -/// different labels. The widening rule's one scan containing every label -/// installs, compiles and is charged once per evaluation, and the selected -/// plan costs no more than it or than the partial-sharing plan (#994). -#[test] -fn scans_projecting_different_labels_are_shared() { - let workload = json!({ - "query_workload": { - "language": "promql", - "query_batch": null, - "repeating_queries": [ - every_10s_within_1_percent("quantile_over_time(0.5, data[1m])"), - every_10s_within_1_percent("quantile_over_time(0.9, data[5m])"), - every_10s_within_1_percent("quantile by (host, job) (0.5, data)"), - every_10s_within_1_percent("quantile by (job) (0.5, data)"), - ], - }, - "data_workload": { - "arrival": {"ingesting": {"rows_per_second": 666.67}}, - "sources": {"data": {"series_count": 10000, "metric_type": "gauge", "sample_interval": 15000}}, - }, - }); - let document = document( - &workload, - &["--run-until", "stage3", "--ignore-accuracy-guarantees"], - ); - let workload: PlanningWorkload = serde_json::from_value(workload).unwrap(); - let entries: Vec<_> = workload.query_workload.entries().collect(); - let data = workload.data_workload.as_ref().unwrap(); - let roots = lower_promql_roots(&workload).unwrap(); - use asap_logical_asap_optimization::test_support::with_union_scan_labels; - use asap_types::physical::{Materialization, Medium}; - // Plans `roots` with identical sub-DAGs made one node, as the memo's - // groups are, and a node read by several roots kept for the evaluation. - let plan = |name: &str, roots: &[QueryRoot]| { - let shared: Vec<_> = asap_logical_asap_optimization::test_support::share_common_sub_dags( - roots - .iter() - .enumerate() - .map(|(id, root)| match root { - QueryRoot::Operator(node) => (id, std::rc::Rc::clone(node)), - QueryRoot::Scalar(_) => panic!("operator root"), - }) - .collect(), - ) - .into_iter() - .map(|(_, node)| QueryRoot::Operator(node)) - .collect(); - let physical = asap_physical_asap_optimization::optimize_physical_asap_assigned( - name, - &shared, - &entries, - data, - false, - &mut |readers| { - Some(if readers.len() > 1 { - Materialization::MatAtQuery(Medium::Memory) - } else { - Materialization::NotMaterialized - }) - }, - ) - .unwrap() - .expect("a node read by several roots can be kept for the evaluation"); - let offered = asap_devtools::memo_pricing::reference_select( - std::slice::from_ref(&physical), - &entries, - data, - asap_planner::PlanningModels::builtin(), - ) - .unwrap(); - (physical, offered) - }; - // The widened scan: one scan with every label, shared by all four. - let widened = with_union_scan_labels(&roots).unwrap().unwrap(); - let (physical, offered) = plan("widened-scan-rule", &widened); - // The partial-sharing reference: the two windows share a narrow scan, and - // the two instant selectors a scan widened to their labels. - let mut partial = roots[..2].to_vec(); - partial.extend(with_union_scan_labels(&roots[2..]).unwrap().unwrap()); - let (_, reference) = plan("partial-sharing", &partial); - let (installed, _) = asap_planner::installable_plan_of( - &physical.roots, - None, - &offered.tumbling_window_retention_ms, - &entries, - ) - .unwrap(); - installed.validate().unwrap(); - installed.check_compiles().unwrap(); - let dag = &physical.dag; - let scans: Vec<_> = dag - .nodes - .iter() - .filter(|n| n.payload.kind_name() == "Scan") - .collect(); - let [scan] = &scans[..] else { - panic!("one scan: {scans:?}") - }; - let field_names: Vec<&str> = scan - .output_schema - .fields - .iter() - .map(|f| f.name.as_str()) - .collect(); - assert_eq!( - field_names, - ["ts", "value", "host", "job", "$promql_series_identity"] - ); - let cost = &offered.costs[&offered.selected]; - let scan_cost = &cost.per_node[&scan.id]; - assert_eq!(scan_cost.multiplier, 1, "{scan_cost:?}"); - // 10,000 series × 20 samples in 5 m, once per 10 s evaluation. - let detail = &scan_cost.detail; - assert!( - detail.starts_with("scan 200000 samples; x 0.1000/s"), - "{detail}" - ); - let shared_cost = cost.cost_per_second.unwrap(); - assert!(shared_cost.is_finite() && shared_cost > 0.0); - let selected_cost = document["stage3_plan_selection"]["cost"]["cost_per_second"] - .as_f64() - .unwrap(); - let reference_cost = reference.costs[&reference.selected] - .cost_per_second - .unwrap(); - assert!( - selected_cost <= shared_cost, - "{selected_cost} > {shared_cost}" - ); - assert!( - selected_cost <= reference_cost * (1.0 + 1e-9), - "{selected_cost} > partial sharing {reference_cost}" - ); -} - /// The search is the candidate-connection search; its report has no /// legacy route. #[test] diff --git a/crates/devtools/tests/asap_planner_acceptance.rs b/crates/devtools/tests/asap_planner_acceptance.rs index 7043d0767..7567cbde3 100644 --- a/crates/devtools/tests/asap_planner_acceptance.rs +++ b/crates/devtools/tests/asap_planner_acceptance.rs @@ -73,13 +73,10 @@ const SECTIONS: [(&str, &[&str]); 5] = [ "stage0", &[ "Stage 0 output: the frontends' roots", - "Stage 0 → Stage 1: DagOfLogicalMemoGroups", + "Stage 0 → Stage 1: Memo (the roots hash-consed, without candidates)", ], ), - ( - "stage1_pass1", - &["Stage 1 Pass 1 output: PerComputationLogicalMemoCandidates"], - ), + ("stage1_pass1", &["Stage 1 Pass 1 output: Memo"]), ( "stage1_pass2", &[ diff --git a/crates/devtools/tests/corpus_snapshots/o11y_bench/q02.json b/crates/devtools/tests/corpus_snapshots/o11y_bench/q02.json index d803d90f7..5cb411f72 100644 --- a/crates/devtools/tests/corpus_snapshots/o11y_bench/q02.json +++ b/crates/devtools/tests/corpus_snapshots/o11y_bench/q02.json @@ -721,7 +721,7 @@ } }, "id": 0, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -861,8 +861,8 @@ { "guarantee": null, "id": 1, - "materialization": { - "MatAtIngestion": "Memory" + "lifetime": { + "AcrossEvaluations": "Memory" }, "output_schema": { "closed": true, @@ -954,7 +954,7 @@ { "guarantee": null, "id": 2, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1044,7 +1044,7 @@ } }, "id": 3, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1112,7 +1112,7 @@ { "guarantee": null, "id": 4, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1197,7 +1197,7 @@ } }, "id": 5, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1257,7 +1257,7 @@ } }, "id": 6, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": false, "fields": [ diff --git a/crates/devtools/tests/corpus_snapshots/o11y_bench/q05.json b/crates/devtools/tests/corpus_snapshots/o11y_bench/q05.json index b986b993f..4d42935a1 100644 --- a/crates/devtools/tests/corpus_snapshots/o11y_bench/q05.json +++ b/crates/devtools/tests/corpus_snapshots/o11y_bench/q05.json @@ -880,7 +880,7 @@ } }, "id": 0, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1034,8 +1034,8 @@ { "guarantee": null, "id": 1, - "materialization": { - "MatAtIngestion": "Memory" + "lifetime": { + "AcrossEvaluations": "Memory" }, "output_schema": { "closed": true, @@ -1127,7 +1127,7 @@ { "guarantee": null, "id": 2, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1217,7 +1217,7 @@ } }, "id": 3, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1285,7 +1285,7 @@ { "guarantee": null, "id": 4, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1356,7 +1356,7 @@ } }, "id": 5, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1404,7 +1404,7 @@ } }, "id": 6, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1528,8 +1528,8 @@ { "guarantee": null, "id": 7, - "materialization": { - "MatAtIngestion": "Memory" + "lifetime": { + "AcrossEvaluations": "Memory" }, "output_schema": { "closed": true, @@ -1613,7 +1613,7 @@ { "guarantee": null, "id": 8, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1695,7 +1695,7 @@ } }, "id": 9, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1755,7 +1755,7 @@ { "guarantee": null, "id": 10, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1826,7 +1826,7 @@ } }, "id": 11, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1874,7 +1874,7 @@ } }, "id": 12, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1932,7 +1932,7 @@ } }, "id": 13, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1971,7 +1971,7 @@ { "guarantee": null, "id": 14, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -2040,7 +2040,7 @@ } }, "id": 15, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ diff --git a/crates/devtools/tests/corpus_snapshots/o11y_bench/q06.json b/crates/devtools/tests/corpus_snapshots/o11y_bench/q06.json index 99affb00a..8cca94ae9 100644 --- a/crates/devtools/tests/corpus_snapshots/o11y_bench/q06.json +++ b/crates/devtools/tests/corpus_snapshots/o11y_bench/q06.json @@ -569,7 +569,7 @@ } }, "id": 0, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -709,8 +709,8 @@ { "guarantee": null, "id": 1, - "materialization": { - "MatAtIngestion": "Memory" + "lifetime": { + "AcrossEvaluations": "Memory" }, "output_schema": { "closed": true, @@ -802,7 +802,7 @@ { "guarantee": null, "id": 2, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -892,7 +892,7 @@ } }, "id": 3, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -960,7 +960,7 @@ { "guarantee": null, "id": 4, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1045,7 +1045,7 @@ } }, "id": 5, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1105,7 +1105,7 @@ } }, "id": 6, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": false, "fields": [ @@ -1172,7 +1172,7 @@ } }, "id": 7, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": false, "fields": [ @@ -1211,7 +1211,7 @@ { "guarantee": null, "id": 8, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": false, "fields": [ @@ -1280,7 +1280,7 @@ } }, "id": 9, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": false, "fields": [ diff --git a/crates/devtools/tests/corpus_snapshots/o11y_bench/q11.json b/crates/devtools/tests/corpus_snapshots/o11y_bench/q11.json index 16f3c5c73..33b444675 100644 --- a/crates/devtools/tests/corpus_snapshots/o11y_bench/q11.json +++ b/crates/devtools/tests/corpus_snapshots/o11y_bench/q11.json @@ -328,7 +328,7 @@ } }, "id": 0, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -438,7 +438,7 @@ } }, "id": 1, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -495,7 +495,7 @@ { "guarantee": null, "id": 2, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -566,7 +566,7 @@ } }, "id": 3, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -614,7 +614,7 @@ } }, "id": 4, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -653,7 +653,7 @@ { "guarantee": null, "id": 5, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -722,7 +722,7 @@ } }, "id": 6, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ diff --git a/crates/devtools/tests/corpus_snapshots/o11y_bench/q12.json b/crates/devtools/tests/corpus_snapshots/o11y_bench/q12.json index 2a4d99a83..8f1b22d92 100644 --- a/crates/devtools/tests/corpus_snapshots/o11y_bench/q12.json +++ b/crates/devtools/tests/corpus_snapshots/o11y_bench/q12.json @@ -133,7 +133,7 @@ } }, "id": 0, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -274,7 +274,7 @@ } }, "id": 1, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ diff --git a/crates/devtools/tests/corpus_snapshots/o11y_bench/q18.json b/crates/devtools/tests/corpus_snapshots/o11y_bench/q18.json index cc49899a7..a491b2f63 100644 --- a/crates/devtools/tests/corpus_snapshots/o11y_bench/q18.json +++ b/crates/devtools/tests/corpus_snapshots/o11y_bench/q18.json @@ -7148,7 +7148,7 @@ } }, "id": 0, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7319,7 +7319,7 @@ } }, "id": 1, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7391,8 +7391,8 @@ { "guarantee": null, "id": 2, - "materialization": { - "MatAtIngestion": "Memory" + "lifetime": { + "AcrossEvaluations": "Memory" }, "output_schema": { "closed": true, @@ -7484,7 +7484,7 @@ { "guarantee": null, "id": 3, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7574,7 +7574,7 @@ } }, "id": 4, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7642,7 +7642,7 @@ { "guarantee": null, "id": 5, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7713,7 +7713,7 @@ } }, "id": 6, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7761,7 +7761,7 @@ } }, "id": 7, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7902,7 +7902,7 @@ } }, "id": 8, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7966,8 +7966,8 @@ { "guarantee": null, "id": 9, - "materialization": { - "MatAtIngestion": "Memory" + "lifetime": { + "AcrossEvaluations": "Memory" }, "output_schema": { "closed": true, @@ -8051,7 +8051,7 @@ { "guarantee": null, "id": 10, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -8133,7 +8133,7 @@ } }, "id": 11, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -8193,7 +8193,7 @@ { "guarantee": null, "id": 12, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -8264,7 +8264,7 @@ } }, "id": 13, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -8312,7 +8312,7 @@ } }, "id": 14, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ diff --git a/crates/devtools/tests/corpus_snapshots/o11y_bench/q19.json b/crates/devtools/tests/corpus_snapshots/o11y_bench/q19.json index 2d5150c66..2e30b8d58 100644 --- a/crates/devtools/tests/corpus_snapshots/o11y_bench/q19.json +++ b/crates/devtools/tests/corpus_snapshots/o11y_bench/q19.json @@ -7012,7 +7012,7 @@ } }, "id": 0, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7166,8 +7166,8 @@ { "guarantee": null, "id": 1, - "materialization": { - "MatAtIngestion": "Memory" + "lifetime": { + "AcrossEvaluations": "Memory" }, "output_schema": { "closed": true, @@ -7259,7 +7259,7 @@ { "guarantee": null, "id": 2, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7349,7 +7349,7 @@ } }, "id": 3, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7417,7 +7417,7 @@ { "guarantee": null, "id": 4, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7488,7 +7488,7 @@ } }, "id": 5, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7536,7 +7536,7 @@ } }, "id": 6, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7660,8 +7660,8 @@ { "guarantee": null, "id": 7, - "materialization": { - "MatAtIngestion": "Memory" + "lifetime": { + "AcrossEvaluations": "Memory" }, "output_schema": { "closed": true, @@ -7745,7 +7745,7 @@ { "guarantee": null, "id": 8, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7827,7 +7827,7 @@ } }, "id": 9, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7887,7 +7887,7 @@ { "guarantee": null, "id": 10, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7958,7 +7958,7 @@ } }, "id": 11, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -8006,7 +8006,7 @@ } }, "id": 12, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ diff --git a/crates/devtools/tests/corpus_snapshots/o11y_bench/q20.json b/crates/devtools/tests/corpus_snapshots/o11y_bench/q20.json index 6b44c713b..3adc6ca62 100644 --- a/crates/devtools/tests/corpus_snapshots/o11y_bench/q20.json +++ b/crates/devtools/tests/corpus_snapshots/o11y_bench/q20.json @@ -219,7 +219,7 @@ } }, "id": 0, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -329,7 +329,7 @@ } }, "id": 1, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -386,7 +386,7 @@ { "guarantee": null, "id": 2, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -457,7 +457,7 @@ } }, "id": 3, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ diff --git a/crates/devtools/tests/corpus_snapshots/o11y_bench/q21.json b/crates/devtools/tests/corpus_snapshots/o11y_bench/q21.json index 8546dd2b2..3ffc88b31 100644 --- a/crates/devtools/tests/corpus_snapshots/o11y_bench/q21.json +++ b/crates/devtools/tests/corpus_snapshots/o11y_bench/q21.json @@ -694,7 +694,7 @@ } }, "id": 0, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -835,7 +835,7 @@ } }, "id": 1, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -899,8 +899,8 @@ { "guarantee": null, "id": 2, - "materialization": { - "MatAtIngestion": "Memory" + "lifetime": { + "AcrossEvaluations": "Memory" }, "output_schema": { "closed": true, @@ -984,7 +984,7 @@ { "guarantee": null, "id": 3, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1066,7 +1066,7 @@ } }, "id": 4, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1126,7 +1126,7 @@ { "guarantee": null, "id": 5, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1197,7 +1197,7 @@ } }, "id": 6, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ diff --git a/crates/devtools/tests/corpus_snapshots/o11y_bench/q22.json b/crates/devtools/tests/corpus_snapshots/o11y_bench/q22.json index b7c91f320..e0a83a8d0 100644 --- a/crates/devtools/tests/corpus_snapshots/o11y_bench/q22.json +++ b/crates/devtools/tests/corpus_snapshots/o11y_bench/q22.json @@ -630,7 +630,7 @@ } }, "id": 0, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -754,8 +754,8 @@ { "guarantee": null, "id": 1, - "materialization": { - "MatAtIngestion": "Memory" + "lifetime": { + "AcrossEvaluations": "Memory" }, "output_schema": { "closed": true, @@ -839,7 +839,7 @@ { "guarantee": null, "id": 2, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -921,7 +921,7 @@ } }, "id": 3, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -981,7 +981,7 @@ { "guarantee": null, "id": 4, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1052,7 +1052,7 @@ } }, "id": 5, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ diff --git a/crates/devtools/tests/corpus_snapshots/o11y_bench/q23.json b/crates/devtools/tests/corpus_snapshots/o11y_bench/q23.json index 2ad030eed..5156b5315 100644 --- a/crates/devtools/tests/corpus_snapshots/o11y_bench/q23.json +++ b/crates/devtools/tests/corpus_snapshots/o11y_bench/q23.json @@ -560,7 +560,7 @@ } }, "id": 0, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -684,8 +684,8 @@ { "guarantee": null, "id": 1, - "materialization": { - "MatAtIngestion": "Memory" + "lifetime": { + "AcrossEvaluations": "Memory" }, "output_schema": { "closed": true, @@ -769,7 +769,7 @@ { "guarantee": null, "id": 2, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -851,7 +851,7 @@ } }, "id": 3, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -911,7 +911,7 @@ { "guarantee": null, "id": 4, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -982,7 +982,7 @@ } }, "id": 5, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ diff --git a/crates/devtools/tests/corpus_snapshots/o11y_bench/q25.json b/crates/devtools/tests/corpus_snapshots/o11y_bench/q25.json index 9f4988b12..c69ea0939 100644 --- a/crates/devtools/tests/corpus_snapshots/o11y_bench/q25.json +++ b/crates/devtools/tests/corpus_snapshots/o11y_bench/q25.json @@ -5332,7 +5332,7 @@ } }, "id": 0, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -5486,8 +5486,8 @@ { "guarantee": null, "id": 1, - "materialization": { - "MatAtIngestion": "Memory" + "lifetime": { + "AcrossEvaluations": "Memory" }, "output_schema": { "closed": true, @@ -5579,7 +5579,7 @@ { "guarantee": null, "id": 2, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -5669,7 +5669,7 @@ } }, "id": 3, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -5737,7 +5737,7 @@ { "guarantee": null, "id": 4, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -5808,7 +5808,7 @@ } }, "id": 5, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -5856,7 +5856,7 @@ } }, "id": 6, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -5980,8 +5980,8 @@ { "guarantee": null, "id": 7, - "materialization": { - "MatAtIngestion": "Memory" + "lifetime": { + "AcrossEvaluations": "Memory" }, "output_schema": { "closed": true, @@ -6065,7 +6065,7 @@ { "guarantee": null, "id": 8, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -6147,7 +6147,7 @@ } }, "id": 9, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -6207,7 +6207,7 @@ { "guarantee": null, "id": 10, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -6278,7 +6278,7 @@ } }, "id": 11, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -6326,7 +6326,7 @@ } }, "id": 12, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ diff --git a/crates/devtools/tests/corpus_snapshots/o11y_bench/q26.json b/crates/devtools/tests/corpus_snapshots/o11y_bench/q26.json index e33bf4553..fbdeba67f 100644 --- a/crates/devtools/tests/corpus_snapshots/o11y_bench/q26.json +++ b/crates/devtools/tests/corpus_snapshots/o11y_bench/q26.json @@ -2618,7 +2618,7 @@ } }, "id": 0, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -2711,8 +2711,8 @@ { "guarantee": null, "id": 1, - "materialization": { - "MatAtIngestion": "Memory" + "lifetime": { + "AcrossEvaluations": "Memory" }, "output_schema": { "closed": true, @@ -2788,7 +2788,7 @@ { "guarantee": null, "id": 2, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -2862,7 +2862,7 @@ } }, "id": 3, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -2914,7 +2914,7 @@ { "guarantee": null, "id": 4, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -2985,7 +2985,7 @@ } }, "id": 5, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ diff --git a/crates/devtools/tests/corpus_snapshots/o11y_bench/q27.json b/crates/devtools/tests/corpus_snapshots/o11y_bench/q27.json index e119b2b13..e87f425a7 100644 --- a/crates/devtools/tests/corpus_snapshots/o11y_bench/q27.json +++ b/crates/devtools/tests/corpus_snapshots/o11y_bench/q27.json @@ -219,7 +219,7 @@ } }, "id": 0, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -329,7 +329,7 @@ } }, "id": 1, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -386,7 +386,7 @@ { "guarantee": null, "id": 2, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -457,7 +457,7 @@ } }, "id": 3, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ diff --git a/crates/devtools/tests/main.rs b/crates/devtools/tests/main.rs index 4952ae75e..ed12546d0 100644 --- a/crates/devtools/tests/main.rs +++ b/crates/devtools/tests/main.rs @@ -5,6 +5,5 @@ mod asap_planner; mod asap_planner_acceptance; mod cross_language; -mod memo_pricing; mod planner_fuzz; mod viewer_contract; diff --git a/crates/devtools/tests/memo_pricing.rs b/crates/devtools/tests/memo_pricing.rs deleted file mode 100644 index 1bdc9e850..000000000 --- a/crates/devtools/tests/memo_pricing.rs +++ /dev/null @@ -1,385 +0,0 @@ -//! Pricing a memo's enumerated choices: on small workloads every choice is -//! priced, and none that is valid is cheaper than the plan the search -//! selected. -use asap_cost::CostInputs; -use asap_devtools::memo_pricing::{price_choices, PricedChoice, SelectedBaseline}; -use asap_devtools::planner_layering_examples::example1; -use asap_devtools::planning_stages::{lower_promql_roots, run_stages_until, RunUntil}; -use asap_logical_asap_optimization::enumerate; -use asap_logical_asap_optimization::generate_dag_of_logical_memo_groups; -use asap_logical_asap_optimization::test_support::generate_per_computation_candidates; -use asap_logical_asap_optimization::test_support::try_populate_pass1; -use asap_logical_asap_optimization::DeploymentCapabilities; -use asap_planner::PlanningModels; -use asap_types::ir::properties::AccuracyTarget; -use asap_types::ir::QueryRoot; -use asap_types::stages::Memo; -use asap_types::workload::{ - DataWorkload, DurationMs, PlanningWorkload, Predictability, Query, QueryLanguage, - QueryRequirements, QueryWorkload, QueryWorkloadEntry, RepeatedDemand, RepeatingEntry, - RepetitionInterval, -}; - -/// The issue's workload file: the query every minute, ε = 0.01, δ = 0.001. -const RATIO_OF_QUANTILES_WORKLOAD: &str = - "../../tools/dag-viewer/memo-examples/promql/ratio-of-quantiles-5m.json"; - -const RATIO_OF_QUANTILES: &str = "avg by (job) (quantile_over_time(0.5, data[5m]) \ - / quantile_over_time(0.5, data[5m]) + quantile_over_time(0.9, data[5m]))"; - -/// One query evaluated every minute at max_error 0.01, failure probability -/// 0.001, over 1,000 series of `data`, each sampled every 15 s. -fn ratio_of_quantiles() -> PlanningWorkload { - PlanningWorkload { - query_workload: QueryWorkload { - language: QueryLanguage::PromQL, - query_batch: None, - repeating_queries: Some(vec![RepeatingEntry { - query: Query(RATIO_OF_QUANTILES.into()), - demand: RepeatedDemand::FixedInterval(RepetitionInterval(60_000)), - requirements: QueryRequirements { - accuracy: AccuracyTarget::max_error(0.01).failure_probability(0.001), - max_latency_ms: None, - }, - predictability: Predictability::Unknown, - range_evaluation: None, - }]), - }, - data_workload: Some({ - let mut data = DataWorkload::sampled_every(["data".to_string()], DurationMs(15_000)); - for facts in data.sources.values_mut() { - facts.series_count = Some(1_000); - } - data - }), - } -} - -/// The workload's roots, entries and data, and its Pass 1 memo. -struct Planned { - roots: Vec, - entries: Vec, - data: DataWorkload, - memo: Memo, -} - -/// `asap_planner` lowers each series' identity as a column -/// (`lower_promql_roots`); without it (`series_identity` false, as the -/// memo-enumeration tests lower), the ratio's two `quantile 0.5` have no -/// provable unique key, but they are still one group. -fn planned(workload: &PlanningWorkload, series_identity: bool) -> Planned { - let roots = if series_identity { - lower_promql_roots(workload).unwrap() - } else { - asap_frontend_promql::lower_promql_query_workload(workload) - .unwrap() - .into_iter() - .collect() - }; - let initial = generate_dag_of_logical_memo_groups(&roots); - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&roots), - ) - .unwrap(); - Planned { - roots, - entries: workload.query_workload.entries().collect(), - data: workload.data_workload.clone().unwrap(), - memo: try_populate_pass1(initial, &inventory).unwrap(), - } -} - -/// `asap_planner`'s models: the executor's summary families, -/// `settings` (accuracy guarantees ignored unless enforced). -fn models<'a>( - settings: &'a Settings, - deployment: &'a DeploymentCapabilities, -) -> PlanningModels<'a> { - PlanningModels::builtin() - .with_cost_inputs(&settings.cost_inputs) - .with_deployment_capabilities(deployment) - .with_ignore_accuracy_guarantees(settings.ignore_accuracy_guarantees) -} - -fn deployment() -> DeploymentCapabilities { - DeploymentCapabilities { - supported_summaries: Some(asap_executor::supported_summary_families()), - ..DeploymentCapabilities::default() - } -} - -/// `asap_planner`'s cost inputs, and whether it ignores accuracy -/// guarantees. -struct Settings { - cost_inputs: CostInputs, - ignore_accuracy_guarantees: bool, -} - -fn settings(enforce_accuracy_guarantees: bool) -> Settings { - Settings { - cost_inputs: CostInputs::default(), - ignore_accuracy_guarantees: !enforce_accuracy_guarantees, - } -} - -/// `asap_planner`'s selection of `planned`'s workload. -fn selected(planned: &Planned, models: PlanningModels<'_>) -> SelectedBaseline { - let run = run_stages_until( - planned.roots.clone(), - &planned.entries, - &planned.data, - models, - RunUntil::Stage3, - ) - .unwrap(); - SelectedBaseline::of(run.run.selection.as_ref().expect("a selection")) -} - -/// Every choice of `planned`'s Pass 1 memo (the enumeration is complete), -/// priced against `baseline`. -fn price_every_choice( - planned: &Planned, - models: PlanningModels<'_>, - baseline: SelectedBaseline, -) -> Vec { - let enumeration = enumerate(&planned.memo, 100_000); - assert!(enumeration.complete); - price_choices( - &planned.memo, - &enumeration.all_chosen_candidates, - &planned.roots, - &planned.entries, - &planned.data, - models, - Some(baseline), - ) -} - -/// Every choice of `workload`'s Pass 1 memo, as `asap_planner` builds it -/// (the enumeration is complete), priced against the plan `asap_planner` -/// selects: none is cheaper, every valid one costs at least as much, and -/// the others say why they were rejected. Returns the priced choices. -fn assert_no_pass1_choice_is_cheaper( - workload: &PlanningWorkload, - enforce_accuracy_guarantees: bool, -) -> Vec { - let planned = planned(workload, true); - let (settings, deployment) = (settings(enforce_accuracy_guarantees), deployment()); - let models = models(&settings, &deployment); - let run = run_stages_until( - planned.roots.clone(), - &planned.entries, - &planned.data, - models, - RunUntil::Stage3, - ) - .unwrap(); - let selected = SelectedBaseline::of(run.run.selection.as_ref().expect("a selection")); - let memo = &run.run.per_computation_candidates.as_ref().unwrap().memo; - let enumeration = enumerate(memo, 100_000); - assert!(enumeration.complete); - let priced = price_choices( - memo, - &enumeration.all_chosen_candidates, - &planned.roots, - &planned.entries, - &planned.data, - models, - Some(selected), - ); - let selected = selected.cost_per_second.unwrap(); - for choice in &priced { - assert!(!choice.cheaper_than_selected, "{choice:?}"); - if choice.valid { - assert!(choice.cost_per_second.unwrap() >= selected, "{choice:?}"); - assert_eq!(choice.rejection, None); - } else { - assert!(choice.rejection.is_some(), "{choice:?}"); - } - } - assert!(priced.iter().any(|choice| choice.valid)); - priced -} - -/// #509 Example 1: its Pass 1 memo stands for 3 DAGs -/// (Q2's top-k group: exact, Count-Min with a heap and Count Sketch with a -/// heap); -/// Q2's summary reads miss its 100 ms latency bound, and no valid DAG is -/// cheaper than the selection. -#[test] -fn example1_no_pass1_choice_is_cheaper_than_the_selected_plan() { - let choices = assert_no_pass1_choice_is_cheaper(&example1(), false); - assert_eq!(choices.len(), 3); - assert!(choices - .iter() - .any(|c| c.rejection.as_deref() == Some("latency_bound_exceeded"))); -} - -/// The issue's ratio of quantiles as `asap_planner` lowers it (series -/// identity: the two `quantile 0.5` share one group): no Pass 1 DAG is -/// cheaper than the selection, with accuracy guarantees ignored or -/// enforced (the default). Enforced, Pass 1 prunes every summary: no query -/// has a guarantee through `/` and `+` over a KLL rank error or a DDSketch -/// relative one, so every listed DAG is exact and valid. -#[test] -fn ratio_of_quantiles_no_pass1_choice_is_cheaper_than_the_selected_plan() { - let workload: PlanningWorkload = - serde_json::from_str(&std::fs::read_to_string(RATIO_OF_QUANTILES_WORKLOAD).unwrap()) - .unwrap(); - let ignored = assert_no_pass1_choice_is_cheaper(&workload, false); - assert!(ignored.len() >= 25); - let enforced = assert_no_pass1_choice_is_cheaper(&workload, true); - assert!(!enforced.is_empty()); - assert!(enforced.iter().all(|choice| choice.valid)); -} - -/// Lowered without series identity, the two `quantile 0.5` have no unique -/// key but return the same rows, so they are still one group: 5 × 5 = 25 -/// DAGs (two distinct approximate inputs), 9 of them sized for the whole -/// budget. The executor compiles none of their per-series summaries without -/// the series identity, so only the exact DAG compiles and is valid. Its price is not -/// compared with the plan selected for the series-identity lowering: its -/// rows carry no identity column, so it prices lower than the same exact -/// DAG with the identity (#1035). -#[test] -fn ratio_of_quantiles_25_pass1_choices_without_identity_only_exact_is_valid() { - let without_identity = planned(&ratio_of_quantiles(), false); - let enumeration = enumerate(&without_identity.memo, 100_000); - let whole_budget = enumeration - .all_chosen_candidates - .iter() - .filter(|choice| { - choice.iter().all(|(group, &index)| { - without_identity.memo.candidates[group.0 as usize][index].accuracy_budget_parts == 1 - }) - }) - .count(); - assert_eq!( - (enumeration.all_chosen_candidates.len(), whole_budget), - (25, 9) - ); - let with_identity = planned(&ratio_of_quantiles(), true); - let deployment = deployment(); - for enforce in [false, true] { - let settings = settings(enforce); - let models = models(&settings, &deployment); - let baseline = selected(&with_identity, models); - let priced = price_every_choice(&without_identity, models, baseline); - assert_eq!(priced.iter().filter(|p| p.valid).count(), 1); - } -} - -/// A selection one step worse than the cheapest valid choice (the -/// next-cheapest cost) makes exactly the cheaper choices -/// `cheaper_than_selected`. -#[test] -fn a_worse_selection_is_caught_by_cheaper_than_selected() { - let planned = planned(&example1(), true); - let (settings, deployment) = (settings(false), deployment()); - let models = models(&settings, &deployment); - let choices = enumerate(&planned.memo, 100).all_chosen_candidates; - let price = |selected| { - price_choices( - &planned.memo, - &choices, - &planned.roots, - &planned.entries, - &planned.data, - models, - selected, - ) - }; - let unpriced = price(None); - assert!(unpriced.iter().all(|p| !p.cheaper_than_selected)); - let mut costs: Vec = unpriced.iter().filter_map(|p| p.cost_per_second).collect(); - costs.sort_by(f64::total_cmp); - costs.dedup(); - assert!(costs.len() >= 2, "two valid choices of different cost"); - let worse = SelectedBaseline { - cost_per_second: Some(costs[1]), - calibration_factor: None, - }; - let priced = price(Some(worse)); - for (before, after) in unpriced.iter().zip(&priced) { - assert_eq!(before.cost_per_second, after.cost_per_second); - assert_eq!( - after.cheaper_than_selected, - after.cost_per_second == Some(costs[0]) - ); - } - assert!(priced.iter().any(|p| p.cheaper_than_selected)); -} - -/// A query whose root is a scalar expression reading two operator DAGs: -/// each reference is replaced by its own root group's composed DAG, so -/// every choice composes and is priced. -#[test] -fn scalar_root_choices_are_composed_and_priced() { - let mut workload = ratio_of_quantiles(); - workload.query_workload.repeating_queries.as_mut().unwrap()[0].query = Query( - "scalar(sum(quantile_over_time(0.5, data[5m]))) + scalar(max(rate(data[1m])))".into(), - ); - let planned = planned(&workload, true); - assert!(matches!(planned.roots[0], QueryRoot::Scalar(_))); - let (settings, deployment) = (settings(false), deployment()); - let enumeration = enumerate(&planned.memo, 100_000); - assert!(enumeration.complete && enumeration.all_chosen_candidates.len() > 1); - let priced = price_choices( - &planned.memo, - &enumeration.all_chosen_candidates, - &planned.roots, - &planned.entries, - &planned.data, - models(&settings, &deployment), - None, - ); - for p in &priced { - assert_ne!(p.rejection.as_deref(), Some("compose_failed"), "{p:?}"); - } - assert!(priced.iter().any(|p| p.valid)); -} - -/// A choice whose selected plan installation refuses for a reason other -/// than the executor (here a raw-data window under a subquery, which no -/// runtime can place) is `not_installable`, never valid, so it is never -/// flagged cheaper than the selected plan. -#[test] -fn a_choice_installation_refuses_is_not_installable() { - let mut workload = ratio_of_quantiles(); - workload.query_workload.repeating_queries.as_mut().unwrap()[0].query = - Query("max_over_time(rate(data[5m])[1h:1m])".into()); - workload - .query_workload - .repeating_queries - .as_mut() - .unwrap() - .truncate(1); - let planned = planned(&workload, true); - let (settings, deployment) = (settings(false), deployment()); - let enumeration = enumerate(&planned.memo, 100_000); - let priced = price_choices( - &planned.memo, - &enumeration.all_chosen_candidates, - &planned.roots, - &planned.entries, - &planned.data, - models(&settings, &deployment), - None, - ); - assert!(!priced.is_empty()); - for p in &priced { - assert!(!p.valid && !p.cheaper_than_selected, "{p:?}"); - } - let refused: Vec<_> = priced - .iter() - .filter(|p| p.rejection.as_deref() == Some("not_installable")) - .collect(); - assert!(!refused.is_empty(), "{priced:?}"); - for p in refused { - let reason = p.rejection_reason.as_deref().unwrap(); - assert!( - reason.contains("a subquery or a TimeShift above the range"), - "{reason}" - ); - } -} diff --git a/crates/devtools/tests/sketch_plan_snapshots.rs b/crates/devtools/tests/sketch_plan_snapshots.rs index e3c9d1be5..1ab0097d1 100644 --- a/crates/devtools/tests/sketch_plan_snapshots.rs +++ b/crates/devtools/tests/sketch_plan_snapshots.rs @@ -6,7 +6,7 @@ //! queries or nested windows, answer a ratio of two quantiles from one tumbling window //! chain, and materialize tumbling windows, under the MVP deployment (raw data not //! retained) and raw data retained. A change to sketch kinds, parameters, -//! sizes, sharing, timing, materialization or cost shows up as a snapshot +//! sizes, sharing, timing, lifetime or cost shows up as a snapshot //! diff. //! //! `tests/sketch_plan_snapshots/manifest.json` lists each workload by id: diff --git a/crates/devtools/tests/sketch_plan_snapshots/fuzz-issue903-raw-retained.json b/crates/devtools/tests/sketch_plan_snapshots/fuzz-issue903-raw-retained.json index 225a81a8a..f27b75631 100644 --- a/crates/devtools/tests/sketch_plan_snapshots/fuzz-issue903-raw-retained.json +++ b/crates/devtools/tests/sketch_plan_snapshots/fuzz-issue903-raw-retained.json @@ -378,7 +378,7 @@ } }, "id": 0, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -488,8 +488,8 @@ } }, "id": 1, - "materialization": { - "MatAtQuery": "Memory" + "lifetime": { + "OneEvaluation": "Memory" }, "output_schema": { "closed": true, @@ -564,7 +564,7 @@ } }, "id": 2, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -634,7 +634,7 @@ { "guarantee": null, "id": 3, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -750,7 +750,7 @@ } }, "id": 4, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -834,7 +834,7 @@ } }, "id": 5, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ diff --git a/crates/devtools/tests/sketch_plan_snapshots/nested-windows-15m-1h-mvp.json b/crates/devtools/tests/sketch_plan_snapshots/nested-windows-15m-1h-mvp.json index 9b4fb2c61..54a2d6f95 100644 --- a/crates/devtools/tests/sketch_plan_snapshots/nested-windows-15m-1h-mvp.json +++ b/crates/devtools/tests/sketch_plan_snapshots/nested-windows-15m-1h-mvp.json @@ -2802,7 +2802,7 @@ } }, "id": 0, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -2895,8 +2895,8 @@ { "guarantee": null, "id": 1, - "materialization": { - "MatAtIngestion": "Memory" + "lifetime": { + "AcrossEvaluations": "Memory" }, "output_schema": { "closed": true, @@ -2986,7 +2986,7 @@ { "guarantee": null, "id": 2, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -3085,7 +3085,7 @@ } }, "id": 3, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -3176,7 +3176,7 @@ } }, "id": 4, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -3249,7 +3249,7 @@ { "guarantee": null, "id": 5, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -3347,7 +3347,7 @@ } }, "id": 6, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -3438,7 +3438,7 @@ } }, "id": 7, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ diff --git a/crates/devtools/tests/sketch_plan_snapshots/quantile-grouped-mvp.json b/crates/devtools/tests/sketch_plan_snapshots/quantile-grouped-mvp.json index 3baaa111e..37361b98d 100644 --- a/crates/devtools/tests/sketch_plan_snapshots/quantile-grouped-mvp.json +++ b/crates/devtools/tests/sketch_plan_snapshots/quantile-grouped-mvp.json @@ -1898,7 +1898,7 @@ } }, "id": 0, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -2007,8 +2007,8 @@ { "guarantee": null, "id": 1, - "materialization": { - "MatAtIngestion": "Memory" + "lifetime": { + "AcrossEvaluations": "Memory" }, "output_schema": { "closed": true, @@ -2106,7 +2106,7 @@ { "guarantee": null, "id": 2, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -2212,7 +2212,7 @@ } }, "id": 3, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -2311,7 +2311,7 @@ } }, "id": 4, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -2398,7 +2398,7 @@ { "guarantee": null, "id": 5, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -2573,7 +2573,7 @@ } }, "id": 6, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -2616,7 +2616,7 @@ { "guarantee": null, "id": 7, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -2701,7 +2701,7 @@ } }, "id": 8, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -2761,7 +2761,7 @@ } }, "id": 9, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -2932,7 +2932,7 @@ } }, "id": 10, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -3092,7 +3092,7 @@ } }, "id": 11, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ diff --git a/crates/devtools/tests/sketch_plan_snapshots/quantile-single-mvp.json b/crates/devtools/tests/sketch_plan_snapshots/quantile-single-mvp.json index 4f357790a..6583082f6 100644 --- a/crates/devtools/tests/sketch_plan_snapshots/quantile-single-mvp.json +++ b/crates/devtools/tests/sketch_plan_snapshots/quantile-single-mvp.json @@ -1461,7 +1461,7 @@ } }, "id": 0, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1554,8 +1554,8 @@ { "guarantee": null, "id": 1, - "materialization": { - "MatAtIngestion": "Memory" + "lifetime": { + "AcrossEvaluations": "Memory" }, "output_schema": { "closed": true, @@ -1647,7 +1647,7 @@ { "guarantee": null, "id": 2, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1739,7 +1739,7 @@ } }, "id": 3, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1823,7 +1823,7 @@ } }, "id": 4, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ diff --git a/crates/devtools/tests/sketch_plan_snapshots/random-mixed-seed-862-long-w01-q3-mvp.json b/crates/devtools/tests/sketch_plan_snapshots/random-mixed-seed-862-long-w01-q3-mvp.json index 04ea77719..d569fff3a 100644 --- a/crates/devtools/tests/sketch_plan_snapshots/random-mixed-seed-862-long-w01-q3-mvp.json +++ b/crates/devtools/tests/sketch_plan_snapshots/random-mixed-seed-862-long-w01-q3-mvp.json @@ -12158,7 +12158,7 @@ } }, "id": 0, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -12267,8 +12267,8 @@ { "guarantee": null, "id": 1, - "materialization": { - "MatAtIngestion": "Memory" + "lifetime": { + "AcrossEvaluations": "Memory" }, "output_schema": { "closed": true, @@ -12366,7 +12366,7 @@ { "guarantee": null, "id": 2, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -12472,7 +12472,7 @@ } }, "id": 3, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -12571,7 +12571,7 @@ } }, "id": 4, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -12658,7 +12658,7 @@ { "guarantee": null, "id": 5, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -12833,7 +12833,7 @@ } }, "id": 6, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -12876,7 +12876,7 @@ { "guarantee": null, "id": 7, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -12961,7 +12961,7 @@ } }, "id": 8, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -13021,7 +13021,7 @@ } }, "id": 9, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -13192,7 +13192,7 @@ } }, "id": 10, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -13352,7 +13352,7 @@ } }, "id": 11, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ diff --git a/crates/devtools/tests/sketch_plan_snapshots/random-mixed-seed-862-w01-q3-q8-mvp.json b/crates/devtools/tests/sketch_plan_snapshots/random-mixed-seed-862-w01-q3-q8-mvp.json index f9db2cbc5..95d124a1c 100644 --- a/crates/devtools/tests/sketch_plan_snapshots/random-mixed-seed-862-w01-q3-q8-mvp.json +++ b/crates/devtools/tests/sketch_plan_snapshots/random-mixed-seed-862-w01-q3-q8-mvp.json @@ -6490,7 +6490,7 @@ } }, "id": 0, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -6599,8 +6599,8 @@ { "guarantee": null, "id": 1, - "materialization": { - "MatAtIngestion": "Memory" + "lifetime": { + "AcrossEvaluations": "Memory" }, "output_schema": { "closed": true, @@ -6698,7 +6698,7 @@ { "guarantee": null, "id": 2, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -6804,7 +6804,7 @@ } }, "id": 3, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -6903,7 +6903,7 @@ } }, "id": 4, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -6990,7 +6990,7 @@ { "guarantee": null, "id": 5, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7165,7 +7165,7 @@ } }, "id": 6, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7208,7 +7208,7 @@ { "guarantee": null, "id": 7, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7293,7 +7293,7 @@ } }, "id": 8, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7353,7 +7353,7 @@ } }, "id": 9, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7524,7 +7524,7 @@ } }, "id": 10, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7684,7 +7684,7 @@ } }, "id": 11, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7759,7 +7759,7 @@ } }, "id": 12, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7868,8 +7868,8 @@ { "guarantee": null, "id": 13, - "materialization": { - "MatAtIngestion": "Memory" + "lifetime": { + "AcrossEvaluations": "Memory" }, "output_schema": { "closed": true, @@ -7953,7 +7953,7 @@ { "guarantee": null, "id": 14, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -8035,7 +8035,7 @@ } }, "id": 15, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -8112,7 +8112,7 @@ } }, "id": 16, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -8201,7 +8201,7 @@ } }, "id": 17, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -8283,7 +8283,7 @@ } }, "id": 18, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ diff --git a/crates/devtools/tests/sketch_plan_snapshots/random-mixed-seed-862-w02-q2-q7-mvp.json b/crates/devtools/tests/sketch_plan_snapshots/random-mixed-seed-862-w02-q2-q7-mvp.json index 6de034914..94d046d27 100644 --- a/crates/devtools/tests/sketch_plan_snapshots/random-mixed-seed-862-w02-q2-q7-mvp.json +++ b/crates/devtools/tests/sketch_plan_snapshots/random-mixed-seed-862-w02-q2-q7-mvp.json @@ -7250,7 +7250,7 @@ } }, "id": 0, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7359,8 +7359,8 @@ { "guarantee": null, "id": 1, - "materialization": { - "MatAtIngestion": "Memory" + "lifetime": { + "AcrossEvaluations": "Memory" }, "output_schema": { "closed": true, @@ -7458,7 +7458,7 @@ { "guarantee": null, "id": 2, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7564,7 +7564,7 @@ } }, "id": 3, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7663,7 +7663,7 @@ } }, "id": 4, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7750,7 +7750,7 @@ { "guarantee": null, "id": 5, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7925,7 +7925,7 @@ } }, "id": 6, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7968,7 +7968,7 @@ { "guarantee": null, "id": 7, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -8053,7 +8053,7 @@ } }, "id": 8, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -8113,7 +8113,7 @@ } }, "id": 9, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -8284,7 +8284,7 @@ } }, "id": 10, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -8444,7 +8444,7 @@ } }, "id": 11, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -8519,7 +8519,7 @@ } }, "id": 12, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -8628,8 +8628,8 @@ { "guarantee": null, "id": 13, - "materialization": { - "MatAtIngestion": "Memory" + "lifetime": { + "AcrossEvaluations": "Memory" }, "output_schema": { "closed": true, @@ -8713,7 +8713,7 @@ { "guarantee": null, "id": 14, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -8795,7 +8795,7 @@ } }, "id": 15, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -8872,7 +8872,7 @@ } }, "id": 16, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -8961,7 +8961,7 @@ } }, "id": 17, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -9043,7 +9043,7 @@ } }, "id": 18, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ diff --git a/crates/devtools/tests/sketch_plan_snapshots/ratio-of-quantiles-1h-mvp.json b/crates/devtools/tests/sketch_plan_snapshots/ratio-of-quantiles-1h-mvp.json index 0d170be65..266118af6 100644 --- a/crates/devtools/tests/sketch_plan_snapshots/ratio-of-quantiles-1h-mvp.json +++ b/crates/devtools/tests/sketch_plan_snapshots/ratio-of-quantiles-1h-mvp.json @@ -2890,7 +2890,7 @@ } }, "id": 0, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -2983,8 +2983,8 @@ { "guarantee": null, "id": 1, - "materialization": { - "MatAtIngestion": "Memory" + "lifetime": { + "AcrossEvaluations": "Memory" }, "output_schema": { "closed": true, @@ -3074,7 +3074,7 @@ { "guarantee": null, "id": 2, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -3172,7 +3172,7 @@ } }, "id": 3, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -3263,7 +3263,7 @@ } }, "id": 4, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -3336,7 +3336,7 @@ { "guarantee": null, "id": 5, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -3434,7 +3434,7 @@ } }, "id": 6, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -3525,7 +3525,7 @@ } }, "id": 7, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -3598,7 +3598,7 @@ { "guarantee": null, "id": 8, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ diff --git a/crates/devtools/tests/sketch_plan_snapshots/shared-quantiles-1h-mvp.json b/crates/devtools/tests/sketch_plan_snapshots/shared-quantiles-1h-mvp.json index 227c9d39c..fa094b72a 100644 --- a/crates/devtools/tests/sketch_plan_snapshots/shared-quantiles-1h-mvp.json +++ b/crates/devtools/tests/sketch_plan_snapshots/shared-quantiles-1h-mvp.json @@ -2802,7 +2802,7 @@ } }, "id": 0, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -2895,8 +2895,8 @@ { "guarantee": null, "id": 1, - "materialization": { - "MatAtIngestion": "Memory" + "lifetime": { + "AcrossEvaluations": "Memory" }, "output_schema": { "closed": true, @@ -2986,7 +2986,7 @@ { "guarantee": null, "id": 2, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -3084,7 +3084,7 @@ } }, "id": 3, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -3175,7 +3175,7 @@ } }, "id": 4, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -3248,7 +3248,7 @@ { "guarantee": null, "id": 5, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -3346,7 +3346,7 @@ } }, "id": 6, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -3437,7 +3437,7 @@ } }, "id": 7, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ diff --git a/crates/devtools/tests/sketch_plan_snapshots/topk-rate-guarantees-ignored-mvp.json b/crates/devtools/tests/sketch_plan_snapshots/topk-rate-guarantees-ignored-mvp.json index b49a890c2..3d269fb2c 100644 --- a/crates/devtools/tests/sketch_plan_snapshots/topk-rate-guarantees-ignored-mvp.json +++ b/crates/devtools/tests/sketch_plan_snapshots/topk-rate-guarantees-ignored-mvp.json @@ -1571,7 +1571,7 @@ } }, "id": 0, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1680,8 +1680,8 @@ { "guarantee": null, "id": 1, - "materialization": { - "MatAtIngestion": "Memory" + "lifetime": { + "AcrossEvaluations": "Memory" }, "output_schema": { "closed": true, @@ -1765,7 +1765,7 @@ { "guarantee": null, "id": 2, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1847,7 +1847,7 @@ } }, "id": 3, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1907,7 +1907,7 @@ { "guarantee": null, "id": 4, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1984,7 +1984,7 @@ { "guarantee": null, "id": 5, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -2032,7 +2032,7 @@ { "guarantee": null, "id": 6, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ diff --git a/crates/executor/src/capability.rs b/crates/executor/src/capability.rs index f7a2e3afd..00aa39b12 100644 --- a/crates/executor/src/capability.rs +++ b/crates/executor/src/capability.rs @@ -8,12 +8,11 @@ //! //! Stored-state encodings belong to deployments. Full plan acceptance is //! owned by `binding`, which also validates schemas, expressions and inputs. -use crate::summary_kernels::hll_sketch::HLL_PRECISIONS; -use crate::summary_kernels::weighted_frequency::{self, FrequencyAlgorithm}; + use crate::Error; use planner_types::ir::schema::{ ExactKind, ExactParams, FamilyCapability, FieldDataType as SummaryFamilyType, GroupingStrategy, - SketchAlgorithm, SketchParams, SketchStatistic, StatisticKind, SummaryUpdate, + SketchAlgorithm, SketchStatistic, StatisticKind, SummaryUpdate, }; /// Check the same contract used by `create_planner_accumulator` before a plan @@ -47,76 +46,8 @@ pub fn validate_summary_kernel( if layout != grouping { return Err("Planner family and operator grouping disagree".into()); } - use SketchAlgorithm as A; - use SketchParams as P; - match (kind.algorithm(), kind.params()) { - // `asap_sketchlib` silently caps a larger `k`. - (A::Kll, P::Kll { k, .. }) - if (8..=planner_types::ir::schema::sketch::KLL_MAX_K).contains(k) => - { - false - } - (A::DDSketch, P::DDSketch { alpha }) - if alpha.is_finite() && *alpha > 0.0 && *alpha < 1.0 => - { - false - } - (A::Hll, P::Hll { precision, .. }) if HLL_PRECISIONS.contains(precision) => false, - (A::Kmv, P::Kmv { k, .. }) if *k >= crate::summary_kernels::kmv::KMV_MIN_K => false, - (A::Cms, P::Cms { width, depth }) if valid_matrix(*width, *depth) => true, - // The kernel's shape rules (an odd depth, for the median). - (A::CountSketch, P::CountSketch { width, depth }) - if valid_matrix(*width, *depth) - && weighted_frequency::check_shape( - FrequencyAlgorithm::CountSketch, - *width as usize, - *depth as usize, - ) - .is_ok() => - { - true - } - ( - A::CmsWithHeap, - P::CmsWithHeap { - width, - depth, - heap_size, - }, - ) - | ( - A::CountSketchWithHeap, - P::CountSketchWithHeap { - width, - depth, - heap_size, - }, - ) if valid_matrix(*width, *depth) && *heap_size > 0 => true, - ( - A::UnivMon, - P::UnivMon { - heap_size, - sketch_rows, - sketch_cols, - layers, - .. - }, - ) if crate::summary_kernels::univmon::check_shape( - *heap_size as usize, - *sketch_rows as usize, - *sketch_cols as usize, - usize::from(*layers), - ) - .is_ok() => - { - false - } - _ => { - return Err(format!( - "unsupported kernel or invalid parameters: {kind:?}" - )) - } - } + let _ = kind; + todo!("#1207 PR 6: check θ, which the exported node carries") } _ => return Err(format!("unsupported summary kernel {family:?}")), }; @@ -188,41 +119,20 @@ pub(crate) fn constant_row_weight(weight: f64) -> Result Result<(usize, usize, usize, usize), String> { - use planner_types::ir::schema::{HydraKind, HydraParams}; + use planner_types::ir::schema::HydraKind; let SummaryFamilyType::Sketch( kind, GroupingStrategy::SharedMultiSubpopulation { kind: HydraKind::HydraCms, - params: - HydraParams::HydraCms { - width, - depth, - shared_rows, - shared_columns, - }, }, ) = family else { return Err("a shared-grouping (Hydra) family has no registered kernel".into()); }; - if kind.algorithm() != &SketchAlgorithm::Cms - || kind.params() - != &(SketchParams::Cms { - width: *width, - depth: *depth, - }) - { + if kind != &SketchAlgorithm::Cms { return Err("HydraCms family must be Count-Min with the Hydra width and depth".into()); } - if !valid_matrix(*width, *depth) || !valid_matrix(*shared_columns, *shared_rows) { - return Err("invalid HydraCms dimensions".into()); - } - Ok(( - *shared_rows as usize, - *shared_columns as usize, - *width as usize, - *depth as usize, - )) + todo!("#1207 PR 6: the Hydra grid's sizes are θ, which the exported node carries") } /// A filtered `SummaryAgg` never runs at ingestion time. An ingestion-time @@ -230,7 +140,7 @@ pub(crate) fn hydra_cms_shape( /// none of the query's columns for the filter predicate to read, so only a /// query-time build can apply it. The ingestion-time compiler and the /// binding of any ingestion-timed `SummaryAgg` apply this check. The planner -/// applies the same rule: Stage 2 never offers `MatAtIngestion` for a +/// applies the same rule: Stage 2 never offers materializing at ingestion time for a /// filtered build (`IngestionPruneReason::Filtered`), and materialization /// timing rejects an explicit one (`FilteredSummaryAtIngestion`). This check /// guards plans that did not come from the planner. @@ -243,6 +153,7 @@ pub fn validate_ingestion_time_summary( } } +#[allow(dead_code)] // Used again in #1207 PR 6, with θ from the exported node. fn valid_matrix(width: u32, depth: u32) -> bool { // Construction uses the kernel's native row hashing, so no encoded-size // limit applies here. @@ -279,36 +190,19 @@ pub fn validate_native_family(family: &SummaryFamilyType) -> Result<(), Error> { // Plain Count-Min is native as stored state only: it merges and reads // its bare count, but the DAG does not build it from rows. HydraCms // is built by the all-groups summary build. - if let (A::Cms, SketchParams::Cms { width, depth }) = (kind.algorithm(), kind.params()) { + if let A::Cms = kind { if grouping != &GroupingStrategy::PerSubpopulationInstance { return hydra_cms_shape(family).map(|_| ()).map_err(Error::Invalid); } - return if valid_matrix(*width, *depth) { - Ok(()) - } else { - Err(Error::Invalid( - "invalid Count-Min dimensions or grouping strategy".into(), - )) - }; + todo!("#1207 PR 6: check the Count-Min θ, which the exported node carries") } - if matches!(kind.algorithm(), A::CmsWithHeap | A::CountSketchWithHeap) { - let (_, width, depth, _) = weighted_frequency::WeightedFrequency::configuration(kind)?; - return if valid_matrix(width as u32, depth as u32) && grouping == &Default::default() { - Ok(()) - } else { - Err(Error::Invalid( - "invalid weighted frequency dimensions or grouping strategy".into(), - )) - }; + if matches!(kind, A::CmsWithHeap | A::CountSketchWithHeap) { + todo!("#1207 PR 6: check the weighted-frequency θ, which the exported node carries") } } match family { SummaryFamilyType::ExactAggregate(..) => {} - SummaryFamilyType::Sketch(kind, _) - if matches!( - kind.algorithm(), - A::Kll | A::DDSketch | A::Hll | A::Kmv | A::UnivMon - ) => {} + SummaryFamilyType::Sketch(A::Kll | A::DDSketch | A::Hll | A::Kmv | A::UnivMon, _) => {} _ => { return Err(Error::Invalid( "summary family has no native DAG state implementation".into(), @@ -359,7 +253,7 @@ fn evaluates(family: &SummaryFamilyType, readout: StatisticKind) -> bool { SummaryFamilyType::Sketch(kind, GroupingStrategy::PerSubpopulationInstance) => { EVALUATED_READOUTS .iter() - .any(|(family, readouts)| family == kind.algorithm() && readouts.contains(&readout)) + .any(|(family, readouts)| family == kind && readouts.contains(&readout)) } SummaryFamilyType::Sketch(_, GroupingStrategy::SharedMultiSubpopulation { kind, .. }) => { matches!(kind, HydraKind::HydraCms | HydraKind::HydraKll) diff --git a/crates/executor/src/operators/summary/mod.rs b/crates/executor/src/operators/summary/mod.rs index 273597d46..4f40f2a06 100644 --- a/crates/executor/src/operators/summary/mod.rs +++ b/crates/executor/src/operators/summary/mod.rs @@ -40,58 +40,8 @@ impl Operator { row_weight: RowWeight, groups: Vec, ) -> Result { - use crate::summary_kernels::weighted_frequency::WeightedFrequency; - crate::values::validate_family(&family)?; - let SummaryFamilyType::Sketch(kind, _) = &family else { - return Err(invalid("keyed sketch required")); - }; - WeightedFrequency::configuration(kind)?; - validate_groups(&input, &groups)?; - if items.is_empty() { - return Err(invalid("keyed summary requires item columns")); - } - if let RowWeight::Column(weight) = row_weight { - if !matches!( - plain(&input, weight)?.0, - DataType::Float64 | DataType::Int64 - ) { - return Err(invalid("keyed summary weights must be Float64 or Int64")); - } - } - for &item in &items { - if !matches!( - plain(&input, item)?.0, - DataType::Utf8 - | DataType::Timestamp - | DataType::Int64 - | DataType::Float64 - | DataType::Bool - | DataType::Null - ) { - return Err(invalid("unsupported keyed summary identity type")); - } - } - let mut fields = groups - .iter() - .map(|&i| input.fields[i].clone()) - .collect::>(); - fields.push(SummaryField { - name: "state".into(), - dtype: family.clone(), - nullable: false, - table: None, - }); - Ok(Self { - kind: Kind::PerGroupKeyedSummaryBuild { - family, - items, - row_weight, - groups, - filter: None, - }, - inputs: vec![input], - output: schema(fields), - }) + let _ = (input, family, items, row_weight, groups); + todo!("#1207 PR 6: the weighted-frequency θ, which the exported node carries") } /// One summary state shared by all groups (a HydraCms grid), emitted as /// one row per group that reads its own part of it. Each row adds its @@ -177,29 +127,8 @@ impl Operator { k: usize, output: SchemaRef, ) -> Result { - use crate::summary_kernels::weighted_frequency::WeightedFrequency; - crate::values::validate_family(&field(&input, state)?.dtype)?; - let SummaryFamilyType::Sketch(kind, _) = &field(&input, state)?.dtype else { - return Err(invalid("keyed evaluation requires summary state")); - }; - let (_, _, _, capacity) = WeightedFrequency::configuration(kind)?; - if k > capacity || output.fields.len() <= input.fields.len() { - return Err(invalid("invalid keyed evaluation shape or capacity")); - } - if state + 1 != input.fields.len() - || output.fields[..state] != input.fields[..state] - || output.fields.last().unwrap().dtype != SummaryFamilyType::Plain(DataType::Float64) - { - return Err(invalid( - "keyed evaluation must preserve partitions and return a Float64 score", - )); - } - crate::values::validate_schema(&output)?; - Ok(Self { - kind: Kind::KeyedEvaluation { state, k }, - inputs: vec![input], - output, - }) + let _ = (input, state, k, output); + todo!("#1207 PR 6: the weighted-frequency capacity of θ, which the exported node carries") } /// One summary state per group, updated from the value of one input /// column per row (`time` orders counter samples): an exact aggregate, @@ -366,8 +295,8 @@ fn accepts_update_type(family: &SummaryFamilyType, dtype: &DataType) -> bool { }; match dtype { DataType::Float64 | DataType::Int64 => true, - DataType::Utf8 => matches!(kind.algorithm(), A::Kmv | A::Hll | A::UnivMon), - DataType::Bool => kind.algorithm() == &A::UnivMon, + DataType::Utf8 => matches!(kind, A::Kmv | A::Hll | A::UnivMon), + DataType::Bool => kind == &A::UnivMon, _ => false, } } @@ -401,7 +330,7 @@ impl Operator { /// non-integral total rather than rounding it. fn integral_count(family: &SummaryFamilyType, query: &SummaryEvaluation) -> bool { matches!(family, SummaryFamilyType::Sketch(kind, _) - if kind.algorithm() == &planner_types::ir::schema::SketchAlgorithm::Cms) + if kind == &planner_types::ir::schema::SketchAlgorithm::Cms) && matches!( query, SummaryEvaluation::Sketch(planner_types::ir::schema::SketchStatistic::PointCount { diff --git a/crates/executor/src/physical_planner/logical.rs b/crates/executor/src/physical_planner/logical.rs index fc4f78c91..4b4d58ff1 100644 --- a/crates/executor/src/physical_planner/logical.rs +++ b/crates/executor/src/physical_planner/logical.rs @@ -36,8 +36,6 @@ pub(super) fn restore(dag: &PhysicalASAPDAG) -> Result bool { +fn built_from_relation(state: &std::rc::Rc) -> bool { match state.asap() { - Some(ASAPOp::SummaryMerge { children, .. }) => { - children.iter().all(|child| built_from_relation(child)) - } + Some(ASAPOp::SummaryMerge { children, .. }) => children.iter().all(built_from_relation), Some( ASAPOp::SummaryAgg { child, .. } | ASAPOp::SummaryTumblingWindowChain { child, .. }, ) => child.result_kind == planner_types::ir::OperatorResultKind::Relation, // A filter keeping whole states reads them from their build. _ => planner_types::ir::operator::asap::selected_from(state) - .is_some_and(|states| built_from_relation(states)), + .is_some_and(|states| built_from_relation(&states)), } } @@ -1336,7 +1334,7 @@ fn summary_agg_build( let PlannerReduction::Reduce(keys) = reduction else { return Err(invalid("keyed summary requires explicit partitions")); }; - if matches!(family, FieldDataType::Sketch(kind, _) if kind.algorithm() == &planner_types::ir::schema::SketchAlgorithm::CmsWithHeap) + if matches!(family, FieldDataType::Sketch(kind, _) if kind == &planner_types::ir::schema::SketchAlgorithm::CmsWithHeap) && !matches!( update.weight_domain, planner_types::ir::schema::WeightDomain::NonNegative { .. } @@ -1406,7 +1404,7 @@ fn row_weight(input: &SchemaRef, weight: &SummaryInputExpr) -> Result bool { matches!(family, FieldDataType::Sketch(kind, _) if matches!( - kind.algorithm(), + kind, planner_types::ir::schema::SketchAlgorithm::UnivMon | planner_types::ir::schema::SketchAlgorithm::Hll )) @@ -1579,7 +1577,6 @@ fn local_scalar(expr: &ScalarExpr) -> Result PhysicalASAPDAG { PhysicalASAPDAG { @@ -1598,130 +1595,17 @@ mod compilation_tests { Err(Error::Invalid(_)) )); } - - /// `window_count` one-minute windows over one table `Scan`, the `i`-th - /// shifted `i` minutes back, each window a root (the shape of a - /// tumbling-window-expanded DAG), with the roots' node ids. - fn shifted_windows_over_one_scan(window_count: i64) -> (PhysicalASAPDAG, Vec) { - use planner_types::ir::operator::{Source, TimeShift}; - use planner_types::ir::physical_export::compile_physical_asap_workload; - use planner_types::ir::schema::{Field, Schema}; - use planner_types::ir::{ - apply_materialization_timings, MaterializationAssignment, NodeToTimedNodeCache, - Operator, QueryRoot, - }; - let minute_ms = 60_000; - let scan = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Scan { - source: Source::Table { - table_ref: "t".into(), - }, - predicates: vec![], - schema: Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("value", DataType::Float64, false), - ], - 0, - vec![], - ), - })) - .unwrap(); - let windows: Vec<_> = (0..window_count) - .map(|i| { - let shifted = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeShift { - shift: TimeShift { - offset_ms: i * minute_ms, - at: None, - }, - child: std::rc::Rc::clone(&scan), - })) - .unwrap(); - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeRange { - range: Duration::from_millis(minute_ms as u64), - kind: TimeRangeKind::Range, - child: shifted, - })) - .unwrap() - }) - .collect(); - // One timing cache, so every window reads the one `Scan`. The cache is - // keyed by node address, so `windows` stays alive while it runs. - let mut cache = NodeToTimedNodeCache::new(); - let roots: Vec<_> = windows - .iter() - .map(|window| { - let timed = apply_materialization_timings( - window, - &MaterializationAssignment::default(), - &mut cache, - ) - .unwrap(); - QueryRoot::Operator(timed) - }) - .collect(); - let dag = compile_physical_asap_workload(&roots).unwrap(); - let roots = dag.roots.iter().map(|&id| id as NodeId).collect(); - (dag, roots) - } - - /// Three one-minute windows shifted 0, 1 and 2 minutes back over one - /// `Scan`: the `Scan` reads the union `(T - 3m, T]`, and each window's - /// `TimeRange` ends its shift before `T` and is one minute wide. - #[test] - fn shifted_windows_over_one_scan_compile_to_their_windows() { - let (dag, roots) = shifted_windows_over_one_scan(3); - let compiled = serde_json::to_value(compile(&dag, &[], &roots).unwrap()).unwrap(); - let nodes = compiled["nodes"].as_object().unwrap(); - let scan_windows: Vec<_> = nodes - .values() - .filter_map(|node| node.get("RawData")) - .map(|slot| slot["window"].clone()) - .collect(); - assert_eq!( - scan_windows, - [serde_json::json!({"Within": {"end_offset_ms": 0, "width_ms": 180_000}})] - ); - let mut time_ranges: Vec<(i64, i64)> = nodes - .values() - .filter_map(|node| node.pointer("/Operator/operator/kind/TimeRange")) - .map(|kind| { - ( - kind["end_offset_ms"].as_i64().unwrap(), - kind["width_ms"].as_i64().unwrap(), - ) - }) - .collect(); - time_ranges.sort(); - assert_eq!( - time_ranges, - [(0, 60_000), (60_000, 60_000), (120_000, 60_000)] - ); - } - - /// 20,000 shifted windows over one `Scan` compile in linear time. A - /// search over every node per window, and over every root per reader of - /// the `Scan`, made the compile quadratic in the number of windows - /// (#993): in a debug build, 10,000 windows compiled in 5.4 s and 20,000 - /// in over 10 s before, 1.4 s and 3.1 s after. - #[test] - #[ignore = "slow: run by hand"] - fn many_windows_over_one_scan_compile_in_linear_time() { - let (dag, roots) = shifted_windows_over_one_scan(20_000); - let started = Instant::now(); - compile(&dag, &[], &roots).unwrap(); - eprintln!("compiled 20,000 windows in {:?}", started.elapsed()); - } } /// Helpers for the Planner's own tests (`test-support`). #[cfg(feature = "test-support")] pub mod test_support { use super::*; - use planner_types::physical::{Materialization, Medium}; + use planner_types::physical::{Lifetime, StorageMedium}; use std::collections::HashMap; /// The materialized views at `nodes`, in that order: each node that is - /// not a view is marked `MatAtIngestion`, and it and every node below it + /// not a view is marked materialized at ingestion time, and it and every node below it /// run at ingestion time (their outputs and outgoing edges); then every /// view's `retention_ms` is set ([`PhysicalASAPDAG::set_retention`]) /// with no tumbling window or range-evaluation span. For tests that bind states at @@ -1743,7 +1627,10 @@ pub mod test_support { .iter() .filter(|node| { nodes.contains(&(node.id as NodeId)) - && !planner_types::test_support::is_materialized_view(node.materialization) + && !matches!( + node.lifetime, + planner_types::physical::Lifetime::AcrossEvaluations(_) + ) }) .map(|node| node.id as NodeId) .collect(); @@ -1757,7 +1644,7 @@ pub mod test_support { for node in &mut dag.nodes { let id = node.id as NodeId; if marked.contains(&id) { - node.materialization = Materialization::MatAtIngestion(Medium::Memory); + node.lifetime = Lifetime::AcrossEvaluations(StorageMedium::Memory); } if at_ingestion.contains(&id) { node.output_state.timing = ExecutionTiming::IngestionTime; diff --git a/crates/executor/src/summary_kernels/factory.rs b/crates/executor/src/summary_kernels/factory.rs index e3345397f..890a16fb8 100644 --- a/crates/executor/src/summary_kernels/factory.rs +++ b/crates/executor/src/summary_kernels/factory.rs @@ -7,7 +7,7 @@ use crate::summary_kernels::{ }; use crate::AggregateCore; use planner_types::ir::schema::{ - FieldDataType as SummaryFamilyType, SketchAlgorithm, SketchParams, + FieldDataType as SummaryFamilyType, SketchAlgorithm, SketchKind, SketchParams, }; /// Generate the clone-based `AccumulatorUpdater` methods for updaters whose @@ -462,8 +462,12 @@ fn cms_dims(params: &SketchParams) -> (usize, usize) { /// Construct the kernel declared by a Planner SummaryAgg. No deployment config /// tags participate in this dispatch and unsupported payloads are errors. +/// +/// `parameters` is θ, which the exported node carries +/// (`PhysicalASAPDAGNode::parameters`); `None` for an exact accumulator. pub fn create_planner_accumulator( family: &SummaryFamilyType, + parameters: Option<&SketchParams>, input: &planner_types::ir::schema::SummaryUpdate, grouping: &planner_types::ir::schema::GroupingStrategy, ) -> Result, String> { @@ -500,7 +504,10 @@ pub fn create_planner_accumulator( if family_grouping != grouping { return Err("Planner family and operator grouping disagree".into()); } - let updater: Box = match (kind.algorithm(), kind.params()) { + let Some(params) = parameters else { + return Err("a sketch summary needs its parameters θ".into()); + }; + let updater: Box = match (kind, params) { (SketchAlgorithm::Kll, SketchParams::Kll { k, .. }) => { if *k > planner_types::ir::schema::sketch::KLL_MAX_K { return Err("KLL k exceeds runtime bound".into()); @@ -525,7 +532,8 @@ pub fn create_planner_accumulator( } (SketchAlgorithm::CmsWithHeap | SketchAlgorithm::CountSketchWithHeap, _) => { let (algorithm, width, depth, capacity) = - WeightedFrequency::configuration(kind).map_err(|e| e.to_string())?; + WeightedFrequency::configuration(&SketchKind::new(*kind, params.clone())) + .map_err(|e| e.to_string())?; Box::new( WeightedFrequencyUpdater::new(algorithm, width, depth, capacity) .map_err(|e| e.to_string())?, @@ -557,7 +565,7 @@ pub fn create_planner_accumulator( }), _ => { return Err(format!( - "unsupported Planner algorithm/parameters: {kind:?}" + "unsupported Planner algorithm/parameters: {kind:?} {params:?}" )) } }; @@ -672,132 +680,6 @@ impl AccumulatorUpdater for UnivMonUpdater { } } -#[cfg(test)] -mod planner_parameter_regression { - use super::*; - use planner_types::ir::schema::{SketchKind, SummaryInputExpr, SummaryUpdate}; - - // Planner width is the bucket count; depth is the independent hash-row count. - #[test] - fn planner_sketch_dimensions_are_not_transposed() { - for (algorithm, params) in [ - ( - SketchAlgorithm::Cms, - SketchParams::Cms { - width: 128, - depth: 3, - }, - ), - ( - SketchAlgorithm::CountSketch, - SketchParams::CountSketch { - width: 128, - depth: 3, - }, - ), - ( - SketchAlgorithm::CmsWithHeap, - SketchParams::CmsWithHeap { - width: 128, - depth: 3, - heap_size: 8, - }, - ), - ( - SketchAlgorithm::CountSketchWithHeap, - SketchParams::CountSketchWithHeap { - width: 128, - depth: 3, - heap_size: 8, - }, - ), - ] { - let family = - SummaryFamilyType::Sketch(SketchKind::new(algorithm, params), Default::default()); - let update = SummaryUpdate { - item: Some(SummaryInputExpr::Column( - planner_types::ir::scalar::ColumnRef::Named("host".into()), - )), - weight: SummaryInputExpr::Constant(1.0), - weight_domain: Default::default(), - }; - let state = create_planner_accumulator(&family, &update, &Default::default()) - .unwrap() - .snapshot_accumulator(); - let dims = match algorithm { - SketchAlgorithm::Cms => { - let s = state - .as_any() - .downcast_ref::() - .unwrap(); - (s.inner.rows(), s.inner.cols()) - } - SketchAlgorithm::CountSketch - | SketchAlgorithm::CmsWithHeap - | SketchAlgorithm::CountSketchWithHeap => { - let (width, depth, _) = state - .as_any() - .downcast_ref::() - .unwrap() - .shape(); - (depth, width) - } - _ => unreachable!(), - }; - assert_eq!(dims, (3, 128), "{algorithm:?}"); - } - } - - // HydraCms builds one shared grid shaped by its Hydra params; it is never - // a per-group updater, and its family must be the matching Count-Min. - #[test] - fn hydra_cms_builds_one_shared_grid() { - use planner_types::ir::schema::{ - GroupingStrategy, HydraKind, HydraParams, NonNegativeWeightProof, WeightDomain, - }; - let grouping = |width| GroupingStrategy::SharedMultiSubpopulation { - kind: HydraKind::HydraCms, - params: HydraParams::HydraCms { - width, - depth: 3, - shared_rows: 5, - shared_columns: 32, - }, - }; - let family = |width| { - SummaryFamilyType::Sketch( - SketchKind::new( - SketchAlgorithm::Cms, - SketchParams::Cms { - width: 128, - depth: 3, - }, - ), - grouping(width), - ) - }; - assert_eq!( - create_hydra_cms(&family(128)).unwrap().shape(), - (5, 32, 128, 3) - ); - assert!(create_hydra_cms(&family(64)).is_err()); - let update = SummaryUpdate { - item: Some(SummaryInputExpr::Column( - planner_types::ir::scalar::ColumnRef::Named("host".into()), - )), - weight: SummaryInputExpr::Constant(1.0), - weight_domain: WeightDomain::NonNegative { - proof: NonNegativeWeightProof::UnitCount, - }, - }; - assert!( - crate::capability::validate_summary_kernel(&family(128), &update, &grouping(128)) - .is_ok() - ); - assert!(create_planner_accumulator(&family(128), &update, &grouping(128)).is_err()); - } -} - #[cfg(test)] mod ddsketch_domain { use super::*; diff --git a/crates/executor/src/summary_kernels/state_column_updater.rs b/crates/executor/src/summary_kernels/state_column_updater.rs index e059ab25c..358abed27 100644 --- a/crates/executor/src/summary_kernels/state_column_updater.rs +++ b/crates/executor/src/summary_kernels/state_column_updater.rs @@ -2,13 +2,13 @@ //! group: the code the `SummaryAgg` build runs row by row, exposed so a //! runtime that ingests sample by sample builds the same states. use super::exact::ScalarState; -use super::factory::{create_hydra_cms, create_planner_accumulator, AccumulatorUpdater}; +use super::factory::{create_hydra_cms, AccumulatorUpdater}; use super::hydra_cms::{hydra_group_name, hydra_item, hydra_weight, HydraCms, HydraCmsGroup}; use super::weighted_frequency::WeightedFrequency; use crate::values::{StateColumn, Value}; use crate::{AggregateCore, Error}; -use planner_types::ir::scalar::ColumnRef; -use planner_types::ir::schema::{ExactKind, FieldDataType, GroupingStrategy, SummaryUpdate}; + +use planner_types::ir::schema::{ExactKind, FieldDataType, GroupingStrategy}; use std::sync::Arc; /// The open summary states of one family, one per group, updated one sample @@ -47,6 +47,7 @@ enum GroupStates { Sketches(Vec>), /// A keyed frequency sketch (Count-Min or Count Sketch with a heap) per /// group. + #[allow(dead_code)] // Used again in #1207 PR 6, with θ from the exported node. Keyed { /// An empty state of the family's shape, cloned for each new group /// (boxed: much larger than the other kinds' fields). @@ -89,15 +90,15 @@ impl StateColumnUpdater { empty: ScalarState::empty(family).map_err(Error::Invalid)?, states: Vec::new(), }, - FieldDataType::Sketch(kind, grouping) + FieldDataType::Sketch(algorithm, grouping) if grouping == &GroupingStrategy::PerSubpopulationInstance - && WeightedFrequency::configuration(kind).is_ok() => + && matches!( + algorithm, + planner_types::ir::schema::SketchAlgorithm::CmsWithHeap + | planner_types::ir::schema::SketchAlgorithm::CountSketchWithHeap + ) => { - let (algorithm, width, depth, capacity) = WeightedFrequency::configuration(kind)?; - GroupStates::Keyed { - empty: Box::new(WeightedFrequency::new(algorithm, width, depth, capacity)?), - states: Vec::new(), - } + todo!("#1207 PR 6: the keyed frequency state of θ, which the exported node carries") } FieldDataType::Sketch(..) => { // Fails for a family with no per-group value kernel. @@ -125,12 +126,8 @@ impl StateColumnUpdater { } fn sketch_updater(family: &FieldDataType) -> Result, Error> { - create_planner_accumulator( - family, - &SummaryUpdate::column(ColumnRef::SampleValue), - &Default::default(), - ) - .map_err(Error::Invalid) + let _ = family; + todo!("#1207 PR 6: build it with `create_planner_accumulator` and θ from the exported node's `parameters`") } /// The number of groups. diff --git a/crates/executor/src/supported_summaries.rs b/crates/executor/src/supported_summaries.rs index 6ad48c3e1..973ce0cfb 100644 --- a/crates/executor/src/supported_summaries.rs +++ b/crates/executor/src/supported_summaries.rs @@ -24,13 +24,12 @@ mod tests { use super::*; use crate::capability::validate_sketch_evaluation; use crate::factory::create_planner_accumulator; - use crate::summary_kernels::weighted_frequency::WeightedFrequency; + use planner_types::ir::operator::operator_properties::{Reduction, Source}; use planner_types::ir::scalar::ColumnRef; use planner_types::ir::schema::{ DataType, FamilyCapability, Field, FieldDataType, GroupingStrategy, Schema, - SketchAlgorithm, SketchKind, SketchParams, SketchStatistic, StatisticKind, - SummaryInputExpr, SummaryUpdate, + SketchAlgorithm, SketchStatistic, StatisticKind, SummaryInputExpr, SummaryUpdate, }; use planner_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode}; @@ -56,57 +55,8 @@ mod tests { StatisticKind::TopK, ]; - /// Valid parameters of `algorithm`, as the Planner sizes them. - fn params(algorithm: &SketchAlgorithm) -> SketchParams { - use SketchAlgorithm as A; - match algorithm { - A::UnivMon => SketchParams::UnivMon { - heap_size: 32, - sketch_rows: 3, - sketch_cols: 1024, - layers: 8, - distinct_failure_probability: None, - }, - A::Kll => SketchParams::Kll { - k: 200, - failure_probability: None, - }, - A::Cms => SketchParams::Cms { - width: 64, - depth: 3, - }, - A::Hll => SketchParams::Hll { - precision: 12, - relative_error: None, - }, - A::DDSketch => SketchParams::DDSketch { alpha: 0.01 }, - A::CmsWithHeap => SketchParams::CmsWithHeap { - width: 64, - depth: 3, - heap_size: 10, - }, - A::Kmv => SketchParams::Kmv { - k: 1024, - relative_error: None, - }, - A::CountSketch => SketchParams::CountSketch { - width: 64, - depth: 3, - }, - A::CountSketchWithHeap => SketchParams::CountSketchWithHeap { - width: 64, - depth: 3, - heap_size: 10, - }, - other => unreachable!("{other:?} is not probed"), - } - } - fn per_instance(algorithm: &SketchAlgorithm) -> FieldDataType { - FieldDataType::Sketch( - SketchKind::new(*algorithm, params(algorithm)), - GroupingStrategy::PerSubpopulationInstance, - ) + FieldDataType::Sketch(*algorithm, GroupingStrategy::PerSubpopulationInstance) } /// A statistic of kind `readout`. @@ -142,8 +92,8 @@ mod tests { item: Some(SummaryInputExpr::Column(ColumnRef::Named("item".into()))), ..value.clone() }; - create_planner_accumulator(family, &value, grouping).is_ok() - || create_planner_accumulator(family, &keyed, grouping).is_ok() + create_planner_accumulator(family, None, &value, grouping).is_ok() + || create_planner_accumulator(family, None, &keyed, grouping).is_ok() } /// Whether the executor answers `readout` from `family`'s state: top-k @@ -151,9 +101,11 @@ mod tests { /// sketch evaluation, whose kernel must then read it from a built state. fn evaluation_answers(family: &FieldDataType, readout: StatisticKind) -> bool { match (readout, family) { - (StatisticKind::TopK, FieldDataType::Sketch(kind, _)) => { - WeightedFrequency::configuration(kind).is_ok() - } + (StatisticKind::TopK, FieldDataType::Sketch(kind, _)) => matches!( + kind, + planner_types::ir::schema::SketchAlgorithm::CmsWithHeap + | planner_types::ir::schema::SketchAlgorithm::CountSketchWithHeap + ), _ => { let accepted = validate_sketch_evaluation(family, &statistic(readout)).is_ok(); if accepted { @@ -175,8 +127,8 @@ mod tests { item: Some(SummaryInputExpr::Column(ColumnRef::Named("item".into()))), ..value.clone() }; - let Ok(mut updater) = create_planner_accumulator(family, &value, grouping) - .or_else(|_| create_planner_accumulator(family, &keyed, grouping)) + let Ok(mut updater) = create_planner_accumulator(family, None, &value, grouping) + .or_else(|_| create_planner_accumulator(family, None, &keyed, grouping)) else { return false; }; diff --git a/crates/executor/src/values/batch.rs b/crates/executor/src/values/batch.rs index 2bc7e0284..12f06cf90 100644 --- a/crates/executor/src/values/batch.rs +++ b/crates/executor/src/values/batch.rs @@ -609,9 +609,7 @@ impl Batch { mod tests { use super::*; use crate::summary_kernels::DDSketchAccumulator; - use planner_types::ir::schema::{ - DataType, Field, Schema, SketchAlgorithm, SketchKind, SketchParams, - }; + use planner_types::ir::schema::{DataType, Field, Schema, SketchAlgorithm}; fn field(name: &str, dtype: DataType, nullable: bool) -> Field { Field { @@ -774,13 +772,7 @@ mod tests { } fn dd_family() -> FieldDataType { - FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::DDSketch, - SketchParams::DDSketch { alpha: 0.01 }, - ), - Default::default(), - ) + FieldDataType::Sketch(SketchAlgorithm::DDSketch, Default::default()) } // Rows sharing a state keep sharing one state; filtering rows shares the @@ -1134,22 +1126,11 @@ mod tests { /// exact sums. fn state_batch(count: u32, exact: bool) -> Batch { use crate::StateColumnUpdater; - use planner_types::ir::schema::{ - ExactKind, ExactParams, SketchAlgorithm, SketchKind, SketchParams, - }; + use planner_types::ir::schema::{ExactKind, ExactParams, SketchAlgorithm}; let family = if exact { FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum) } else { - FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: 200, - failure_probability: None, - }, - ), - Default::default(), - ) + FieldDataType::Sketch(SketchAlgorithm::Kll, Default::default()) }; let mut updater = StateColumnUpdater::new(&family).unwrap(); updater.ensure_groups(count as usize).unwrap(); diff --git a/crates/executor/src/values/mod.rs b/crates/executor/src/values/mod.rs index a28ec8f44..40c273375 100644 --- a/crates/executor/src/values/mod.rs +++ b/crates/executor/src/values/mod.rs @@ -210,29 +210,10 @@ pub(super) fn validate_state( family: &SummaryFamilyType, state: &dyn AggregateCore, ) -> Result<(), Error> { - use crate::summary_kernels::{ - count_min_sketch::CountMinSketchAccumulator, datasketches_kll::DatasketchesKLLAccumulator, - dd_sketch::DDSketchAccumulator, exact::ExactAccumulator, hll_sketch::HllSketchAccumulator, - kmv::KmvAccumulator, univmon::UnivMonAccumulator, - }; - use planner_types::ir::schema::SketchParams; + use crate::summary_kernels::exact::ExactAccumulator; + validate_family(family)?; let valid = match family { - SummaryFamilyType::Sketch(kind, _) - if matches!( - kind.params(), - SketchParams::CmsWithHeap { .. } | SketchParams::CountSketchWithHeap { .. } - ) => - { - use crate::summary_kernels::weighted_frequency::WeightedFrequency; - let (algorithm, width, depth, capacity) = WeightedFrequency::configuration(kind)?; - state - .as_any() - .downcast_ref::() - .is_some_and(|state| { - state.algorithm() == algorithm && state.shape() == (width, depth, capacity) - }) - } SummaryFamilyType::Sketch(_, grouping) if grouping != &planner_types::ir::schema::GroupingStrategy::PerSubpopulationInstance => @@ -247,49 +228,9 @@ pub(super) fn validate_state( .as_any() .downcast_ref::() .is_some_and(|s| s.family() == family && !s.is_keyed()), - SummaryFamilyType::Sketch(kind, _) => match kind.params() { - SketchParams::Kll { k, .. } => state - .as_any() - .downcast_ref::() - .is_some_and(|s| s.inner.k() == *k as usize), - SketchParams::DDSketch { alpha } => state - .as_any() - .downcast_ref::() - .is_some_and(|s| s.inner.alpha() == *alpha), - SketchParams::Hll { precision, .. } => state - .as_any() - .downcast_ref::() - .is_some_and(|s| s.precision() == *precision), - SketchParams::Kmv { k, .. } => state - .as_any() - .downcast_ref::() - .is_some_and(|s| s.inner.k == *k as usize), - SketchParams::UnivMon { - heap_size, - sketch_rows, - sketch_cols, - layers, - .. - } => state - .as_any() - .downcast_ref::() - .is_some_and(|s| { - s.dimensions() - == ( - *heap_size as usize, - *sketch_rows as usize, - *sketch_cols as usize, - usize::from(*layers), - ) - }), - SketchParams::Cms { width, depth } => state - .as_any() - .downcast_ref::() - .is_some_and(|s| { - (s.inner.rows(), s.inner.cols()) == (*depth as usize, *width as usize) - }), - _ => false, - }, + SummaryFamilyType::Sketch(..) => { + todo!("#1207 PR 6: compare the state with θ, which the exported node carries") + } _ => false, }; if valid { @@ -343,50 +284,22 @@ pub(crate) fn plain(schema: &SchemaRef, column: usize) -> Result<(&DataType, boo mod weighted_state_tests { use super::*; use crate::summary_kernels::weighted_frequency::{FrequencyAlgorithm, WeightedFrequency}; - use planner_types::ir::schema::{SketchAlgorithm, SketchKind, SketchParams}; + use planner_types::ir::schema::SketchAlgorithm; // A state cannot acquire a different family or shape merely by relabeling its batch. #[test] fn weighted_state_family_and_shape_must_match() { - let cms = SummaryFamilyType::Sketch( - SketchKind::new( - SketchAlgorithm::CmsWithHeap, - SketchParams::CmsWithHeap { - width: 32, - depth: 5, - heap_size: 8, - }, - ), - Default::default(), - ); - let cs = SummaryFamilyType::Sketch( - SketchKind::new( - SketchAlgorithm::CountSketchWithHeap, - SketchParams::CountSketchWithHeap { - width: 32, - depth: 5, - heap_size: 8, - }, - ), - Default::default(), - ); + let cms = SummaryFamilyType::Sketch(SketchAlgorithm::CmsWithHeap, Default::default()); + let cs = + SummaryFamilyType::Sketch(SketchAlgorithm::CountSketchWithHeap, Default::default()); let state = WeightedFrequency::new(FrequencyAlgorithm::CountSketch, 32, 5, 8).unwrap(); assert!(validate_state(&cs, &state).is_ok()); assert!(validate_state(&cms, &state).is_err()); let wrong_shape = WeightedFrequency::new(FrequencyAlgorithm::CountSketch, 64, 5, 8).unwrap(); assert!(validate_state(&cs, &wrong_shape).is_err()); - let even_depth = SummaryFamilyType::Sketch( - SketchKind::new( - SketchAlgorithm::CountSketchWithHeap, - SketchParams::CountSketchWithHeap { - width: 32, - depth: 4, - heap_size: 8, - }, - ), - Default::default(), - ); + let even_depth = + SummaryFamilyType::Sketch(SketchAlgorithm::CountSketchWithHeap, Default::default()); assert!(validate_family(&even_depth).is_err()); } } diff --git a/crates/executor/tests/blocking_resources.rs b/crates/executor/tests/blocking_resources.rs index ee3b4435b..e397b15d8 100644 --- a/crates/executor/tests/blocking_resources.rs +++ b/crates/executor/tests/blocking_resources.rs @@ -186,7 +186,7 @@ fn cooperative_sort_preserves_ties_across_chunks() { // The integrated weighted-summary path obeys the same cooperative cancellation contract. #[test] fn weighted_summary_build_yields_within_a_batch() { - use planner_types::ir::schema::{SketchAlgorithm, SketchKind, SketchParams}; + use planner_types::ir::schema::SketchAlgorithm; let input = Arc::new(planner_types::ir::schema::Schema { unique_keys: vec![], @@ -222,17 +222,7 @@ fn weighted_summary_build_yields_within_a_batch() { Operator::source(input.clone(), vec![batch]).unwrap(), ) .unwrap(); - let family = FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::CmsWithHeap, - SketchParams::CmsWithHeap { - width: 64, - depth: 3, - heap_size: 8, - }, - ), - Default::default(), - ); + let family = FieldDataType::Sketch(SketchAlgorithm::CmsWithHeap, Default::default()); let operator = Operator::per_group_keyed_summary_build( input, family, diff --git a/crates/executor/tests/common/mod.rs b/crates/executor/tests/common/mod.rs index 857d76bf9..31187af65 100644 --- a/crates/executor/tests/common/mod.rs +++ b/crates/executor/tests/common/mod.rs @@ -1,21 +1,7 @@ #![allow(dead_code)] -use planner_types::ir::physical_export::PhysicalASAPDAG; -use planner_types::ir::{ - apply_materialization_timings, MaterializationAssignment, NodeToTimedNodeCache, OperatorNode, - QueryRoot, -}; -use std::rc::Rc; -pub fn compile_physical_asap_dag( - root: &Rc, -) -> Result> { - let root = apply_materialization_timings( - root, - &MaterializationAssignment::default(), - &mut NodeToTimedNodeCache::default(), - )?; - Ok(planner_types::ir::physical_export::compile_physical_asap_dag(&root)?) -} +use planner_types::ir::{OperatorNode, QueryRoot}; +use std::rc::Rc; /// The logical DAG the planner pipeline (`asap_planner::run_planning`) selects for the /// single query `root` at `accuracy`, over a continuously ingested source, @@ -43,9 +29,7 @@ pub fn computation_candidates(root: Rc) -> Vec> { compose_logical_candidate, generate_per_computation_candidates, }; let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(root), - ]), + &asap_logical_asap_optimization::memo_of_queries(&[QueryRoot::Operator(root)]).unwrap(), ) .expect("Pass 1 accepts the query"); let mut choices = vec![vec![0; inventory.targets.len()]]; diff --git a/crates/executor/tests/current_series_heap.rs b/crates/executor/tests/current_series_heap.rs index a78efe426..1368b006f 100644 --- a/crates/executor/tests/current_series_heap.rs +++ b/crates/executor/tests/current_series_heap.rs @@ -1,5 +1,5 @@ //! Spatial heap weights come from a fresh instant vector, never sample history. -use crate::common; + use asap_executor::sources::InMemoryDataStore; use asap_executor::{ operators::Operator, @@ -10,7 +10,6 @@ use asap_executor::{ runtime::{Limits, RunContext, Scope}, values::{Batch, Value}, }; -use common::compile_physical_asap_dag; use futures::{executor::block_on, StreamExt}; use planner_types::ir::physical_export::PhysicalASAPOperatorPayload; use planner_types::ir::properties::AccuracyTarget; @@ -161,7 +160,7 @@ fn spatial_heap_ranks_latest_values_in_independent_runs() { SketchAlgorithm::CmsWithHeap, SketchAlgorithm::CountSketchWithHeap, ] { - let params = match algorithm { + let _params = match algorithm { SketchAlgorithm::CmsWithHeap => SketchParams::CmsWithHeap { width: 2048, depth: 5, @@ -173,7 +172,7 @@ fn spatial_heap_ranks_latest_values_in_independent_runs() { heap_size: 100, }, }; - let family = FieldDataType::Sketch(SketchKind::new(algorithm, params), Default::default()); + let family = FieldDataType::Sketch(algorithm, Default::default()); let build = Operator::per_group_keyed_summary_build( schema(), family, @@ -341,7 +340,7 @@ fn population_fixture_compiles_with_dynamic_identity() { topk, &["job"], ); - let logical = compile_physical_asap_dag(&selected).unwrap(); + let logical = planner_types::test_support::physical_asap_dag_at_query_time(&selected).unwrap(); let (source, raw_schema) = logical .nodes .iter() diff --git a/crates/executor/tests/current_timestamp.rs b/crates/executor/tests/current_timestamp.rs index a774d2289..bbd1c7a40 100644 --- a/crates/executor/tests/current_timestamp.rs +++ b/crates/executor/tests/current_timestamp.rs @@ -65,7 +65,8 @@ fn scan_plan(predicates: Vec>) -> PhysicalASAPDAG { }, ), output_state: ExecutionDataState::QUERY_ROWS, - materialization: Default::default(), + lifetime: Default::default(), + parameters: None, output_schema: schema(), guarantee: None, retention_ms: None, diff --git a/crates/executor/tests/deployment.rs b/crates/executor/tests/deployment.rs index 70804d9f4..e79652f97 100644 --- a/crates/executor/tests/deployment.rs +++ b/crates/executor/tests/deployment.rs @@ -1,27 +1,20 @@ //! Exercise the public library without a backend server, store, or scheduler. use asap_executor::planner::{ ir::scalar::ColumnRef, - ir::schema::{ - FieldDataType, GroupingStrategy, SketchAlgorithm, SketchKind, SketchParams, SummaryUpdate, - }, + ir::schema::{FieldDataType, GroupingStrategy, SketchAlgorithm, SketchParams, SummaryUpdate}, }; use asap_executor::{factory::create_planner_accumulator, AggregateCore}; -fn family(k: u32) -> FieldDataType { +fn family(_k: u32) -> FieldDataType { FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k, - failure_probability: None, - }, - ), + SketchAlgorithm::Kll, GroupingStrategy::PerSubpopulationInstance, ) } fn build(values: &[f64]) -> Box { let mut operator = create_planner_accumulator( &family(512), + None, &SummaryUpdate::column(ColumnRef::SampleValue), &Default::default(), ) @@ -58,6 +51,7 @@ fn raw_partial_and_fully_precomputed_use_the_same_kernels() { fn invalid_kll_parameters_are_rejected_at_binding() { let result = create_planner_accumulator( &family(0), + None, &SummaryUpdate::column(ColumnRef::SampleValue), &Default::default(), ); @@ -69,20 +63,11 @@ fn invalid_kll_parameters_are_rejected_at_binding() { #[test] fn native_count_sketch_dimensions_are_not_packed_wire_dimensions() { use asap_executor::planner::ir::schema::SummaryInputExpr; - let family = FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::CountSketchWithHeap, - SketchParams::CountSketchWithHeap { - width: 1200, - depth: 55, - heap_size: 3, - }, - ), - Default::default(), - ); + let family = FieldDataType::Sketch(SketchAlgorithm::CountSketchWithHeap, Default::default()); let mut update = SummaryUpdate::column(ColumnRef::SampleValue); update.item = Some(SummaryInputExpr::Column(ColumnRef::Named("host".into()))); - let mut operator = create_planner_accumulator(&family, &update, &Default::default()).unwrap(); + let mut operator = + create_planner_accumulator(&family, None, &update, &Default::default()).unwrap(); operator .update_keyed(&["a".to_string()], 7.0, 1000) .unwrap(); @@ -99,14 +84,8 @@ fn native_count_sketch_dimensions_are_not_packed_wire_dimensions() { ); } -fn plain_count_sketch(width: u32, depth: u32) -> FieldDataType { - FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::CountSketch, - SketchParams::CountSketch { width, depth }, - ), - Default::default(), - ) +fn plain_count_sketch(_width: u32, _depth: u32) -> FieldDataType { + FieldDataType::Sketch(SketchAlgorithm::CountSketch, Default::default()) } fn host(i: usize) -> [String; 1] { @@ -129,7 +108,8 @@ fn build_weighted_frequency( use asap_executor::planner::ir::schema::SummaryInputExpr; let mut update = SummaryUpdate::column(ColumnRef::SampleValue); update.item = Some(SummaryInputExpr::Column(ColumnRef::Named("host".into()))); - let mut operator = create_planner_accumulator(family, &update, &Default::default()).unwrap(); + let mut operator = + create_planner_accumulator(family, None, &update, &Default::default()).unwrap(); for (at, (i, weight)) in updates.into_iter().enumerate() { operator.validate_single_input(weight).unwrap(); operator.update_keyed(&host(i), weight, at as i64).unwrap(); @@ -221,7 +201,7 @@ fn heap_sketch( depth: u32, heap_size: u32, ) -> FieldDataType { - let params = match algorithm { + let _params = match algorithm { SketchAlgorithm::CmsWithHeap => SketchParams::CmsWithHeap { width, depth, @@ -233,7 +213,7 @@ fn heap_sketch( heap_size, }, }; - FieldDataType::Sketch(SketchKind::new(algorithm, params), Default::default()) + FieldDataType::Sketch(algorithm, Default::default()) } // The factory's top-k sketches use the same per-row-seeded kernel: past 20 @@ -339,7 +319,7 @@ fn count_min_heap_rejects_negative_inputs() { let mut update = SummaryUpdate::column(ColumnRef::SampleValue); update.item = Some(SummaryInputExpr::Column(ColumnRef::Named("host".into()))); let family = heap_sketch(SketchAlgorithm::CmsWithHeap, 64, 3, 4); - let operator = create_planner_accumulator(&family, &update, &Default::default()).unwrap(); + let operator = create_planner_accumulator(&family, None, &update, &Default::default()).unwrap(); assert!(operator.validate_single_input(-1.0).is_err()); assert!(operator.validate_single_input(0.5).is_ok()); } @@ -405,7 +385,8 @@ fn keyed_update_overflow_is_an_error_not_a_panic() { let mut update = SummaryUpdate::column(ColumnRef::SampleValue); update.item = Some(SummaryInputExpr::Column(ColumnRef::Named("host".into()))); let family = plain_count_sketch(64, 3); - let mut operator = create_planner_accumulator(&family, &update, &Default::default()).unwrap(); + let mut operator = + create_planner_accumulator(&family, None, &update, &Default::default()).unwrap(); operator.validate_single_input(f64::MAX).unwrap(); operator.update_keyed(&host(0), f64::MAX, 0).unwrap(); let error = operator.update_keyed(&host(0), f64::MAX, 1).unwrap_err(); diff --git a/crates/executor/tests/deployment_computation.rs b/crates/executor/tests/deployment_computation.rs index 7fad67cef..080454c79 100644 --- a/crates/executor/tests/deployment_computation.rs +++ b/crates/executor/tests/deployment_computation.rs @@ -1,20 +1,20 @@ //! Planner-selected PromQL computation compiles from the timed DAG alone; //! the deployment supplies only raw rows at its time-series scans. -use crate::common; + use asap_executor::sources::InMemoryDataStore; use asap_executor::{ - physical_planner::{compile, promql_rows, test_support::views_at}, + physical_planner::{compile, promql_rows}, runtime::{Limits, RunContext, Scope}, values::{Batch, Value}, }; -use common::{compile_physical_asap_dag, computation_candidates, selected_dag}; + use futures::{executor::block_on, StreamExt}; use planner_types::ir::operator::maintained_population::PopulationStatistic; use planner_types::ir::operator::Source; use planner_types::ir::physical_export::{PhysicalASAPDAG, PhysicalASAPOperatorPayload}; use planner_types::ir::properties::AccuracyTarget; use planner_types::ir::schema::*; -use planner_types::ir::ASAPOp; + use planner_types::workload::*; use std::{collections::BTreeMap, rc::Rc, sync::Arc}; @@ -52,13 +52,6 @@ fn lower_with(query: &str, accuracy: AccuracyTarget) -> Rc PhysicalASAPDAG { - let expression = lower(query); - let root = promql_rows::with_series_identity(&expression).unwrap_or(expression); - compile_physical_asap_dag(&selected_dag(root, AccuracyTarget::exact())).unwrap() -} - /// The population sub-DAG of `query`, which computes `statistic` grouped by /// `grouping` over one selector. fn population_dag( @@ -70,7 +63,7 @@ fn population_dag( let selected = asap_logical_asap_optimization::test_support::population_over_selector( &root, statistic, grouping, ); - compile_physical_asap_dag(&selected).unwrap() + planner_types::test_support::physical_asap_dag_at_query_time(&selected).unwrap() } /// The time-series scans: each metric's raw data slot and its schema. @@ -265,433 +258,6 @@ fn global_population_aggregate_of_no_members_is_empty() { } } -// Scalar operands on either side apply to every grouped value, including negation. -#[test] -fn scalar_literal_arithmetic_applies_to_grouped_values() { - // sum_over_time over 5m per job: api = 4 + 1 + 7 + 2 = 14, db = 5. - for (query, expected) in [ - ( - "sum by (job) (sum_over_time(m[5m])) * 2", - reference(&[("api", 28.), ("db", 10.)]), - ), - ( - "100 - sum by (job) (sum_over_time(m[5m]))", - reference(&[("api", 86.), ("db", 95.)]), - ), - ( - "-sum by (job) (sum_over_time(m[5m]))", - reference(&[("api", -14.), ("db", -5.)]), - ), - ] { - assert_eq!( - run(&exact_dag(query), SAMPLES, 60_000).unwrap(), - expected, - "{query}" - ); - } -} - -// Grouped vectors match one-to-one on labels; unmatched groups are dropped and -// unchecked division by zero yields +Inf as in PromQL. -#[test] -fn grouped_vector_arithmetic_matches_labels() { - let samples: &[RawDataSample] = &[ - ("a", "api", "x", 10_000, 6.), - ("a", "api", "y", 20_000, 3.), - ("a", "db", "x", 10_000, 1.), - ("a", "web", "x", 10_000, 1.), - ("b", "api", "x", 10_000, 3.), - ("b", "db", "x", 10_000, 0.), - ("b", "cache", "x", 10_000, 1.), - ]; - let dag = - exact_dag("sum by (job) (sum_over_time(a[5m])) / sum by (job) (sum_over_time(b[5m]))"); - assert_eq!( - run(&dag, samples, 60_000).unwrap(), - reference(&[("api", 3.), ("db", f64::INFINITY)]) - ); -} - -// Exact observation counts finalize to the Float64 value PromQL declares, -// then roll up per job: api has 2 + 1 + 1 samples in 5m, db has 1. -#[test] -fn exact_count_finalizes_to_declared_float_value() { - let dag = exact_dag("sum by (job) (count_over_time(m[5m]))"); - assert_eq!( - run(&dag, SAMPLES, 60_000).unwrap(), - reference(&[("api", 4.), ("db", 1.)]) - ); -} - -/// Prometheus's `quantile` test series (`promql/promqltest/testdata/aggregators.test` -/// in Prometheus v0.314.0), one job per `test` label and one instance per -/// `point`, all sampled at 50 s. -const QUANTILE_SAMPLES: &[RawDataSample] = &[ - ("data", "two", "a", 50_000, 0.), - ("data", "two", "b", 50_000, 1.), - ("data", "three", "a", 50_000, 0.), - ("data", "three", "b", 50_000, 1.), - ("data", "three", "c", 50_000, 2.), - ("data", "uneven", "a", 50_000, 0.), - ("data", "uneven", "b", 50_000, 1.), - ("data", "uneven", "c", 50_000, 4.), - ("data", "nan", "a", 50_000, 0.), - ("data", "nan", "b", 50_000, 1.), - ("data", "nan", "c", 50_000, f64::NAN), -]; - -/// Asserts `(job, value)` rows equal `expected`, NaN equal to NaN and finite -/// values to within 1e-12 (Prometheus's own tests compare approximately too). -fn assert_rows(actual: &BTreeMap, expected: &[(&str, f64)], query: &str) { - let mut jobs: Vec<&str> = expected.iter().map(|(job, _)| *job).collect(); - jobs.sort(); - assert_eq!(actual.keys().collect::>(), jobs, "{query}"); - for (job, want) in expected { - let got = actual[*job]; - let same = if want.is_finite() { - (got - want).abs() <= 1e-12 * want.abs().max(1.) - } else { - got.to_bits() == want.to_bits() || (got.is_nan() && want.is_nan()) - }; - assert!(same, "{query}: {job} = {got}, want {want}"); - } -} - -// An exact instant `quantile by (G)` compiles to the grouped aggregate and -// returns Prometheus's values: linear interpolation between closest ranks, -// a NaN sample ordered first (so a rank between NaN and 0 is NaN), and ±Inf -// for φ outside [0, 1], NaN for a NaN φ. The 0.8, 0.2 and NaN rows are Prometheus's own expected -// values for `quantile without(point)` over the same series. -#[test] -fn exact_grouped_quantile_matches_prometheus() { - let (inf, nan) = (f64::INFINITY, f64::NAN); - for (query, expected) in [ - ( - "quantile by (job) (0.8, data)", - vec![("two", 0.8), ("three", 1.6), ("uneven", 2.8), ("nan", 0.6)], - ), - ( - "quantile by (job) (0.2, data)", - vec![("two", 0.2), ("three", 0.4), ("uneven", 0.4), ("nan", nan)], - ), - ( - "quantile by (job) (0, data)", - vec![("two", 0.), ("three", 0.), ("uneven", 0.), ("nan", nan)], - ), - ( - "quantile by (job) (1, data)", - vec![("two", 1.), ("three", 2.), ("uneven", 4.), ("nan", 1.)], - ), - ( - "quantile by (job) (-0.5, data)", - vec![ - ("two", -inf), - ("three", -inf), - ("uneven", -inf), - ("nan", -inf), - ], - ), - ( - "quantile by (job) (1.5, data)", - vec![("two", inf), ("three", inf), ("uneven", inf), ("nan", inf)], - ), - // Prometheus answers a NaN φ with NaN for every group. - ( - "quantile by (job) (NaN, data)", - vec![("two", nan), ("three", nan), ("uneven", nan), ("nan", nan)], - ), - // One group over every series: sorted [NaN, 0, 0, 0, 0, 1, 1, 1, 1, 2, 4], - // rank 0.5 * 10 = 5 → 1. - ("quantile(0.5, data)", vec![("", 1.)]), - ] { - let rows = run(&exact_dag(query), QUANTILE_SAMPLES, 60_000).unwrap(); - assert_rows(&rows, &expected, query); - } -} - -// With no input series the result is an empty vector, grouped or not, as in -// Prometheus; a group of one series returns that series' value for any φ in -// [0, 1]. -#[test] -fn exact_grouped_quantile_of_no_series_is_empty() { - for query in ["quantile by (job) (0.5, data)", "quantile(0.5, data)"] { - let rows = run(&exact_dag(query), &[], 60_000).unwrap(); - assert!(rows.is_empty(), "{query}: {rows:?}"); - } - let one: &[RawDataSample] = &[("data", "solo", "a", 50_000, 3.)]; - let query = "quantile by (job) (0.9, data)"; - assert_rows( - &run(&exact_dag(query), one, 60_000).unwrap(), - &[("solo", 3.)], - query, - ); -} - -/// `dag` with its Binary operator replaced by `kind`. -fn with_kind( - mut dag: PhysicalASAPDAG, - kind: planner_types::ir::operator::BinaryOpKind, - bool_result: bool, -) -> PhysicalASAPDAG { - for node in &mut dag.nodes { - if let PhysicalASAPOperatorPayload::NonASAP(planner_types::ir::NonASAPOp::BinaryOp { - operator, - return_bool, - .. - }) = &mut node.payload - { - operator.kind = kind.clone(); - *return_bool = bool_result; - } - } - dag -} - -// A comparison Binary over grouped values keeps the groups whose comparison -// holds, with their value, on either side of the literal; `bool` yields 1 or 0. -#[test] -fn grouped_comparisons_filter_or_return_bool() { - for (query, expected) in [ - ( - "sum by(job)(sum_over_time(m[5m])) > 10", - reference(&[("api", 14.)]), - ), - ( - "sum by(job)(sum_over_time(m[5m])) > bool 10", - reference(&[("api", 1.), ("db", 0.)]), - ), - ( - "10 > sum by(job)(sum_over_time(m[5m]))", - reference(&[("db", 5.)]), - ), - ] { - assert_eq!(run(&exact_dag(query), SAMPLES, 60_000).unwrap(), expected); - } -} - -// A `bool` comparison Binary over per-series evaluations matches one-to-one and -// drops the metric name; a filter keeps the surviving left value. -#[test] -fn per_series_comparisons_filter_or_return_bool() { - use planner_types::ir::operator::BinaryOpKind::*; - use planner_types::ir::scalar::CompareOpKind::*; - - let samples = counter("a", "api", 10., 10.) - .chain(counter("a", "db", 10., 10.)) - .chain(counter("b", "api", 5., 5.)) - .chain(counter("b", "db", 20., 20.)) - .collect::>(); - // rate: a{api} = a{db} = 50/300, b{api} = 25/300, b{db} = 100/300. - let dag = exact_dag("rate(a[5m]) / rate(b[5m])"); - assert_eq!( - run_series( - &with_kind(dag.clone(), Compare(Gt), false), - &samples, - 300_000 - ) - .unwrap(), - series(&[("api", "x", 50. / 300.)]) - ); - assert_eq!( - run_series(&with_kind(dag, Compare(Lt), true), &samples, 300_000).unwrap(), - series(&[("api", "x", 0.), ("db", "x", 1.)]) - ); -} - -/// [`execute`], returning per-series `(identity, value)` rows of the root, -/// with NaN-aware formatting for comparison. -fn run_series( - dag: &PhysicalASAPDAG, - samples: &[RawDataSample], - end: i64, -) -> Result { - let mut rows = BTreeMap::new(); - for batch in execute(dag, samples, end)? { - let schema = batch.schema(); - let column = |name: &str| schema.fields.iter().position(|f| f.name == name).unwrap(); - let (identity, value) = (column(promql_rows::SERIES_IDENTITY_COLUMN), column("value")); - for row in &batch.to_rows() { - let (Value::Utf8(identity), Value::Float64(value)) = (&row[identity], &row[value]) - else { - return Err(format!("unexpected row {row:?}")); - }; - assert!(rows.insert(identity.to_string(), *value).is_none()); - } - } - Ok(format!("{rows:?}")) -} - -fn series(pairs: &[(&str, &str, f64)]) -> String { - let rows = pairs - .iter() - .map(|(job, instance, value)| { - ( - format!(r#"{{"instance":"{instance}","job":"{job}"}}"#), - *value, - ) - }) - .collect::>(); - format!("{rows:?}") -} - -/// Five counter samples per series, one per minute up to 300s, at `base + step * i`. -fn counter( - metric: &'static str, - job: &'static str, - base: f64, - step: f64, -) -> impl Iterator { - (1..=5).map(move |i| (metric, job, "x", i * 60_000, base + step * (i - 1) as f64)) -} - -// avg_over_time over stored per-series sum/count state divides per series and -// drops the metric name, as Prometheus does. An overflowing stored sum fails -// the checked division instead of returning +Inf. -#[test] -fn per_series_average_divides_stored_sum_by_count() { - let dag = exact_dag("avg_over_time(m[5m])"); - assert_eq!( - run_series(&dag, SAMPLES, 60_000).unwrap(), - series(&[ - ("api", "a", 2.5), - ("api", "b", 7.), - ("api", "c", 2.), - ("db", "d", 5.) - ]) - ); - let huge: &[RawDataSample] = &[ - ("m", "api", "a", 10_000, 1.7e308), - ("m", "api", "a", 20_000, 1.7e308), - ]; - assert!(run_series(&dag, huge, 60_000).is_err()); -} - -// rate(a) / rate(b) matches series on their labels without the metric name. -// Unmatched series are dropped; x/0 is +Inf and 0/0 is NaN; an empty side -// yields an empty vector. -#[test] -fn per_series_rate_ratio_matches_prometheus() { - // Window (0, 300s]: first sample at 60s, extrapolated 60s to the start - // (durationToZero is exactly 60s too), so rate = (last - first) * 1.25 / 300. - // a{api} = 40 * 1.25 / 300, b{api} = 20 * 1.25 / 300, so the ratio is 2. - let samples = counter("a", "api", 10., 10.) - .chain(counter("a", "db", 10., 10.)) - .chain(counter("a", "web", 10., 10.)) - .chain(counter("a", "cache", 7., 0.)) - .chain(counter("b", "api", 5., 5.)) - .chain(counter("b", "web", 7., 0.)) - .chain(counter("b", "cache", 7., 0.)) - .chain(counter("b", "other", 5., 5.)) - .collect::>(); - let dag = exact_dag("rate(a[5m]) / rate(b[5m])"); - assert_eq!( - run_series(&dag, &samples, 300_000).unwrap(), - series(&[ - ("api", "x", 2.), - ("cache", "x", f64::NAN), - ("web", "x", f64::INFINITY), - ]) - ); - let only_a = samples - .iter() - .filter(|s| s.0 == "a") - .copied() - .collect::>(); - assert_eq!(run_series(&dag, &only_a, 300_000).unwrap(), series(&[])); -} - -// A literal operand applies to every stored per-series value, on either side, -// and drops the metric name. -#[test] -fn per_series_scalar_arithmetic_applies_to_stored_evaluations() { - let samples = counter("m", "api", 10., 10.).collect::>(); - // rate = 40 * 1.25 / 300 = 1/6. - for (query, expected) in [ - ("rate(m[5m]) * 2", 50. / 300. * 2.), - ("1 - rate(m[5m])", 1. - 50. / 300.), - // The stored sum evaluation keeps `__name__`; the arithmetic drops it. - ("sum_over_time(m[5m]) * 2", 150. * 2.), - ] { - assert_eq!( - run_series(&exact_dag(query), &samples, 300_000).unwrap(), - series(&[("api", "x", expected)]), - "{query}" - ); - } -} - -// A literal operand drops the metric name; series that then share a label set -// are an error, as in Prometheus, rather than duplicate output series. -#[test] -fn per_series_scalar_arithmetic_rejects_label_sets_equal_without_the_name() { - let dag = exact_dag("sum_over_time(m[5m]) * 2"); - let samples = counter("m", "api", 10., 10.) - .chain(counter("m", "api", 1., 1.).map(|s| (s.0, s.1, "y", s.3, s.4))) - .collect::>(); - let distinct = BTreeMap::from([("y", ("n", "y"))]); - assert!(execute_relabeled(&dag, &samples, 300_000, &distinct).is_ok()); - // Both series are then {instance="x",job="api"}, named m and n. - let equal = BTreeMap::from([("y", ("n", "x"))]); - let Err(error) = execute_relabeled(&dag, &samples, 300_000, &equal) else { - panic!("duplicate label sets were accepted"); - }; - assert!(error.contains("same labelset"), "{error}"); -} - -fn with_vector_match( - mut dag: PhysicalASAPDAG, - kind: planner_types::ir::operator::VectorMatchKind, - labels: &[&str], -) -> PhysicalASAPDAG { - for node in &mut dag.nodes { - if let PhysicalASAPOperatorPayload::NonASAP(planner_types::ir::NonASAPOp::BinaryOp { - operator, - .. - }) = &mut node.payload - { - operator.vector_match = Some(planner_types::ir::operator::VectorMatch { - kind: kind.clone(), - labels: labels.iter().map(|l| l.to_string()).collect(), - grouping: None, - }); - } - } - dag -} - -// `on` and `ignoring` reduce each side to the matching labels, which become -// the result's labels; a duplicate match group on either side is an error. -#[test] -fn per_series_vector_matching_follows_on_and_ignoring() { - use planner_types::ir::operator::VectorMatchKind; - let samples = counter("a", "api", 10., 10.) - .chain(counter("b", "api", 5., 5.)) - .chain(counter("b", "db", 5., 5.)) - .collect::>(); - let dag = exact_dag("rate(a[5m]) / rate(b[5m])"); - for (kind, labels) in [ - (VectorMatchKind::On, ["job"]), - (VectorMatchKind::Ignoring, ["instance"]), - ] { - let dag = with_vector_match(dag.clone(), kind, &labels); - assert_eq!( - run_series(&dag, &samples, 300_000).unwrap(), - format!( - "{:?}", - BTreeMap::from([(r#"{"job":"api"}"#.to_string(), 2.)]) - ) - ); - let mut duplicate = samples.clone(); - duplicate.extend(counter("b", "api", 5., 5.).map(|s| (s.0, s.1, "y", s.3, s.4))); - let error = run_series(&dag, &duplicate, 300_000).unwrap_err(); - assert!(error.contains("duplicate series"), "{error}"); - let mut duplicate = samples.clone(); - duplicate.extend(counter("a", "api", 5., 5.).map(|s| (s.0, s.1, "y", s.3, s.4))); - let error = run_series(&dag, &duplicate, 300_000).unwrap_err(); - assert!(error.contains("many-to-one"), "{error}"); - } -} - // Current-series sums and averages are compensated like Prometheus, and an // overflowing running sum does not turn the average into +Inf. #[test] @@ -728,98 +294,3 @@ fn population_sums_and_averages_are_compensated() { ); } } - -// A bare count over stored Count-Min state compiles to a Planner evaluation that -// returns the sketch's total update weight, including colliding items. -#[test] -fn stored_count_min_bare_count_compiles_to_a_evaluation() { - use asap_executor::summary_kernels::CountMinSketchAccumulator; - let accuracy = AccuracyTarget::max_error(0.02); - let root = lower_with("count(up)", accuracy.clone()); - let mut dag = computation_candidates(root) - .into_iter() - .find_map(|node| { - let dag = compile_physical_asap_dag(&node).ok()?; - let bare_count = dag.nodes.iter().any(|n| { - matches!( - &n.payload, - PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryEstimate { - query: SketchStatistic::PointCount { value: None, .. }, - .. - }) - ) - }); - let count_min = dag.nodes.iter().any(|n| { - matches!(&n.payload, PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), .. - }) if kind.algorithm() == &SketchAlgorithm::Cms) - }); - (bare_count && count_min).then_some(dag) - }) - .expect("Planner lists a Count-Min candidate for count(up)"); - let state = dag - .nodes - .iter() - .find(|n| { - matches!( - n.payload, - PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryAgg { .. }) - ) - }) - .unwrap(); - let PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), - .. - }) = &state.payload - else { - unreachable!() - }; - let SketchParams::Cms { width, depth } = kind.params() else { - unreachable!() - }; - let (state_id, schema) = (state.id as u64, Arc::new(state.output_schema.clone())); - let (width, depth) = (*width, *depth); - let views = views_at(&mut dag, &[state_id]).unwrap(); - let program = compile(&dag, &views, &[dag.roots[0] as u64]).unwrap(); - let mut sketch = CountMinSketchAccumulator::new(depth as usize, width as usize).unwrap(); - sketch.update("a", 3.0); - sketch.update("b", 7.0); - let row = schema - .fields - .iter() - .map(|field| match &field.dtype { - FieldDataType::Plain(_) => panic!("unexpected stored column {field:?}"), - family => Value::Summary { - family: family.clone(), - state: Arc::new(sketch.clone()), - }, - }) - .collect(); - let batch = Batch::try_new(schema.clone(), vec![row]).unwrap(); - let store = InMemoryDataStore::default() - .with_materialized_view(state_id as usize, vec![batch]) - .unwrap(); - let physical_dag = program.instantiate(Arc::new(store)).unwrap(); - let context = RunContext::new( - Scope::Query { - evaluation_time_ms: 0, - }, - Limits::default(), - ) - .unwrap(); - let values = block_on(async { - let mut stream = physical_dag - .execute(program.roots(), context) - .unwrap() - .remove(0); - let mut values = Vec::new(); - while let Some(batch) = stream.next().await { - values.extend(batch.unwrap().to_rows().iter().map(|row| row[0].clone())); - } - values - }); - assert!( - matches!(values.as_slice(), [Value::Int64(10)]), - "{values:?}" - ); -} diff --git a/crates/executor/tests/hydra_cms_execution.rs b/crates/executor/tests/hydra_cms_execution.rs index 7502bc10d..fc23257c5 100644 --- a/crates/executor/tests/hydra_cms_execution.rs +++ b/crates/executor/tests/hydra_cms_execution.rs @@ -17,7 +17,7 @@ use planner_types::ir::scalar::{ColumnRef, CompareOpKind, ScalarValue}; use planner_types::ir::schema::DataType; use planner_types::ir::schema::*; use planner_types::ir::{ASAPOp, ExprSemantics, Predicate, ScalarExpr}; -use planner_types::physical::Materialization; +use planner_types::physical::Lifetime; use std::{collections::BTreeMap, sync::Arc}; const WIDTH: u32 = 64; @@ -39,26 +39,11 @@ const ROWS: [(&str, &str, f64); 9] = [ fn grouping() -> GroupingStrategy { GroupingStrategy::SharedMultiSubpopulation { kind: HydraKind::HydraCms, - params: HydraParams::HydraCms { - width: WIDTH, - depth: 3, - shared_rows: 3, - shared_columns: SHARED_COLUMNS, - }, } } fn family() -> FieldDataType { - FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Cms, - SketchParams::Cms { - width: WIDTH, - depth: 3, - }, - ), - grouping(), - ) + FieldDataType::Sketch(SketchAlgorithm::Cms, grouping()) } fn schema(fields: Vec) -> Schema { @@ -75,7 +60,8 @@ fn node(id: usize, payload: Payload, output: Schema) -> PhysicalASAPDAGNode { id, payload, output_state: ExecutionDataState::QUERY_ROWS, - materialization: Materialization::NotMaterialized, + lifetime: Lifetime::None, + parameters: None, output_schema: output, guarantee: None, retention_ms: None, diff --git a/crates/executor/tests/keyed_summary_row_weight.rs b/crates/executor/tests/keyed_summary_row_weight.rs index 91f8bf032..19ecf85b9 100644 --- a/crates/executor/tests/keyed_summary_row_weight.rs +++ b/crates/executor/tests/keyed_summary_row_weight.rs @@ -17,7 +17,7 @@ use planner_types::ir::properties::ExecutionDataState; use planner_types::ir::scalar::ColumnRef; use planner_types::ir::schema::*; use planner_types::ir::ASAPOp; -use planner_types::physical::Materialization; +use planner_types::physical::Lifetime; use std::sync::Arc; /// `(job, service)` rows: jobs are the groups, services the items. @@ -32,14 +32,7 @@ const ROWS: [(&str, &str); 6] = [ fn heap_family() -> FieldDataType { FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::CmsWithHeap, - SketchParams::CmsWithHeap { - width: 1024, - depth: 3, - heap_size: 8, - }, - ), + SketchAlgorithm::CmsWithHeap, GroupingStrategy::PerSubpopulationInstance, ) } @@ -159,7 +152,8 @@ fn query_time_count_heap_binds_a_constant_row_weight() { filter: None, }), output_state: ExecutionDataState::QUERY_ROWS, - materialization: Materialization::NotMaterialized, + lifetime: Lifetime::None, + parameters: None, output_schema: (*schema(vec![ Field::plain("job", DataType::Utf8, false), Field { @@ -248,14 +242,7 @@ fn int64_weight_column_equals_float64_column() { counts(limit as f64) ); let count_sketch = FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::CountSketchWithHeap, - SketchParams::CountSketchWithHeap { - width: 1024, - depth: 3, - heap_size: 8, - }, - ), + SketchAlgorithm::CountSketchWithHeap, GroupingStrategy::PerSubpopulationInstance, ); let signed = Operator::per_group_keyed_summary_build( diff --git a/crates/executor/tests/main.rs b/crates/executor/tests/main.rs index 89abe7da8..ee0b768e5 100644 --- a/crates/executor/tests/main.rs +++ b/crates/executor/tests/main.rs @@ -17,7 +17,6 @@ mod physical_dag; mod physical_semantics; mod plan_properties; mod planspace_series_identity_heap; -mod precompute_candidates; mod precompute_population; mod promql_binary; mod promql_values; diff --git a/crates/executor/tests/node_ids.rs b/crates/executor/tests/node_ids.rs index 04a46fb54..961aa096d 100644 --- a/crates/executor/tests/node_ids.rs +++ b/crates/executor/tests/node_ids.rs @@ -1,7 +1,6 @@ //! Compilation rejects Planner node IDs outside the range its helper IDs avoid. -use crate::common; + use asap_executor::physical_planner::compile; -use common::compile_physical_asap_dag; use planner_types::ir::scalar::ScalarValue; use planner_types::ir::schema::{DataType, Field, Schema}; use planner_types::ir::{NonASAPOp, Operator, OperatorNode, Predicate, ScalarExpr}; @@ -20,7 +19,7 @@ fn node_ids_above_u32_are_rejected() { child: values, })) .unwrap(); - let mut dag = compile_physical_asap_dag(&filter).unwrap(); + let mut dag = planner_types::test_support::physical_asap_dag_at_query_time(&filter).unwrap(); let root = dag.roots[0] as u64; assert!(compile(&dag, &[], &[root]).is_ok()); diff --git a/crates/executor/tests/physical_dag.rs b/crates/executor/tests/physical_dag.rs index bd22b97d5..7a0595e9e 100644 --- a/crates/executor/tests/physical_dag.rs +++ b/crates/executor/tests/physical_dag.rs @@ -350,17 +350,11 @@ fn binding_rejects_unsupported_operations() { // KLL is one family example: precomputation changes input sources, not operators. #[test] fn kll_raw_partial_and_precomputed_are_native_dags() { - use planner_types::ir::schema::{GroupingStrategy, SketchAlgorithm, SketchKind, SketchParams}; + use planner_types::ir::schema::{GroupingStrategy, SketchAlgorithm}; let input = schema(&[("value", DataType::Float64, false)]); let family = FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: 512, - failure_probability: None, - }, - ), + SketchAlgorithm::Kll, GroupingStrategy::PerSubpopulationInstance, ); let build = Operator::per_group_summary_build(input.clone(), family, 0, None, vec![]).unwrap(); @@ -539,7 +533,8 @@ fn bind_physical_asap_dag_before_execution() { id, payload, output_state: ExecutionDataState::QUERY_ROWS, - materialization: Default::default(), + lifetime: Default::default(), + parameters: None, output_schema: (*schema).clone(), guarantee: None, retention_ms: None, @@ -700,7 +695,8 @@ fn planner_semijoin_sort_limit_contract_at_both_phases() { payload, output_schema: (**schema).clone(), output_state: ExecutionDataState::QUERY_ROWS, - materialization: Default::default(), + lifetime: Default::default(), + parameters: None, guarantee: None, retention_ms: None, }; @@ -975,7 +971,7 @@ fn weighted_rate_topk_preserves_partitions_fractional_scores_and_evaluation_scop } } fn assert_weighted_rate_topk(count_sketch: bool) { - use planner_types::ir::schema::{SketchAlgorithm, SketchKind, SketchParams}; + use planner_types::ir::schema::SketchAlgorithm; let raw = schema(&[ ("service", DataType::Utf8, false), @@ -1015,26 +1011,11 @@ fn assert_weighted_rate_topk(count_sketch: bool) { ) .unwrap(); let family = FieldDataType::Sketch( - SketchKind::new( - if count_sketch { - SketchAlgorithm::CountSketchWithHeap - } else { - SketchAlgorithm::CmsWithHeap - }, - if count_sketch { - SketchParams::CountSketchWithHeap { - width: 4096, - depth: 5, - heap_size: 8, - } - } else { - SketchParams::CmsWithHeap { - width: 4096, - depth: 5, - heap_size: 8, - } - }, - ), + if count_sketch { + SketchAlgorithm::CountSketchWithHeap + } else { + SketchAlgorithm::CmsWithHeap + }, Default::default(), ); let build = Operator::per_group_keyed_summary_build( @@ -1136,7 +1117,8 @@ fn grouped_temporal_schema_compiles_and_executes_topk() { id, payload, output_state: ExecutionDataState::QUERY_ROWS, - materialization: Default::default(), + lifetime: Default::default(), + parameters: None, output_schema: (*input).clone(), guarantee: None, retention_ms: None, @@ -1226,7 +1208,8 @@ fn certified_pruning_rejects_missing_authoritative_values_after_recovery() { id: 2, output_schema: (*schema).clone(), output_state: ExecutionDataState::QUERY_ROWS, - materialization: Default::default(), + lifetime: Default::default(), + parameters: None, guarantee: None, retention_ms: None, payload: PhysicalASAPOperatorPayload::NonASAP(planner_types::ir::NonASAPOp::Join { @@ -1331,7 +1314,8 @@ fn compiled_ingestion_binary_preserves_alignment_and_rejects_missing_updates() { id: 2, output_schema: (*input).clone(), output_state: ExecutionDataState::INGESTION_ROWS, - materialization: Default::default(), + lifetime: Default::default(), + parameters: None, guarantee: None, retention_ms: None, payload: PhysicalASAPOperatorPayload::NonASAP(planner_types::ir::NonASAPOp::BinaryOp { diff --git a/crates/executor/tests/physical_semantics.rs b/crates/executor/tests/physical_semantics.rs index 2831befe1..78bca3e36 100644 --- a/crates/executor/tests/physical_semantics.rs +++ b/crates/executor/tests/physical_semantics.rs @@ -385,7 +385,8 @@ fn global_extrema_bind_with_planner_derived_schema() { child: 0, }), output_state: ExecutionDataState::QUERY_ROWS, - materialization: Default::default(), + lifetime: Default::default(), + parameters: None, output_schema: (*output).clone(), guarantee: None, retention_ms: None, @@ -554,19 +555,10 @@ fn boolean_truth_tables_agree_between_expression_paths() { // Partial/final execution must agree with one build for an uncompacted KLL population. #[test] fn kll_partial_merge_and_multiple_evaluations_preserve_population() { - use planner_types::ir::schema::{SketchAlgorithm, SketchKind, SketchParams}; + use planner_types::ir::schema::SketchAlgorithm; let input = schema(&[("v", DataType::Float64, false)]); - let family = FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: 512, - failure_probability: None, - }, - ), - Default::default(), - ); + let family = FieldDataType::Sketch(SketchAlgorithm::Kll, Default::default()); let mut dag = PhysicalDAG::default(); for (id, range) in [(0, 0..64), (1, 64..128), (2, 0..128)] { let rows = range.map(|n| vec![Value::Float64(n as f64)]).collect(); @@ -650,7 +642,7 @@ fn kll_partial_merge_and_multiple_evaluations_preserve_population() { // single-build rank error: 60 tumbling windows of 1000 values at k = 200. #[test] fn compacted_kll_tumbling_windows_merge_within_the_calibrated_rank_error() { - use planner_types::ir::schema::{SketchAlgorithm, SketchKind, SketchParams, SketchStatistic}; + use planner_types::ir::schema::{SketchAlgorithm, SketchStatistic}; const TUMBLING_WINDOW_COUNT: u64 = 60; const ROWS_PER_TUMBLING_WINDOW: u64 = 1000; const K: u32 = 200; @@ -659,16 +651,7 @@ fn compacted_kll_tumbling_windows_merge_within_the_calibrated_rank_error() { let bound = 2.296 / f64::from(K).powf(0.9723); let n = (TUMBLING_WINDOW_COUNT * ROWS_PER_TUMBLING_WINDOW) as f64; let input = schema(&[("v", DataType::Float64, false)]); - let family = FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: K, - failure_probability: None, - }, - ), - Default::default(), - ); + let family = FieldDataType::Sketch(SketchAlgorithm::Kll, Default::default()); let build = || { Operator::per_group_summary_build(input.clone(), family.clone(), 0, None, vec![]).unwrap() }; @@ -935,7 +918,8 @@ fn planner_node( id: 1, payload, output_state: planner_types::ir::properties::ExecutionDataState::QUERY_ROWS, - materialization: Default::default(), + lifetime: Default::default(), + parameters: None, output_schema: planner_types::ir::schema::Schema::new( fields .iter() @@ -1032,21 +1016,11 @@ fn filtered_summary_build_keeps_empty_groups() { use planner_types::ir::operator::{GroupKeys, Reduction as Plan}; use planner_types::ir::scalar::ColumnRef; use planner_types::ir::schema::{ - ExactKind, ExactParams, SketchAlgorithm, SketchKind, SketchParams, SketchStatistic, - SummaryUpdate, + ExactKind, ExactParams, SketchAlgorithm, SketchStatistic, SummaryUpdate, }; use planner_types::ir::ASAPOp; let (input, rows) = filtered_input(); - let kll = FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: 200, - failure_probability: None, - }, - ), - Default::default(), - ); + let kll = FieldDataType::Sketch(SketchAlgorithm::Kll, Default::default()); let count = FieldDataType::ExactAggregate(ExactKind::Count, ExactParams::Count); let cases = [ ( @@ -1192,7 +1166,8 @@ fn exact_aggregate_node( child: 0, }), output_state: ExecutionDataState::QUERY_ROWS, - materialization: Default::default(), + lifetime: Default::default(), + parameters: None, output_schema: (*schema(&[(name, dtype, false)])).clone(), guarantee: None, retention_ms: None, diff --git a/crates/executor/tests/plan_properties.rs b/crates/executor/tests/plan_properties.rs index 3b664a12a..5b39368fc 100644 --- a/crates/executor/tests/plan_properties.rs +++ b/crates/executor/tests/plan_properties.rs @@ -49,7 +49,8 @@ fn sorted_scan(schema: &Schema, identity: &Source) -> PhysicalASAPDAG { id, payload, output_state: ExecutionDataState::QUERY_ROWS, - materialization: Default::default(), + lifetime: Default::default(), + parameters: None, output_schema: schema.clone(), guarantee: None, retention_ms: None, @@ -128,72 +129,6 @@ fn blocking_inputs_over_a_raw_data_slot_are_finite() { assert_eq!(opens.load(Ordering::SeqCst), 1); } -/// A KLL kernel accepts `k` only up to the 26 602 `asap_sketchlib` caps it at. -#[test] -fn kll_kernel_rejects_a_k_above_the_library_cap() { - use asap_executor::capability::validate_summary_kernel; - use planner_types::ir::scalar::ColumnRef; - use planner_types::ir::schema::{ - GroupingStrategy, SketchAlgorithm, SketchKind, SketchParams, SummaryInputExpr, - SummaryUpdate, - }; - let grouping = GroupingStrategy::default(); - let update = SummaryUpdate { - item: None, - weight: SummaryInputExpr::Column(ColumnRef::SampleValue), - weight_domain: Default::default(), - }; - let kll = |k| { - FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k, - failure_probability: None, - }, - ), - grouping.clone(), - ) - }; - assert!(validate_summary_kernel(&kll(26_602), &update, &grouping).is_ok()); - assert!(validate_summary_kernel(&kll(26_603), &update, &grouping).is_err()); -} - -/// A plain CountSketch reads the median of its rows, so plan validation and -/// kernel construction both require a positive odd depth. -#[test] -fn count_sketch_kernel_requires_an_odd_depth() { - use asap_executor::{capability::validate_summary_kernel, factory::create_planner_accumulator}; - use planner_types::ir::scalar::ColumnRef; - use planner_types::ir::schema::{ - GroupingStrategy, SketchAlgorithm, SketchKind, SketchParams, SummaryInputExpr, - SummaryUpdate, - }; - let grouping = GroupingStrategy::default(); - let mut update = SummaryUpdate::column(ColumnRef::SampleValue); - update.item = Some(SummaryInputExpr::Column(ColumnRef::Named("host".into()))); - let count_sketch = |depth| { - FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::CountSketch, - SketchParams::CountSketch { width: 64, depth }, - ), - grouping.clone(), - ) - }; - for (depth, valid) in [(5, true), (4, false)] { - let family = count_sketch(depth); - assert_eq!( - validate_summary_kernel(&family, &update, &grouping).is_ok(), - valid - ); - assert_eq!( - create_planner_accumulator(&family, &update, &grouping).is_ok(), - valid - ); - } -} - // Kernel support must not be mistaken for executable native state/evaluation support. #[test] fn summary_capability_levels_are_distinct() { @@ -202,20 +137,9 @@ fn summary_capability_levels_are_distinct() { planner::ir::schema::SketchStatistic, }; use planner_types::ir::scalar::ColumnRef; - use planner_types::ir::schema::{ - GroupingStrategy, SketchAlgorithm, SketchKind, SketchParams, SummaryUpdate, - }; + use planner_types::ir::schema::{GroupingStrategy, SketchAlgorithm, SummaryUpdate}; let grouping = GroupingStrategy::default(); - let cms = FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Cms, - SketchParams::Cms { - width: 64, - depth: 4, - }, - ), - grouping.clone(), - ); + let cms = FieldDataType::Sketch(SketchAlgorithm::Cms, grouping.clone()); let update = SummaryUpdate { item: Some(planner_types::ir::schema::SummaryInputExpr::Column( ColumnRef::Named("host".into()), @@ -239,16 +163,7 @@ fn summary_capability_levels_are_distinct() { } ) .is_err()); - let kll = FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: 128, - failure_probability: None, - }, - ), - grouping, - ); + let kll = FieldDataType::Sketch(SketchAlgorithm::Kll, grouping); assert!(validate_native_family(&kll).is_ok()); assert!(validate_sketch_evaluation(&kll, &SketchStatistic::Quantile { q: 1.5 }).is_err()); assert!(validate_sketch_evaluation(&kll, &SketchStatistic::Cardinality).is_err()); diff --git a/crates/executor/tests/planspace_series_identity_heap.rs b/crates/executor/tests/planspace_series_identity_heap.rs index 18919c26c..4ae5aff80 100644 --- a/crates/executor/tests/planspace_series_identity_heap.rs +++ b/crates/executor/tests/planspace_series_identity_heap.rs @@ -2,7 +2,7 @@ //! the query's rows do not carry: the planner pipeline plans the logical DAG //! the frontend lowered, and a deployment prices heaps. use crate::common; -use common::{compile_physical_asap_dag, selected_dag}; +use common::selected_dag; use planner_types::ir::OperatorNode; use asap_executor::physical_planner::promql_rows::SERIES_IDENTITY_COLUMN; @@ -44,7 +44,7 @@ fn lower(query: &str, accuracy: &AccuracyTarget) -> Rc { } fn carries_identity(root: &Rc) -> bool { - compile_physical_asap_dag(root) + planner_types::test_support::physical_asap_dag_at_query_time(root) .unwrap() .nodes .iter() diff --git a/crates/executor/tests/precompute_candidates.rs b/crates/executor/tests/precompute_candidates.rs deleted file mode 100644 index b540f8110..000000000 --- a/crates/executor/tests/precompute_candidates.rs +++ /dev/null @@ -1,363 +0,0 @@ -//! A grouped Rate→Sum answers the same whether the deployment stores the -//! Rate state or the grouped Sum state. -use crate::common; -use asap_executor::sources::InMemoryDataStore; -use asap_executor::{ - factory::create_planner_accumulator, - physical_planner::{compile, test_support::views_at, CompiledPhysicalDAG}, - runtime::{Limits, RunContext, Scope}, - values::{Batch, Value}, -}; -use common::{compile_physical_asap_dag, computation_candidates, selected_dag}; -use futures::{executor::block_on, StreamExt}; -use planner_types::ir::physical_export::{PhysicalASAPDAG, PhysicalASAPOperatorPayload}; -use planner_types::ir::properties::AccuracyTarget; -use planner_types::ir::schema::{DataType, *}; -use planner_types::ir::ASAPOp; -use planner_types::workload::*; -use std::{collections::BTreeMap, sync::Arc}; - -/// `sum by(job)(rate(m[1m]))` with its rows carrying the series identity. -fn grouped_rate_root() -> std::rc::Rc { - exact_promql_root("sum by(job)(rate(m[1m]))") -} - -/// `query`, exact, lowered with its rows carrying the series identity. -fn exact_promql_root(query: &str) -> std::rc::Rc { - let workload = PlanningWorkload { - query_workload: QueryWorkload { - language: QueryLanguage::PromQL, - query_batch: Some(vec![BatchEntry { - query: Query(query.into()), - requirements: QueryRequirements { - accuracy: AccuracyTarget::exact(), - ..Default::default() - }, - predictability: Predictability::Unknown, - invocations: 1, - execute_at: None, - range_evaluation: None, - }]), - repeating_queries: None, - }, - data_workload: Some(DataWorkload::sampled_every( - asap_frontend_promql::promql_metric_names(query).unwrap_or_default(), - DurationMs(1000), - )), - }; - let root = asap_frontend_promql::lower_promql_query_workload(&workload) - .unwrap() - .remove(0) - .as_operator() - .cloned() - .expect("operator root"); - asap_executor::physical_planner::promql_rows::with_series_identity(&root).unwrap() -} - -fn grouped_rate() -> PhysicalASAPDAG { - compile_physical_asap_dag(&selected_dag(grouped_rate_root(), AccuracyTarget::exact())).unwrap() -} -fn run(plan: &CompiledPhysicalDAG, store: InMemoryDataStore, scope: Scope) -> Vec { - let dag = plan.instantiate(Arc::new(store)).unwrap(); - block_on(async { - let context = RunContext::new(scope, Limits::default()).unwrap(); - let mut output = dag.execute(plan.roots(), context).unwrap().remove(0); - let mut batches = vec![]; - while let Some(batch) = output.next().await { - batches.push((*batch.unwrap()).clone()); - } - batches - }) -} - -/// The root's output when the deployment stores the Rate `state` (bound to -/// `states`) and, with `stored`, also the output of node `stored`, which an -/// ingestion run over `ingestion` computes from the states. -fn answer( - dag: &PhysicalASAPDAG, - state: u64, - states: &Batch, - stored: Option, - ingestion: Scope, - query: Scope, -) -> Vec { - let mut dag = dag.clone(); - let views = views_at( - &mut dag, - &[state].into_iter().chain(stored).collect::>(), - ) - .unwrap(); - let state_store = || { - InMemoryDataStore::default() - .with_materialized_view(state as usize, vec![states.clone()]) - .unwrap() - }; - let root = [dag.roots[0] as u64]; - let Some(stored) = stored else { - return run(&compile(&dag, &views, &root).unwrap(), state_store(), query); - }; - let outputs = run( - &compile(&dag, &views[..1], &[stored]).unwrap(), - state_store(), - ingestion, - ); - assert_eq!(outputs.len(), 1); - let store = InMemoryDataStore::default() - .with_materialized_view(stored as usize, outputs) - .unwrap(); - run(&compile(&dag, &views[1..], &root).unwrap(), store, query) -} - -/// Rate evaluations and grouped Sum can run together at ingestion into a -/// stored Sum state, which answers as the query-time Rate→Sum over the stored -/// Rate state. Moving Sum before Rate would answer differently across -/// counter resets. -#[test] -fn grouped_rate_can_be_materialized_before_or_after_grouped_sum() { - let dag = grouped_rate(); - let state = dag - .nodes - .iter() - .find(|node| { - matches!( - node.payload, - PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryAgg { - family: FieldDataType::ExactAggregate(ExactKind::Rate, _), - .. - }) - ) - }) - .unwrap(); - let input_schema = Arc::new(state.output_schema.clone()); - let (family, update, grouping) = match &state.payload { - PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryAgg { - family, - input, - grouping, - .. - }) => (family, input, grouping), - _ => unreachable!(), - }; - let range_ms = Some((-58_000, 2_000)); - let mut expected_rate_sum = 0.; - let rows = [[100., 0., 100.], [100., 200., 0.]] - .into_iter() - .enumerate() - .map(|(index, values)| { - let mut accumulator = create_planner_accumulator(family, update, grouping).unwrap(); - for (i, value) in values.into_iter().enumerate() { - accumulator.update_single(value, i as i64 * 1000).unwrap(); - } - let state = accumulator.into_accumulator(); - expected_rate_sum += state - .as_any() - .downcast_ref::() - .unwrap() - .evaluation(ExactKind::Rate, range_ms, None) - .unwrap() - .unwrap(); - let summary = Value::Summary { - family: family.clone(), - state: Arc::from(state), - }; - input_schema - .fields - .iter() - .map(|field| match &field.dtype { - FieldDataType::ExactAggregate(..) => summary.clone(), - FieldDataType::Plain(DataType::Timestamp) => Value::Timestamp(2000), - FieldDataType::Plain(DataType::Utf8) => Value::Utf8(if field.name == "job" { - "api".into() - } else { - serde_json::to_string(&BTreeMap::from([ - ("__name__", "m".to_string()), - ("job", "api".to_string()), - ("instance", format!("series-{index}")), - ])) - .unwrap() - .into() - }), - _ => panic!("unexpected input field {field:?}"), - }) - .collect() - }) - .collect(); - let batch = Batch::try_new(input_schema.clone(), rows).unwrap(); - let state_id = state.id as u64; - let sum_id = dag - .nodes - .iter() - .find(|node| { - matches!( - node.payload, - PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryAgg { - family: FieldDataType::ExactAggregate(ExactKind::Sum, _), - .. - }) - ) - }) - .unwrap() - .id as u64; - let ingestion = Scope::Ingestion { - window_start_ms: -58_000, - window_end_ms: 2000, - }; - let query = Scope::Query { - evaluation_time_ms: 2000, - }; - let mut results = vec![]; - for stored in [None, Some(sum_id)] { - let output = answer( - &dag, - state_id, - &batch, - stored, - ingestion.clone(), - query.clone(), - ); - assert_eq!(output.len(), 1, "{stored:?}"); - assert_eq!(output[0].to_rows().len(), 1); - assert!(matches!(&output[0].to_rows()[0][0], Value::Utf8(job) if job.as_ref() == "api")); - assert!( - matches!(output[0].to_rows()[0][1], Value::Float64(value) if value == expected_rate_sum) - ); - results.push( - output[0].to_rows()[0] - .iter() - .map(|value| value.key().unwrap()) - .collect::>(), - ); - } - assert_eq!(results[0], results[1]); - let mut wrong_order = create_planner_accumulator(family, update, grouping).unwrap(); - for (i, value) in [200., 200., 100.].into_iter().enumerate() { - wrong_order.update_single(value, i as i64 * 1000).unwrap(); - } - let rate_of_sum = wrong_order - .into_accumulator() - .as_any() - .downcast_ref::() - .unwrap() - .evaluation(ExactKind::Rate, range_ms, None) - .unwrap() - .unwrap(); - assert_ne!( - expected_rate_sum, rate_of_sum, - "counter resets prohibit moving Sum before Rate" - ); -} - -/// Every Pass 1 computation candidate of the grouped Rate answers a numeric -/// value at query time, from the stored Rate state or a stored grouped Sum. -#[test] -fn enumerated_grouped_rate_candidates_execute_numeric_query_outputs() { - let mut executed = 0; - for root in computation_candidates(grouped_rate_root()) { - let dag = compile_physical_asap_dag(&root).unwrap(); - let Some(state) = dag.nodes.iter().find(|node| { - matches!( - node.payload, - PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryAgg { - family: FieldDataType::ExactAggregate(ExactKind::Rate, _), - .. - }) - ) - }) else { - continue; - }; - let boundary = dag - .nodes - .iter() - .find(|node| { - matches!( - node.payload, - PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryAgg { - family: FieldDataType::ExactAggregate(ExactKind::Sum, _), - .. - }) - ) - }) - .map(|node| node.id as u64) - .unwrap_or(dag.roots[0] as u64); - let (family, input, grouping) = match &state.payload { - PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryAgg { - family, - input, - grouping, - .. - }) => (family, input, grouping), - _ => unreachable!(), - }; - let schema = Arc::new(state.output_schema.clone()); - let rows = ["a", "b"] - .into_iter() - .map(|instance| { - let mut accumulator = create_planner_accumulator(family, input, grouping).unwrap(); - for (timestamp, value) in [(1_000, 1.), (31_000, 31.), (59_000, 59.)] { - accumulator.update_single(value, timestamp).unwrap(); - } - let summary = Value::Summary { - family: family.clone(), - state: Arc::from(accumulator.into_accumulator()), - }; - schema - .fields - .iter() - .map(|field| match &field.dtype { - FieldDataType::ExactAggregate(..) => summary.clone(), - FieldDataType::Plain(DataType::Timestamp) => Value::Timestamp(60_000), - FieldDataType::Plain(DataType::Utf8) - if field.name == "$promql_series_identity" => - { - Value::Utf8( - serde_json::to_string(&BTreeMap::from([ - ("job", "api"), - ("instance", instance), - ])) - .unwrap() - .into(), - ) - } - FieldDataType::Plain(DataType::Utf8) => Value::Utf8("api".into()), - _ => panic!("unexpected input field {field:?}"), - }) - .collect() - }) - .collect(); - let batch = Batch::try_new(schema, rows).unwrap(); - for stored in [None, Some(boundary)] { - let output = answer( - &dag, - state.id as u64, - &batch, - stored, - Scope::Ingestion { - window_start_ms: 0, - window_end_ms: 60_000, - }, - Scope::Query { - evaluation_time_ms: 60_000, - }, - ); - assert_eq!(output.len(), 1); - assert_eq!(output[0].to_rows().len(), 1); - assert!(output[0] - .schema() - .fields - .iter() - .all(|field| matches!(field.dtype, FieldDataType::Plain(_)))); - assert!( - output[0].to_rows()[0] - .iter() - .any(|value| matches!(value, Value::Float64(x) if (*x - 2.).abs() < 1e-12)), - "{:?}", - output[0].to_rows() - ); - executed += 1; - } - } - assert!( - executed >= 2, - "must execute both stored and query-time grouped Rate candidates: {executed}" - ); -} diff --git a/crates/executor/tests/precompute_population.rs b/crates/executor/tests/precompute_population.rs index 81e9b58cb..50cef473b 100644 --- a/crates/executor/tests/precompute_population.rs +++ b/crates/executor/tests/precompute_population.rs @@ -30,7 +30,7 @@ fn logical_schema(family: FieldDataType) -> Schema { time_index: None, } } -/// The stored state at node 0 (`MatAtIngestion`): a global `family` +/// The stored state at node 0 (materialized at ingestion time): a global `family` /// `SummaryAgg` over a minute of raw `m` samples (`Scan` node `scan`, /// `TimeRange` node `scan + 1`). fn view_over_samples( @@ -50,15 +50,16 @@ fn view_over_samples( id, payload, output_state, - materialization: Default::default(), + lifetime: Default::default(), + parameters: None, output_schema, guarantee: None, retention_ms: None, }; [ PhysicalASAPDAGNode { - materialization: planner_types::physical::Materialization::MatAtIngestion( - planner_types::physical::Medium::Memory, + lifetime: planner_types::physical::Lifetime::AcrossEvaluations( + planner_types::physical::StorageMedium::Memory, ), ..node( 0, @@ -137,9 +138,10 @@ fn try_state_dag( group_by: None, }), output_state: ExecutionDataState::INGESTION_SUMMARY, - materialization: planner_types::physical::Materialization::MatAtIngestion( - planner_types::physical::Medium::Memory, + lifetime: planner_types::physical::Lifetime::AcrossEvaluations( + planner_types::physical::StorageMedium::Memory, ), + parameters: None, output_schema: logical_schema(family.clone()), guarantee: None, retention_ms: None, @@ -152,7 +154,8 @@ fn try_state_dag( child: read_id - 1, }), output_state: ExecutionDataState::INGESTION_ROWS, - materialization: Default::default(), + lifetime: Default::default(), + parameters: None, output_schema: logical_schema(FieldDataType::Plain(DataType::Float64)), guarantee: None, retention_ms: None, @@ -169,9 +172,10 @@ fn try_state_dag( child: read_id, }), output_state: ExecutionDataState::INGESTION_SUMMARY, - materialization: planner_types::physical::Materialization::MatAtIngestion( - planner_types::physical::Medium::Memory, + lifetime: planner_types::physical::Lifetime::AcrossEvaluations( + planner_types::physical::StorageMedium::Memory, ), + parameters: None, output_schema: logical_schema(target), guarantee: None, retention_ms: None, @@ -286,13 +290,7 @@ fn explicit_merge_changes_tumbling_window_cardinality() { #[test] fn precompute_rejects_nonfinite_and_out_of_range_dds_updates() { let source = FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum); - let target = FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::DDSketch, - SketchParams::DDSketch { alpha: 0.01 }, - ), - GroupingStrategy::default(), - ); + let target = FieldDataType::Sketch(SketchAlgorithm::DDSketch, GroupingStrategy::default()); let program = state_dag(source.clone(), Some(target), false); for value in [20., -20., 0.] { assert!( diff --git a/crates/executor/tests/promql_binary.rs b/crates/executor/tests/promql_binary.rs index 8bff8a966..47a13aebd 100644 --- a/crates/executor/tests/promql_binary.rs +++ b/crates/executor/tests/promql_binary.rs @@ -81,7 +81,8 @@ fn program_for_bool(operator: BinaryOperator, return_bool: bool) -> Operator { rhs: 1, }), output_state: ExecutionDataState::QUERY_ROWS, - materialization: Default::default(), + lifetime: Default::default(), + parameters: None, output_schema: (*schema).clone(), guarantee: None, retention_ms: None, @@ -416,7 +417,8 @@ fn stored_series_evaluations_support_filters_and_sets() { 5 | 6 => ExecutionDataState::QUERY_ROWS, _ => ExecutionDataState::QUERY_ROWS, }, - materialization: Default::default(), + lifetime: Default::default(), + parameters: None, output_schema: match id { 0 | 1 => (*state_schema).clone(), 5 | 6 => sample_schema(), diff --git a/crates/executor/tests/promql_values.rs b/crates/executor/tests/promql_values.rs index 0585c4198..57f56c020 100644 --- a/crates/executor/tests/promql_values.rs +++ b/crates/executor/tests/promql_values.rs @@ -540,6 +540,7 @@ fn exact_state_evaluations_recover_and_finalize_tumbling_windows() { .map(|samples| { let mut state = create_planner_accumulator( &family, + None, &SummaryUpdate::column(ColumnRef::SampleValue), &GroupingStrategy::PerSubpopulationInstance, ) @@ -588,6 +589,7 @@ fn recovered_exact_counter_uses_window_and_omits_insufficient_samples() { .map(|count| { let mut state = create_planner_accumulator( &family, + None, &SummaryUpdate::column(ColumnRef::SampleValue), &GroupingStrategy::PerSubpopulationInstance, ) diff --git a/crates/executor/tests/raw_scan.rs b/crates/executor/tests/raw_scan.rs index 5cdafc837..7a2ff24bf 100644 --- a/crates/executor/tests/raw_scan.rs +++ b/crates/executor/tests/raw_scan.rs @@ -84,7 +84,8 @@ fn plan( id, payload, output_state: state, - materialization: Default::default(), + lifetime: Default::default(), + parameters: None, output_schema: (**schema).clone(), guarantee: None, retention_ms: None, @@ -504,7 +505,8 @@ fn windowed_plan(shift_ms: Option) -> PhysicalASAPDAG { id, payload: PhysicalASAPOperatorPayload::NonASAP(payload), output_state: state, - materialization: Default::default(), + lifetime: Default::default(), + parameters: None, output_schema: schema.clone(), guarantee: None, retention_ms: None, @@ -837,7 +839,8 @@ fn windowed_count_plan(shift_ms: Option) -> PhysicalASAPDAG { id, payload, output_state, - materialization: Default::default(), + lifetime: Default::default(), + parameters: None, output_schema: output_schema.clone(), guarantee: None, retention_ms: None, diff --git a/crates/executor/tests/state_column_updater.rs b/crates/executor/tests/state_column_updater.rs index a87ee00bf..3957844d8 100644 --- a/crates/executor/tests/state_column_updater.rs +++ b/crates/executor/tests/state_column_updater.rs @@ -1,13 +1,13 @@ //! The per-sample `StateColumnUpdater` builds the same states as the //! batch `SummaryAgg` build and as each family's kernel updated row by row, //! for every family a per-group build supports. -use asap_executor::factory::create_planner_accumulator; + use asap_executor::operators::{Operator, RowWeight}; use asap_executor::plan::PhysicalDAG; use asap_executor::planner::ir::scalar::ColumnRef; use asap_executor::planner::ir::schema::{ - DataType, ExactKind, ExactParams, Field, FieldDataType, GroupingStrategy, HydraKind, - HydraParams, Schema, SketchAlgorithm, SketchKind, SketchParams, SketchStatistic, SummaryUpdate, + DataType, ExactKind, ExactParams, Field, FieldDataType, GroupingStrategy, HydraKind, Schema, + SketchAlgorithm, SketchParams, SketchStatistic, }; use asap_executor::runtime::{Limits, RunContext, Scope}; use asap_executor::summary_kernels::weighted_frequency::WeightedFrequency; @@ -91,63 +91,14 @@ fn input_batch() -> Batch { .unwrap() } -fn sketch(algorithm: SketchAlgorithm, params: SketchParams) -> FieldDataType { - FieldDataType::Sketch(SketchKind::new(algorithm, params), Default::default()) +fn sketch(algorithm: SketchAlgorithm, _params: SketchParams) -> FieldDataType { + FieldDataType::Sketch(algorithm, Default::default()) } fn exact(kind: ExactKind, params: ExactParams) -> FieldDataType { FieldDataType::ExactAggregate(kind, params) } -/// Every family a per-group value build supports. -fn value_families() -> Vec { - vec![ - exact(ExactKind::Sum, ExactParams::Sum), - exact(ExactKind::Count, ExactParams::Count), - exact(ExactKind::Min, ExactParams::Min), - exact(ExactKind::Max, ExactParams::Max), - exact(ExactKind::Rate, ExactParams::Rate), - exact(ExactKind::Increase, ExactParams::Increase), - sketch( - SketchAlgorithm::Kll, - SketchParams::Kll { - // Above any group's sample count (at most 600): no - // compaction, so the random compaction coin never flips. - k: 1024, - failure_probability: None, - }, - ), - sketch( - SketchAlgorithm::DDSketch, - SketchParams::DDSketch { alpha: 0.01 }, - ), - sketch( - SketchAlgorithm::Hll, - SketchParams::Hll { - precision: 10, - relative_error: None, - }, - ), - sketch( - SketchAlgorithm::Kmv, - SketchParams::Kmv { - k: 32, - relative_error: None, - }, - ), - sketch( - SketchAlgorithm::UnivMon, - SketchParams::UnivMon { - heap_size: 64, - sketch_rows: 5, - sketch_cols: 128, - layers: 4, - distinct_failure_probability: None, - }, - ), - ] -} - /// Every keyed family a per-group keyed build supports. fn keyed_families() -> Vec { vec![ @@ -281,223 +232,6 @@ fn batch_build(build: Operator) -> BTreeMap { /// One test sample: series, timestamp, value, item. type TestSample = (String, i64, f64, String); -/// How a test feeds one sample to an updater. -type FeedSample<'a> = dyn FnMut(&mut StateColumnUpdater, u32, &TestSample) + 'a; - -/// The per-sample build: one updater, groups numbered as series appear. -/// With `sharded`, the first half of the samples go to one updater and the -/// rest to another, merged at the end as a runtime's shards are. -fn per_sample_build( - family: &FieldDataType, - sharded: bool, - mut update: impl FnMut(&mut StateColumnUpdater, u32, &(String, i64, f64, String)), -) -> BTreeMap { - let samples = samples(); - let mut groups: BTreeMap = BTreeMap::new(); - for sample in &samples { - let next = groups.len() as u32; - groups.entry(sample.0.clone()).or_insert(next); - } - let shard = |part: &[(String, i64, f64, String)], update: &mut FeedSample<'_>| { - let mut updater = StateColumnUpdater::new(family).unwrap(); - updater.ensure_groups(groups.len()).unwrap(); - for sample in part { - update(&mut updater, groups[&sample.0], sample); - } - updater - }; - let split = if sharded { - samples.len() / 2 - } else { - samples.len() - }; - let (first, second) = samples.split_at(split); - let mut updater = shard(first, &mut update); - if sharded { - updater.merge(shard(second, &mut update)).unwrap(); - } - assert_eq!(updater.group_count(), groups.len()); - let states = updater.seal(); - groups - .into_iter() - .map(|(series, group)| (series, fingerprint(states.state(group as usize).as_ref()))) - .collect() -} - -/// The pre-columnar reference: each family's kernel, one per series, -/// updated row by row. With `sharded`, two halves merged through the -/// kernel's own merge. -fn kernel_build(family: &FieldDataType, keyed: bool, sharded: bool) -> BTreeMap { - let update = if keyed { - SummaryUpdate { - item: Some( - asap_executor::planner::ir::schema::SummaryInputExpr::Column(ColumnRef::Named( - "item".into(), - )), - ), - weight: asap_executor::planner::ir::schema::SummaryInputExpr::Constant(1.0), - weight_domain: Default::default(), - } - } else { - SummaryUpdate::column(ColumnRef::SampleValue) - }; - let samples = samples(); - let split = if sharded { - samples.len() / 2 - } else { - samples.len() - }; - let (first, second) = samples.split_at(split); - let half = |part: &[(String, i64, f64, String)]| { - let mut states = BTreeMap::new(); - for (series, ts, value, item) in part { - let state = states.entry(series.clone()).or_insert_with(|| { - if keyed { - let FieldDataType::Sketch(kind, _) = family else { - unreachable!() - }; - let (algorithm, width, depth, capacity) = match kind.params() { - SketchParams::CmsWithHeap { - width, - depth, - heap_size, - } => ( - asap_executor::summary_kernels::weighted_frequency::FrequencyAlgorithm::Cms, - width, - depth, - heap_size, - ), - SketchParams::CountSketchWithHeap { - width, - depth, - heap_size, - } => ( - asap_executor::summary_kernels::weighted_frequency::FrequencyAlgorithm::CountSketch, - width, - depth, - heap_size, - ), - _ => unreachable!(), - }; - Err(Box::new( - WeightedFrequency::new( - algorithm, - *width as usize, - *depth as usize, - *capacity as usize, - ) - .unwrap(), - )) - } else { - Ok(create_planner_accumulator(family, &update, &Default::default()).unwrap()) - } - }); - match state { - Ok(updater) => updater.update_value(&Value::Float64(*value), *ts).unwrap(), - Err(keyed) => keyed - .update(&[Value::Utf8(item.as_str().into())], 1.0) - .unwrap(), - } - } - states - .into_iter() - .map(|(series, state)| { - ( - series, - match state { - Ok(updater) => Arc::::from(updater.into_accumulator()), - Err(keyed) => Arc::new(*keyed) as Arc, - }, - ) - }) - .collect::>() - }; - let (mut merged, second) = (half(first), half(second)); - for (series, state) in second { - let combined = match merged.get(&series) { - Some(existing) => Arc::from(existing.merge_with(state.as_ref()).unwrap()), - None => state, - }; - merged.insert(series, combined); - } - merged - .into_iter() - .map(|(series, state)| (series, fingerprint(state.as_ref()))) - .collect() -} - -// For every value family: the per-sample updater, the batch `SummaryAgg` -// build and the family's own kernel build the same state for every group, -// byte for byte; two updater shards merged equal the kernel's merge of the -// same two halves. -#[test] -fn per_sample_updates_build_the_batch_states_for_every_value_family() { - for family in value_families() { - let batch = batch_build( - Operator::per_group_summary_build(input_schema(), family.clone(), 2, Some(1), vec![0]) - .unwrap(), - ); - let update = - |updater: &mut StateColumnUpdater, group, sample: &(String, i64, f64, String)| { - updater.update(group, sample.2, sample.1).unwrap() - }; - assert_eq!(batch.len(), 7, "{family:?}"); - assert_eq!( - per_sample_build(&family, false, update), - batch, - "{family:?}: updater vs batch build" - ); - assert_eq!( - kernel_build(&family, false, false), - batch, - "{family:?}: kernel vs batch build" - ); - assert_eq!( - per_sample_build(&family, true, update), - kernel_build(&family, false, true), - "{family:?}: merged shards vs kernel merge" - ); - } -} - -// The same for the keyed families (an item and a weight per sample). -#[test] -fn per_sample_updates_build_the_batch_states_for_every_keyed_family() { - for family in keyed_families() { - let batch = batch_build( - Operator::per_group_keyed_summary_build( - input_schema(), - family.clone(), - vec![3], - RowWeight::Constant(NonZeroU32::new(1).unwrap()), - vec![0], - ) - .unwrap(), - ); - let update = - |updater: &mut StateColumnUpdater, group, sample: &(String, i64, f64, String)| { - updater - .update_item(group, &[Value::Utf8(sample.3.as_str().into())], 1.0) - .unwrap() - }; - assert_eq!(batch.len(), 7, "{family:?}"); - assert_eq!( - per_sample_build(&family, false, update), - batch, - "{family:?}: updater vs batch build" - ); - assert_eq!( - kernel_build(&family, true, false), - batch, - "{family:?}: kernel vs batch build" - ); - assert_eq!( - per_sample_build(&family, true, update), - kernel_build(&family, true, true), - "{family:?}: merged shards vs kernel merge" - ); - } -} // A counter refuses a sample that is not after its group's previous one, // and a value outside the kernel's domain, leaving the state unchanged. @@ -594,21 +328,9 @@ fn an_unknown_group_is_bad_input() { /// A HydraCms family: one shared grid for all groups. fn hydra_family() -> FieldDataType { FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Cms, - SketchParams::Cms { - width: 64, - depth: 3, - }, - ), + SketchAlgorithm::Cms, GroupingStrategy::SharedMultiSubpopulation { kind: HydraKind::HydraCms, - params: HydraParams::HydraCms { - width: 64, - depth: 3, - shared_rows: 3, - shared_columns: 16, - }, }, ) } diff --git a/crates/executor/tests/stored_tumbling_window_memory_limit.rs b/crates/executor/tests/stored_tumbling_window_memory_limit.rs index 350e66212..946e786fb 100644 --- a/crates/executor/tests/stored_tumbling_window_memory_limit.rs +++ b/crates/executor/tests/stored_tumbling_window_memory_limit.rs @@ -5,9 +5,7 @@ use asap_executor::arrow::array::{DictionaryArray, UInt32Array}; use asap_executor::arrow::datatypes::UInt32Type; use asap_executor::operators::{Operator, SortKey}; use asap_executor::plan::PhysicalDAG; -use asap_executor::planner::ir::schema::{ - DataType, Field, FieldDataType, Schema, SketchAlgorithm, SketchKind, SketchParams, -}; +use asap_executor::planner::ir::schema::{DataType, Field, FieldDataType, Schema, SketchAlgorithm}; use asap_executor::runtime::{Limits, RunContext, Scope}; use asap_executor::values::{Batch, BatchColumn}; use asap_executor::{Error, StateColumnUpdater}; @@ -27,16 +25,7 @@ fn tumbling_window() -> (Arc, Batch) { /// One tumbling window of `series` KLL states (k = 200), keyed by series. fn tumbling_window_of(series_count: u32) -> (Arc, Batch) { - let family = FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: 200, - failure_probability: None, - }, - ), - Default::default(), - ); + let family = FieldDataType::Sketch(SketchAlgorithm::Kll, Default::default()); let schema = Arc::new(Schema { fields: vec![ Field { @@ -328,7 +317,8 @@ fn merge_over_a_stored_tumbling_window( payload, output_schema: output_schema.clone(), output_state, - materialization: Default::default(), + lifetime: Default::default(), + parameters: None, guarantee: None, retention_ms: None, }; diff --git a/crates/executor/tests/summary_build_memory_limit.rs b/crates/executor/tests/summary_build_memory_limit.rs index 80af9b222..a14d3c713 100644 --- a/crates/executor/tests/summary_build_memory_limit.rs +++ b/crates/executor/tests/summary_build_memory_limit.rs @@ -3,9 +3,7 @@ //! adds them, a bounded number of rows at a time. use asap_executor::operators::Operator; use asap_executor::plan::PhysicalDAG; -use asap_executor::planner::ir::schema::{ - DataType, Field, FieldDataType, Schema, SketchAlgorithm, SketchKind, SketchParams, -}; +use asap_executor::planner::ir::schema::{DataType, Field, FieldDataType, Schema, SketchAlgorithm}; use asap_executor::runtime::{Limits, RunContext, Scope}; use asap_executor::values::{Batch, Value}; use asap_executor::Error; @@ -57,16 +55,7 @@ fn one_large_batch_fails_near_the_memory_limit() { .map(|i| vec![Value::Utf8(format!("s{i}").into()), Value::Float64(1.0)]) .collect(); let batch = Batch::try_new(input.clone(), rows).unwrap(); - let family = FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: 200, - failure_probability: None, - }, - ), - Default::default(), - ); + let family = FieldDataType::Sketch(SketchAlgorithm::Kll, Default::default()); let mut dag = PhysicalDAG::default(); dag.add( 0, diff --git a/crates/executor/tests/summary_projection.rs b/crates/executor/tests/summary_projection.rs index 392568d84..3e64a23c3 100644 --- a/crates/executor/tests/summary_projection.rs +++ b/crates/executor/tests/summary_projection.rs @@ -52,7 +52,8 @@ fn raw_node(id: usize, payload: NonASAPOp) -> PhysicalASAPDAGNode { payload: PhysicalASAPOperatorPayload::NonASAP(payload), output_schema: sample_schema(), output_state: ExecutionDataState::QUERY_ROWS, - materialization: Default::default(), + lifetime: Default::default(), + parameters: None, guarantee: None, retention_ms: None, } @@ -123,7 +124,8 @@ fn summary_projection_survives_recovery() { }), output_schema: (*schema).clone(), output_state: ExecutionDataState::INGESTION_SUMMARY, - materialization: Default::default(), + lifetime: Default::default(), + parameters: None, guarantee: None, retention_ms: None, }, @@ -151,7 +153,8 @@ fn summary_projection_survives_recovery() { }), output_schema: output.clone(), output_state: ExecutionDataState::INGESTION_SUMMARY, - materialization: Default::default(), + lifetime: Default::default(), + parameters: None, guarantee: None, retention_ms: None, }, @@ -191,6 +194,7 @@ fn summary_projection_survives_recovery() { assert_eq!(*program.output_schema(1).unwrap(), output); let mut accumulator = create_planner_accumulator( &family, + None, &SummaryUpdate::column(ColumnRef::SampleValue), &GroupingStrategy::PerSubpopulationInstance, ) @@ -259,7 +263,8 @@ fn summary_merge_without_inputs_is_rejected() { }), output_schema: schema, output_state: ExecutionDataState::INGESTION_SUMMARY, - materialization: Default::default(), + lifetime: Default::default(), + parameters: None, guarantee: None, retention_ms: None, }], diff --git a/crates/executor/tests/time_series_window.rs b/crates/executor/tests/time_series_window.rs index 792a2881b..db76a9d73 100644 --- a/crates/executor/tests/time_series_window.rs +++ b/crates/executor/tests/time_series_window.rs @@ -8,12 +8,11 @@ use asap_executor::sources::InMemoryDataStore; use asap_executor::values::{Batch, Value}; use futures::{executor::block_on, StreamExt}; use planner_types::ir::operator::{Source, TimeShift}; -use planner_types::ir::physical_export::{compile_physical_asap_dag, PhysicalASAPDAG}; +use planner_types::ir::physical_export::PhysicalASAPDAG; use planner_types::ir::scalar::{CompareOpKind, ScalarValue}; use planner_types::ir::schema::{DataType, Field, Schema, PROMQL_SERIES_IDENTITY}; use planner_types::ir::{ - apply_materialization_timings, ExprSemantics, MaterializationAssignment, NodeToTimedNodeCache, - NonASAPOp, Operator, OperatorNode, Predicate, ScalarExpr, TimeRangeKind, + ExprSemantics, NonASAPOp, Operator, OperatorNode, Predicate, ScalarExpr, TimeRangeKind, }; use std::collections::BTreeMap; use std::sync::Arc; @@ -84,13 +83,7 @@ fn window_over_rows( child, })) .unwrap(); - let timed = apply_materialization_timings( - &range, - &MaterializationAssignment::default(), - &mut NodeToTimedNodeCache::new(), - ) - .unwrap(); - compile_physical_asap_dag(&timed).unwrap() + planner_types::test_support::physical_asap_dag_at_query_time(&range).unwrap() } /// `(instance, ts, value)` samples of `latency_ms` as a store's raw data. diff --git a/crates/executor/tests/univmon_execution.rs b/crates/executor/tests/univmon_execution.rs index 5533e6c31..cbc8e9128 100644 --- a/crates/executor/tests/univmon_execution.rs +++ b/crates/executor/tests/univmon_execution.rs @@ -15,26 +15,14 @@ use planner_types::ir::properties::ExecutionDataState; use planner_types::ir::scalar::ColumnRef; use planner_types::ir::schema::{DataType, Schema}; use planner_types::ir::schema::{ - Field, FieldDataType, GroupingStrategy, NonNegativeWeightProof, SketchAlgorithm, SketchKind, - SketchParams, SketchStatistic, SummaryInputExpr, SummaryUpdate, WeightDomain, + Field, FieldDataType, GroupingStrategy, NonNegativeWeightProof, SketchAlgorithm, + SketchStatistic, SummaryInputExpr, SummaryUpdate, WeightDomain, }; use planner_types::ir::ASAPOp; use std::sync::Arc; fn family() -> FieldDataType { - FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::UnivMon, - SketchParams::UnivMon { - heap_size: 64, - sketch_rows: 5, - sketch_cols: 128, - layers: 4, - distinct_failure_probability: None, - }, - ), - GroupingStrategy::default(), - ) + FieldDataType::Sketch(SketchAlgorithm::UnivMon, GroupingStrategy::default()) } fn context() -> RunContext { @@ -191,7 +179,8 @@ fn planner_unit_frequency_update_binds_as_a_value_frequency_build() -> Result<() filter: None, }), output_state: ExecutionDataState::QUERY_ROWS, - materialization: Default::default(), + lifetime: Default::default(), + parameters: None, output_schema: (*state).clone(), guarantee: None, retention_ms: None, diff --git a/crates/executor/tests/weighted_topk_binding.rs b/crates/executor/tests/weighted_topk_binding.rs index eacadcca7..4e878920a 100644 --- a/crates/executor/tests/weighted_topk_binding.rs +++ b/crates/executor/tests/weighted_topk_binding.rs @@ -1,255 +1,9 @@ //! Planner output binds directly to the shared runtime over raw counter samples. -use crate::common; -use asap_executor::sources::InMemoryDataStore; -use asap_executor::{ - physical_planner::{ - compile, - promql_rows::{series_row, with_series_identity}, - }, - runtime::{Limits, RunContext, Scope}, - values::{Batch, Value}, -}; -use common::{compile_physical_asap_dag, computation_candidates}; -use futures::{executor::block_on, StreamExt}; -use planner_types::ir::physical_export::{PhysicalASAPDAG, PhysicalASAPOperatorPayload}; -use planner_types::ir::properties::AccuracyTarget; use planner_types::ir::schema::*; -use planner_types::ir::ASAPOp; -use std::{collections::BTreeMap, rc::Rc, sync::Arc}; -// The heap candidates of `topk by(job)(2, sum by(service, job)(rate(m[1m])))` -// rank the summed per-series rates of raw counter samples bound at the -// `Scan`, both maintained at ingestion and run at query time. -#[test] -fn planner_weighted_topk_binds_at_either_deployment_phase() { - assert_weighted_binding(SketchAlgorithm::CmsWithHeap); - assert_weighted_binding(SketchAlgorithm::CountSketchWithHeap); -} - -/// Whether `dag` builds a heap sketch of `algorithm`. -fn builds(dag: &PhysicalASAPDAG, algorithm: &SketchAlgorithm) -> bool { - dag.nodes.iter().any(|node| { - matches!(&node.payload, - PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), - .. - }) if kind.algorithm() == algorithm) - }) -} - -/// The logical candidate of `root` at ε = 0.1 that builds a heap sketch of -/// `algorithm`. -fn heap_candidate( - root: Rc, - algorithm: &SketchAlgorithm, -) -> Rc { - computation_candidates(root) - .into_iter() - .find(|candidate| builds(&compile_physical_asap_dag(candidate).unwrap(), algorithm)) - .unwrap_or_else(|| panic!("missing {algorithm:?} candidate")) -} -/// `plan` as Stage 3 hands it to the executor when the data declares `m` a -/// counter: its rate weights' open requirement confirmed non-negative. -fn confirmed_as_counter( - plan: Rc, -) -> Rc { - use planner_types::ir::QueryRoot; - use planner_types::workload::{PrometheusMetricType, SourceFacts}; - let data = DataWorkload { - sources: [( - "m".to_string(), - SourceFacts { - metric_type: Some(PrometheusMetricType::Counter), - ..SourceFacts::default() - }, - )] - .into(), - ..DataWorkload::default() - }; - match planner_types::workload::bounds::with_confirmed_weights( - &[QueryRoot::Operator(plan)], - &data, - ) - .remove(0) - { - QueryRoot::Operator(node) => node, - QueryRoot::Scalar(_) => unreachable!("an operator root"), - } -} - -fn assert_weighted_binding(algorithm: SketchAlgorithm) { - let root = lower_promql( - "topk by(job)(2, sum by(service, job)(rate(m[1m])))", - AccuracyTarget::max_error(0.1), - ) - .unwrap(); - // The per-series rate reads each raw series by its full identity. - let root = with_series_identity(&root).unwrap(); - let plan = confirmed_as_counter(heap_candidate(root, &algorithm)); - let dag = compile_physical_asap_dag(&plan).unwrap(); - let scan = dag - .nodes - .iter() - .find_map(|node| match &node.payload { - PhysicalASAPOperatorPayload::NonASAP(planner_types::ir::NonASAPOp::Scan { - source, - .. - }) => Some((source.clone(), Arc::new(node.output_schema.clone()))), - _ => None, - }) - .unwrap(); - let (source, samples) = scan; - // One counter series per rate: two samples 32 s apart in `(0, 60 s]`, - // the first an increase above zero, so `rate(m[1m])` extrapolates to the - // whole minute and is exactly `increase / 32 s`. The two `auth` series - // differ in `instance`. - let rows = [ - ("auth", "api", "a", 0.125), - ("auth", "api", "b", 0.25), - ("checkout", "api", "a", 0.3125), - ("search", "api", "a", 0.0625), - ("ingest", "batch", "a", 100.), - ("export", "batch", "a", 80.), - ("cleanup", "batch", "a", 20.), - ] - .into_iter() - .flat_map(|(service, job, instance, rate)| { - let labels = BTreeMap::from([ - ("__name__".to_string(), "m".to_string()), - ("instance".to_string(), instance.to_string()), - ("job".to_string(), job.to_string()), - ("service".to_string(), service.to_string()), - ]); - let increase = rate * 32.; - [(14_000, increase), (46_000, 2. * increase)] - .map(|(ts, value)| series_row(&samples, &labels, ts, value).unwrap()) - }) - .collect(); - let batch = Batch::try_new(samples.clone(), rows).unwrap(); - // The summaries `MatAtIngestion` run as data arrives; `NotMaterialized` - // (the default) runs everything when the query does. - for (placed, scope) in [ - ( - continuously_maintained_dag(&plan), - Scope::Ingestion { - window_start_ms: 0, - window_end_ms: 60_000, - }, - ), - ( - dag.clone(), - Scope::Query { - evaluation_time_ms: 60_000, - }, - ), - ] { - let compiled = compile(&placed, &[], &[dag.roots[0] as u64]).unwrap(); - let store = InMemoryDataStore::default() - .with_raw_data(source.clone(), samples.clone(), vec![batch.clone()]) - .unwrap(); - let physical_dag = compiled.instantiate(Arc::new(store)).unwrap(); - let context = RunContext::new(scope, Limits::default()).unwrap(); - let output = block_on(async { - let mut output = Vec::new(); - let mut stream = physical_dag - .execute(&[dag.roots[0] as u64], context) - .unwrap() - .remove(0); - while let Some(batch) = stream.next().await { - output.extend(batch.unwrap().to_rows().iter().cloned()); - } - output - }); - // Two items per `job`: the outer `by(job)` partitions the ranking. - // Each row is `[job, topk_2, service]`: the top-k's columns, then - // the ranked item's label. - let schema = &dag - .nodes - .iter() - .find(|node| node.id == dag.roots[0]) - .unwrap() - .output_schema; - let column = |name: &str| schema.fields.iter().position(|f| f.name == name).unwrap(); - let (job, value) = (column("job"), column("topk_2")); - assert_eq!(column("service"), 2); - let mut ranked = output - .iter() - .map(|row| match (&row[job], &row[value]) { - (Value::Utf8(job), Value::Float64(score)) => (job.to_string(), *score), - other => panic!("unexpected (job, value) {other:?}"), - }) - .collect::>(); - ranked.sort_by(|a, b| a.0.cmp(&b.0).then(a.1.total_cmp(&b.1))); - let expected = [ - ("api", 0.3125), - ("api", 0.375), - ("batch", 80.), - ("batch", 100.), - ]; - assert_eq!( - ranked, - expected.map(|(job, score)| (job.to_string(), score)), - "{}", - if placed == dag { - "query time" - } else { - "ingestion time" - } - ); - } -} - -use asap_frontend_promql::lower_promql_query_workload; -use planner_types::workload::{ - BatchEntry, DataWorkload, DurationMs, PlanningWorkload, Predictability, Query, QueryLanguage, - QueryRequirements, QueryWorkload, -}; -pub fn lower_promql( - query: &str, - accuracy: AccuracyTarget, -) -> Result, asap_frontend_promql::PromqlError> { - let workload = PlanningWorkload { - query_workload: QueryWorkload { - language: QueryLanguage::PromQL, - query_batch: Some(vec![BatchEntry { - query: Query(query.into()), - requirements: QueryRequirements { - accuracy, - ..Default::default() - }, - predictability: Predictability::Unknown, - invocations: 1, - execute_at: None, - range_evaluation: None, - }]), - repeating_queries: None, - }, - data_workload: Some(DataWorkload::sampled_every( - asap_frontend_promql::promql_metric_names(query).unwrap_or_default(), - DurationMs(1_000), - )), - }; - Ok(lower_promql_query_workload(&workload)? - .remove(0) - .as_operator() - .cloned() - .expect("operator root")) -} - -// The old untyped heap updater must not silently round a Planner rate update. #[test] fn rate_updates_cannot_enter_integer_heap_factory() { - let family = FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::CmsWithHeap, - SketchParams::CmsWithHeap { - width: 272, - depth: 5, - heap_size: 100, - }, - ), - Default::default(), - ); + let family = FieldDataType::Sketch(SketchAlgorithm::CmsWithHeap, Default::default()); let input = SummaryUpdate { item: Some(SummaryInputExpr::Column( planner_types::ir::scalar::ColumnRef::Named("service".into()), @@ -261,43 +15,9 @@ fn rate_updates_cannot_enter_integer_heap_factory() { }; assert!(asap_executor::factory::create_planner_accumulator( &family, + None, &input, &Default::default() ) .is_err()); } - -/// `candidate` with every `SummaryAgg` `MatAtIngestion`, the materialization -/// a continuously served query would get: each summary and every input it -/// consumes run at ingestion time, the rest at query time. The candidate pins -/// its finalize boundary to query time, which would put the summaries above -/// it upstream of a query-time node, so the pin is lifted first. -fn continuously_maintained_dag(candidate: &Rc) -> PhysicalASAPDAG { - use planner_types::ir::{ - apply_materialization_timings, MaterializationAssignment, NodeToTimedNodeCache, - Operator as IrOperator, OperatorNode, - }; - use planner_types::physical::{Materialization, Medium}; - fn unpinned(node: &Rc) -> Rc { - let mut copy = (**node).clone(); - copy.operator = node.operator.map_children(unpinned); - if matches!( - copy.operator, - IrOperator::ASAP(ASAPOp::FinalizeExactAccumulator { .. }) - ) { - copy.timing = None; - } - Rc::new(copy) - } - let candidate = unpinned(candidate); - let mut assignment = MaterializationAssignment::default(); - for node in OperatorNode::reachable(&candidate) { - if matches!(node.operator, IrOperator::ASAP(ASAPOp::SummaryAgg { .. })) { - assignment.set(&node, Materialization::MatAtIngestion(Medium::Memory)); - } - } - let timed = - apply_materialization_timings(&candidate, &assignment, &mut NodeToTimedNodeCache::new()) - .unwrap(); - planner_types::ir::physical_export::compile_physical_asap_dag(&timed).unwrap() -} diff --git a/crates/frontend-common/src/canonicalize.rs b/crates/frontend-common/src/canonicalize.rs index 707acd89e..43495769f 100644 --- a/crates/frontend-common/src/canonicalize.rs +++ b/crates/frontend-common/src/canonicalize.rs @@ -1,7 +1,7 @@ //! Post-lowering canonicalization of the operator DAG. //! //! Rewrites equivalent spellings of a query into one shape, so that common -//! sub-DAG sharing (Stage 0 hash-consing, `cse` in `asap_logical_asap_optimization`) can match +//! sub-DAG sharing (hash-consing, `Memo::find_or_add_group`) can match //! them by structural equality, whichever front end produced them. //! //! ## Heavy-hitter promotion diff --git a/crates/frontend-promql/tests/count_planning.rs b/crates/frontend-promql/tests/count_planning.rs index b208f7ae9..09a3b2797 100644 --- a/crates/frontend-promql/tests/count_planning.rs +++ b/crates/frontend-promql/tests/count_planning.rs @@ -90,16 +90,10 @@ fn frequency_count_candidates_use_unit_weights() { continue; }; assert!( - !matches!( - kind.algorithm(), - SketchAlgorithm::Hll | SketchAlgorithm::CountSketch - ), + !matches!(kind, SketchAlgorithm::Hll | SketchAlgorithm::CountSketch), "{query}: {node:?}" ); - if !matches!( - kind.algorithm(), - SketchAlgorithm::Cms | SketchAlgorithm::UnivMon - ) { + if !matches!(kind, SketchAlgorithm::Cms | SketchAlgorithm::UnivMon) { continue; } let dag = physical_asap_dag(node); @@ -110,7 +104,7 @@ fn frequency_count_candidates_use_unit_weights() { )), "post-ASAP DAG must preserve the count update contract" ); - algorithms.push(*kind.algorithm()); + algorithms.push(*kind); assert!(input.item.is_some(), "frequency keys must be explicit"); assert_eq!( input.weight, @@ -242,7 +236,7 @@ fn cms_count_updates_total_ten_for_zero_positive_and_negative_samples() { .any(|node| { matches!(&node.payload, PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryAgg { family: FieldDataType::Sketch(kind, _), .. }) - if kind.algorithm() == &SketchAlgorithm::Cms) + if kind == &SketchAlgorithm::Cms) }) .then_some(dag) }) diff --git a/crates/frontend-promql/tests/support.rs b/crates/frontend-promql/tests/support.rs index 19834b75d..d6a3912e1 100644 --- a/crates/frontend-promql/tests/support.rs +++ b/crates/frontend-promql/tests/support.rs @@ -76,24 +76,14 @@ pub fn promql_scalar(node: &ScalarExpr) -> Option { } } -/// Time `root` under the default materialization assignment (every summary +/// Time `root` under the default physical properties assignment (every summary /// at query time) and export the physical DAG — export needs every node /// timed first. #[allow(dead_code)] pub fn physical_asap_dag( root: &Rc, ) -> asap_types::ir::physical_export::PhysicalASAPDAG { - use asap_types::ir::{ - apply_materialization_timings, MaterializationAssignment, NodeToTimedNodeCache, - }; - let timed = apply_materialization_timings( - root, - &MaterializationAssignment::default(), - &mut NodeToTimedNodeCache::new(), - ) - .expect("default materialization timings"); - asap_types::ir::physical_export::compile_physical_asap_dag(&timed) - .expect("post-ASAP DAG export") + asap_types::test_support::physical_asap_dag_at_query_time(root).expect("post-ASAP DAG export") } #[allow(dead_code)] @@ -159,9 +149,7 @@ fn local_computation_candidates(root: Rc) -> Result) -> Option> { @@ -17,106 +15,10 @@ fn univmon_estimate(root: &Rc) -> Option> { return false; }; matches!(&summary_input.operator, Operator::ASAP(ASAPOp::SummaryAgg { family: FieldDataType::Sketch(kind, _), .. }) - if kind.algorithm() == &SketchAlgorithm::UnivMon) + if kind == &SketchAlgorithm::UnivMon) }) } -/// The logical candidate of `query` that evaluates a UnivMon state. -fn candidate(query: &str, accuracy: AccuracyTarget) -> Rc { - let root = lower_promql(query, accuracy.clone()).unwrap(); - logical_candidates(root) - .into_iter() - .find_map(|node| univmon_estimate(&node)) - .expect("UnivMon candidate") -} - -#[test] -fn four_evaluations_size_their_value_frequency_states_and_keep_honest_guarantees() { - // Pass 1 sizes UnivMon per readout: the distinct count and L2 get their - // own shapes, while the total and entropy keep the baseline shape and so - // share one state. Sharing across readouts is the summary-capability - // rule's job. - let accuracy = AccuracyTarget::max_error(0.02); - let roots: Vec<_> = [ - ("distinct_over_time(m[5m])", accuracy.clone()), - ("count_over_time(m[5m])", accuracy.clone()), - ("l2_over_time(m[5m])", accuracy.clone()), - ("entropy_over_time(m[5m])", accuracy), - ] - .into_iter() - .enumerate() - .map(|(id, (query, accuracy))| (id, candidate(query, accuracy))) - .collect(); - let roots = share_common_sub_dags(roots); - let state = |index: usize| { - let Operator::ASAP(ASAPOp::SummaryEstimate { summary_input, .. }) = - &roots[index].1.operator - else { - panic!() - }; - Rc::clone(summary_input) - }; - assert!(Rc::ptr_eq(&state(1), &state(3)), "total and entropy share"); - for (a, b) in [(0, 1), (0, 2), (1, 2)] { - assert!( - !Rc::ptr_eq(&state(a), &state(b)), - "{a} and {b} are sized apart" - ); - } - for (index, root) in &roots { - let Operator::ASAP(ASAPOp::SummaryEstimate { - summary_input, - query, - }) = &root.operator - else { - panic!() - }; - let Operator::ASAP(ASAPOp::SummaryAgg { input, family, .. }) = &summary_input.operator - else { - panic!() - }; - assert!(matches!(input.item, Some(SummaryInputExpr::Column(_)))); - assert_eq!(input.weight, SummaryInputExpr::Constant(1.0)); - let guarantee = root.guarantee.as_ref(); - match index { - // The distinct count's failure probability waits for the - // source's distinct bound, which Stage 1 does not read. - 0 => { - let guarantee = guarantee.expect("a distinct-count guarantee"); - assert_eq!(guarantee.spec.accuracy_metric, AccuracyMetric::Relative); - assert_eq!( - guarantee.spec.error.evaluate().map(|e| e <= 0.02), - Some(true) - ); - assert_eq!(guarantee.spec.failure_probability.evaluate(), None); - assert!(local_guarantee(family, query).is_none()); - } - 1 => { - assert!(matches!( - query, - SketchStatistic::PointCount { value: None, .. } - )); - assert!(guarantee.is_some_and(|g| g.is_exact())); - } - 2 => { - let guarantee = guarantee.expect("an L2 guarantee"); - assert_eq!(guarantee.spec.accuracy_metric, AccuracyMetric::Relative); - assert!(satisfies( - guarantee, - &AccuracyTarget::max_error(0.02).accuracy_metric(AccuracyMetric::Relative) - )); - assert!(guarantee.spec.failure_probability.evaluate().unwrap() <= 0.01); - } - // Entropy has no certified bound, so composition certifies nothing. - _ => { - assert!(guarantee.is_none()); - assert!(local_guarantee(family, query).is_none()); - } - } - physical_asap_dag(root); - } -} - #[test] fn uncalibrated_frequency_evaluations_do_not_bypass_accuracy_targets() { // An unmeasured heuristic remains inspectable but is never certified or diff --git a/crates/frontend-sql/src/sql/collection_udfs.rs b/crates/frontend-sql/src/sql/collection_udfs.rs index 17711eb83..c4892190c 100644 --- a/crates/frontend-sql/src/sql/collection_udfs.rs +++ b/crates/frontend-sql/src/sql/collection_udfs.rs @@ -118,7 +118,7 @@ impl CollectionUdf { Ok(ScalarExpr::Column(index)) } }) - .collect::>>()?; + .collect::>>()?; match self.function { CollectionFunction::Element => element_access_type(&args, &schema), CollectionFunction::StructField => struct_field_type(&args, &schema), diff --git a/crates/frontend-sql/tests/frequency_l2.rs b/crates/frontend-sql/tests/frequency_l2.rs index dbf709609..2cdbead77 100644 --- a/crates/frontend-sql/tests/frequency_l2.rs +++ b/crates/frontend-sql/tests/frequency_l2.rs @@ -131,9 +131,7 @@ async fn stage1_offers_exact_and_summary_l2_candidates() { .await .unwrap(); let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(root), - ]), + &asap_logical_asap_optimization::memo_of_queries(&[QueryRoot::Operator(root)]).unwrap(), ) .unwrap(); let l2 = inventory diff --git a/crates/frontend-sql/tests/pearson_corr.rs b/crates/frontend-sql/tests/pearson_corr.rs index c6f31079a..167d03fcb 100644 --- a/crates/frontend-sql/tests/pearson_corr.rs +++ b/crates/frontend-sql/tests/pearson_corr.rs @@ -5,10 +5,7 @@ use asap_frontend_sql::{lower_sql, SqlCatalog}; use asap_types::ir::operator::AggIntent; use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::schema::{DataType, Field, Schema}; -use asap_types::ir::{ - apply_materialization_timings, physical_export::compile_physical_asap_dag, - MaterializationAssignment, NodeToTimedNodeCache, NonASAPOp, OperatorNode, ScalarExpr, -}; +use asap_types::ir::{NonASAPOp, OperatorNode, ScalarExpr}; fn catalog() -> SqlCatalog { let schema = Schema::new(vec![ @@ -143,47 +140,3 @@ async fn corr_filter_is_a_measure_filter() { filters(&query) ); } - -// The plan the planner pipeline selects retains the complete typed query and -// compiles to a physical ASAP DAG. -#[tokio::test] -async fn corr_survives_exact_plan_compilation() { - let query = lower("SELECT corr(x, y) AS r FROM a").await; - // `a` holds 60,000 rows: the query scans all of them. - let data: asap_types::workload::DataWorkload = serde_json::from_value( - serde_json::json!({"arrival": "at_rest", "sources": {"a": {"rows_held": 60000}}}), - ) - .unwrap(); - let run = asap_planner::run_planning( - vec![asap_types::ir::QueryRoot::Operator(query.clone())], - &[asap_types::workload::QueryWorkloadEntry { - query: Default::default(), - requirements: asap_types::workload::QueryRequirements { - accuracy: asap_types::ir::properties::AccuracyTarget::exact(), - max_latency_ms: None, - }, - predictability: Default::default(), - recurrence: Default::default(), - range_evaluation: None, - }], - &data, - asap_planner::PlanningModels::builtin(), - asap_planner::PlanSearchSettings::default(), - ) - .unwrap(); - let [asap_types::ir::QueryRoot::Operator(plan)] = run.selected_plan.as_deref().unwrap() else { - panic!("one operator root"); - }; - assert!(plan.guarantee.as_ref().unwrap().is_exact()); - // Pass 1 has no candidate for `corr`: the plan is the query's own - // operator DAG, no ASAP node added. - assert!(!plan.contains_asap(), "expected the exact plan"); - assert_eq!(aggregate(plan).0, aggregate(&query).0); - let timed = apply_materialization_timings( - plan, - &MaterializationAssignment::default(), - &mut NodeToTimedNodeCache::new(), - ) - .unwrap(); - compile_physical_asap_dag(&timed).unwrap(); -} diff --git a/crates/integration-tests/Cargo.toml b/crates/integration-tests/Cargo.toml index 055c031b1..c8ee40b02 100644 --- a/crates/integration-tests/Cargo.toml +++ b/crates/integration-tests/Cargo.toml @@ -10,22 +10,22 @@ asap-types = { path = "../types", features = ["test-support"] } asap-frontend-promql = { path = "../frontend-promql" } asap-frontend-sql = { path = "../frontend-sql" } asap-logical-asap-optimization = { path = "../logical-asap-optimization", features = ["test-support"] } -asap-physical-asap-optimization = { path = "../physical-asap-optimization", features = ["reference-pricer"] } +asap-physical-asap-optimization = { path = "../physical-asap-optimization" } asap-accuracy = { path = "../accuracy" } asap-cost = { path = "../cost" } -asap-plan-selection = { path = "../plan-selection", features = ["reference-pricer"] } +asap-plan-selection = { path = "../plan-selection" } # The fixtures plan through the facade and the developer tools' reference # enumeration. -asap-planner = { path = "../planner", features = ["test-support"] } +asap-planner = { path = "../planner" } asap-devtools = { path = "../devtools" } [dev-dependencies] asap-types = { path = "../types", features = ["test-support"] } asap-accuracy = { path = "../accuracy", features = ["test-support"] } -asap-physical-asap-optimization = { path = "../physical-asap-optimization", features = ["test-support"] } +asap-physical-asap-optimization = { path = "../physical-asap-optimization" } asap-devtools = { path = "../devtools" } # The tests plan PromQL and SQL workloads through the facade. -asap-planner = { path = "../planner", features = ["test-support"] } +asap-planner = { path = "../planner" } asap_sketchlib = { workspace = true } serde_json = "1" tokio = { version = "1", features = ["rt", "macros", "rt-multi-thread"] } diff --git a/crates/integration-tests/src/lib.rs b/crates/integration-tests/src/lib.rs index 49534547c..d145c8228 100644 --- a/crates/integration-tests/src/lib.rs +++ b/crates/integration-tests/src/lib.rs @@ -95,54 +95,21 @@ pub mod fixtures { } } -/// Timing and physical ASAP DAG export helpers for planned candidates. +/// Physical ASAP DAG export helpers for planned candidates. pub mod physical_export { - use asap_types::ir::physical_export::{compile_physical_asap_dag, PhysicalASAPDAG}; - use asap_types::ir::{ - apply_materialization_timings, MaterializationAssignment, NodeToTimedNodeCache, - OperatorNode, - }; - use asap_types::physical::{Materialization, Medium}; + use asap_types::ir::physical_export::PhysicalASAPDAG; + use asap_types::ir::OperatorNode; use std::rc::Rc; - /// Time `root` under the default assignment (every sub-DAG - /// `NotMaterialized`, so every summary computed at query time). Returns the timed copy; read `node.timing` on it. - pub fn timed(root: &Rc) -> Rc { - timed_with(root, &MaterializationAssignment::default()) - } - - /// Time `root` with every sub-DAG `MatAtIngestion` where the DAG allows it. - pub fn at_ingestion(root: &Rc) -> Rc { - timed_with( - root, - &MaterializationAssignment::with_default(Materialization::MatAtIngestion( - Medium::Memory, - )), - ) - } - - fn timed_with( - root: &Rc, - assignment: &MaterializationAssignment, - ) -> Rc { - apply_materialization_timings(root, assignment, &mut NodeToTimedNodeCache::new()) - .expect("materialization timing failed") - } - - /// Time `root` (default assignment), then export the physical DAG. + /// Export the physical DAG of `root`. pub fn physical_asap_dag(root: &Rc) -> PhysicalASAPDAG { - compile_physical_asap_dag(&timed(root)).expect("physical ASAP DAG export failed") - } - - /// Time `root` with every sub-DAG `MatAtIngestion` where the DAG allows - /// it, then export the physical DAG. - pub fn at_ingestion_physical_asap_dag(root: &Rc) -> PhysicalASAPDAG { - compile_physical_asap_dag(&at_ingestion(root)).expect("physical ASAP DAG export failed") + asap_types::test_support::physical_asap_dag_at_query_time(root) + .expect("physical ASAP DAG export failed") } } /// Plans and candidates from the planner pipeline (the facade's -/// `run_planning`, with the developer tools' reference enumeration) for one +/// `run_planning`) for one /// query, with the built-in models. pub mod planning { use asap_types::ir::properties::AccuracyTarget; @@ -184,8 +151,7 @@ pub mod planning { } } - /// Every planning stage's output for the single query `root`, with - /// the reference enumeration of its memo. + /// Every planning stage's output for the single query `root`. pub fn run_all_stages( root: Rc, accuracy: AccuracyTarget, @@ -213,7 +179,6 @@ pub mod planning { range_evaluation: None, }], &data, - None, models, ) .expect("Pass 1 accepts the query") @@ -235,9 +200,7 @@ pub mod planning { compose_logical_candidate, generate_per_computation_candidates, }; let local = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(root), - ]), + &asap_logical_asap_optimization::memo_of_queries(&[QueryRoot::Operator(root)]).unwrap(), ) .unwrap(); let mut choices = std::collections::BTreeSet::from([vec![0; local.targets.len()]]); @@ -273,9 +236,7 @@ pub mod planning { }; use asap_types::stages::HowToComputeAggregateMeasure; let local = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(root), - ]), + &asap_logical_asap_optimization::memo_of_queries(&[QueryRoot::Operator(root)]).unwrap(), ) .unwrap(); let choices = local.targets.iter().map(|target| target.ways_to_compute.iter() @@ -334,99 +295,6 @@ pub mod planning { }) .collect() } - /// Pass 1, local tumbling windows for `entries`' recurrences and every Pass 2 rule - /// on the seed grids (no query pruning). Rule tests read a group's - /// candidates by `OriginOfCandidateForMemoGroup`. - pub fn expanded_memo( - roots: Vec, - entries: &[asap_types::workload::QueryWorkloadEntry], - ) -> asap_types::stages::Memo { - use asap_logical_asap_optimization::test_support::generate_per_computation_candidates; - use asap_logical_asap_optimization::test_support::{ - add_local_window_composition, try_populate_pass1, - }; - let initial = asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&roots); - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&roots), - ) - .unwrap(); - let mut memo = try_populate_pass1(initial, &inventory).unwrap(); - add_local_window_composition(&mut memo, entries); - asap_logical_asap_optimization::test_support::sharing_rules_and_shared_state_groups(memo) - .unwrap() - } - - /// One plan of `memo`'s first `queries` queries (`memo::enumeration::build_logical_asap_dags_from_chosen_candidates`): - /// each query's first root group's DAG, every group `choice` does not - /// name implemented by its first candidate. - pub fn compose_memo_plan( - memo: &asap_types::stages::Memo, - queries: usize, - choice: &std::collections::BTreeMap, - ) -> Vec> { - use asap_logical_asap_optimization::build_logical_asap_dags_from_chosen_candidates; - use asap_types::stages::MemoGroupId; - let choice = choice - .iter() - .map(|(&group, &index)| (MemoGroupId(group as u32), index)) - .collect(); - let roots = build_logical_asap_dags_from_chosen_candidates(memo, &choice) - .expect("the choice composes"); - (0..queries) - .map(|query| { - let (_, root) = roots - .iter() - .find(|(group, _)| memo.query_roots[query].operator_refs().contains(&group)) - .expect("the query's root group"); - Rc::clone(root) - }) - .collect() - } - - /// Stage 2 and plan selection over fixed workload plans, without a timed - /// search: each `(name, roots)` is implemented by Stage 2 (plan ids start - /// with `name`), and all are priced together. - pub fn select_among( - variants: &[(&str, Vec)], - entries: &[asap_types::workload::QueryWorkloadEntry], - data: &DataWorkload, - ) -> ( - Vec, - asap_plan_selection::reference_pricer::WholePlanSelection, - ) { - let models = asap_planner::PlanningModels::builtin(); - let mut physical = Vec::new(); - for (name, roots) in variants { - // Composed nodes lose their guarantees; recompose them as - // `run_explicit_candidates` does. - let roots = asap_accuracy::propagation::recompose_guarantees(roots) - .unwrap_or_else(|error| panic!("{name} composes guarantees: {error}")); - // Composed outside the memo: sized as Pass 1 sizes a candidate. - let targets: Vec<_> = entries - .iter() - .map(|entry| entry.requirements.accuracy.clone()) - .collect(); - let roots = - asap_accuracy::sizing::with_summaries_sized_for_targets(&roots, &targets, data) - .unwrap_or_else(|error| panic!("{name} sizes its summaries: {error}")); - let mut plans = asap_physical_asap_optimization::optimize_physical_asap( - name, - &roots, - entries, - data, - models.deployment_capabilities.query_time_results_retained, - ) - .unwrap_or_else(|error| panic!("Stage 2 implements {name}: {error}")); - for (index, plan) in plans.iter_mut().enumerate() { - plan.id = format!("{name}-{index}"); - } - physical.extend(plans); - } - let selection = - asap_devtools::memo_pricing::reference_select(&physical, entries, data, models) - .unwrap(); - (physical, selection) - } /// Whether `node` builds a summary. fn is_build(node: &Rc) -> bool { @@ -448,12 +316,13 @@ pub mod planning { keep: impl Fn(&asap_types::stages::CandidateForMemoGroup) -> bool, ) -> Vec<(usize, usize)> { let mut found: Vec<_> = memo - .candidates + .groups() .iter() .enumerate() - .filter(|(group, _)| memo.groups[*group].reached_by_queries() == [q]) - .flat_map(|(group, candidates)| { - candidates + .filter(|(_, group)| group.reached_by_queries() == [q]) + .flat_map(|(group, memo_group)| { + memo_group + .candidates .iter() .enumerate() .filter(|(_, c)| keep(c)) @@ -463,9 +332,9 @@ pub mod planning { .collect(); // Group indices don't give the order: Pass 2 appends groups at the // end. Depth from the root does. - let mut depth = vec![usize::MAX; memo.groups.len()]; + let mut depth = vec![usize::MAX; memo.groups().len()]; let mut pending: std::collections::VecDeque<_> = memo - .groups + .groups() .iter() .enumerate() .filter(|(group, _)| { @@ -481,9 +350,10 @@ pub mod planning { } depth[group] = d; pending.extend( - memo.groups[group] - .child_groups - .iter() + memo.groups()[group] + .expression + .children() + .into_iter() .map(|child| (child.0 as usize, d + 1)), ); } @@ -491,105 +361,6 @@ pub mod planning { found } - /// The selected plan's roots among fixed plans of `memo`'s queries, and - /// the selection's lowest prices of `separate` and of the `shared` plans: - /// - `separate{k}`: each query reads its own summary build, its topmost - /// own group's `k`-th Pass 1 candidate that builds one (the last when - /// it has fewer); - /// - `shared{g}`: every query reads the summary state group `g`, its own - /// estimate or a regrouping merge of it, when every query can. - pub fn select_rollup( - memo: &asap_types::stages::Memo, - queries: usize, - entries: &[asap_types::workload::QueryWorkloadEntry], - data: &DataWorkload, - ) -> (Vec>, f64, f64) { - let builds = |node: &Rc| OperatorNode::reachable(node).iter().any(is_build); - let own_builds: Vec> = (0..queries) - .map(|q| { - let found = own_choices(memo, q, |c| { - c.origin == asap_types::stages::OriginOfCandidateForMemoGroup::Pass1 - && builds(&c.root_node) - }); - let top = found - .first() - .unwrap_or_else(|| panic!("Q{} has a summary candidate", q + 1)) - .0; - found - .into_iter() - .filter(|(group, _)| *group == top) - .collect() - }) - .collect(); - let separate_plans = own_builds.iter().map(Vec::len).max().unwrap_or(0); - let mut plans: Vec<(String, Vec>)> = (0..separate_plans) - .map(|k| { - let choice = own_builds - .iter() - .map(|found| found[k.min(found.len() - 1)]) - .collect(); - ( - format!("separate{k}"), - compose_memo_plan(memo, queries, &choice), - ) - }) - .collect(); - for (state, group) in memo.groups.iter().enumerate() { - if !is_build(&group.logical_node) || group.reached_by_queries().len() < queries { - continue; - } - let reading: Option> = (0..queries) - .map(|q| { - own_choices(memo, q, |c| { - c.input_groups.iter().any(|child| child.0 as usize == state) - }) - .into_iter() - .next() - }) - .collect(); - if let Some(choice) = reading { - plans.push(( - format!("shared{state}"), - compose_memo_plan(memo, queries, &choice), - )); - } - } - assert!( - plans.len() > separate_plans, - "a plan in which every query reads one build" - ); - let named: Vec<(&str, Vec)> = plans - .iter() - .map(|(name, roots)| { - ( - name.as_str(), - roots.iter().cloned().map(QueryRoot::Operator).collect(), - ) - }) - .collect(); - let (physical, selection) = select_among(&named, entries, data); - let selected = physical - .iter() - .find(|plan| plan.id == selection.selected) - .expect("the selected plan"); - let roots = plans - .iter() - .find(|(name, _)| *name == selected.from_logical) - .map(|(_, roots)| roots.clone()) - .unwrap(); - let cheapest = |plans: &[(String, Vec>)]| { - plans - .iter() - .map(|(name, _)| cheapest_of(&selection, name)) - .fold(f64::INFINITY, f64::min) - }; - let separate = cheapest(&plans[..separate_plans]); - // Every separate plan rejected would make `shared < separate` hold - // trivially. - assert!(separate.is_finite(), "a valid plan with a build per query"); - (roots, separate, cheapest(&plans[separate_plans..])) - } - /// The summary builds and rollups (regrouping merges) `roots` read, each /// once. pub fn builds_and_rollups(roots: &[Rc]) -> (usize, usize) { @@ -609,18 +380,4 @@ pub mod planning { } (builds, rollups) } - - /// The lowest price per second among `selection`'s plans whose id starts - /// with `name`. - pub fn cheapest_of( - selection: &asap_plan_selection::reference_pricer::WholePlanSelection, - name: &str, - ) -> f64 { - selection - .costs - .iter() - .filter(|(id, _)| id.starts_with(&format!("{name}-"))) - .filter_map(|(_, cost)| cost.cost_per_second) - .fold(f64::INFINITY, f64::min) - } } diff --git a/crates/integration-tests/tests/declared_group_counts.rs b/crates/integration-tests/tests/declared_group_counts.rs deleted file mode 100644 index 6cad286f1..000000000 --- a/crates/integration-tests/tests/declared_group_counts.rs +++ /dev/null @@ -1,118 +0,0 @@ -//! The groups plan selection prices a `by (...)` at, from each label's -//! declared distinct-value count (`SourceFacts::distinct_values_per_label`), -//! over the workload of #840: two per-series quantiles and two grouped -//! quantiles of one gauge with 10,000 series. - -use crate::planner_layering_common; - -use std::collections::BTreeSet; - -use asap_types::workload::PlanningWorkload; -use planner_layering_common::*; - -/// #840's workload (Q1–Q4) with `distinct_values_per_label` declared for -/// `data`. -fn workload(distinct_values_per_label: serde_json::Value) -> PlanningWorkload { - let query = |text: &str| { - serde_json::json!({ - "query": text, "demand": {"fixed_interval": 10000}, - "requirements": {"accuracy": {"max_error": 0.01, "failure_probability": 0.001}}, - "predictability": {"predictable": {"known_at": null}} - }) - }; - serde_json::from_value(serde_json::json!({ - "query_workload": {"language": "promql", "query_batch": null, "repeating_queries": [ - query("quantile_over_time(0.5, data[1m])"), - query("quantile_over_time(0.9, data[5m])"), - query("quantile by (host, job) (0.5, data)"), - query("quantile by (job) (0.5, data)"), - ]}, - "data_workload": {"arrival": {"ingesting": {"rows_per_second": 666.67}}, - "sources": {"data": {"series_count": 10000, "metric_type": "gauge", - "sample_interval": 15000, - "distinct_values_per_label": distinct_values_per_label}}} - })) - .expect("workload") -} - -/// Isolate the declared-cardinality pricing contract from search order. -fn priced_exact(workload: &PlanningWorkload) -> Run { - let roots = lower_promql(workload); - run_explicit_candidates( - workload, - roots.clone(), - vec![roots], - asap_planner::PlanningModels::builtin(), - ) -} - -/// The group counts every priced plan gives the builds and aggregates read -/// only by `query`, from their cost details (`... into N states`, `... into -/// N groups`). -fn priced_groups(run: &Run, query: usize) -> BTreeSet { - let mut groups = BTreeSet::new(); - for physical in &run.physical { - let Some(cost) = run.selection.costs.get(&physical.id) else { - continue; - }; - for node in &physical.dag().nodes { - if readers(physical.dag(), &physical.query_roots, node.id) != BTreeSet::from([query]) { - continue; - } - let detail = &cost.per_node[&node.id].detail; - let words: Vec<&str> = detail.split([' ', ':', ';']).collect(); - for window in words.windows(3) { - if window[0] == "into" && matches!(window[2], "states" | "groups") { - groups.insert(window[1].parse().expect("a group count")); - } - } - } - } - groups -} - -const Q3_BY_HOST_JOB: usize = 2; -const Q4_BY_JOB: usize = 3; - -/// With `host` and `job` declared, `by (host, job)` is priced at 1,000 × 10 -/// groups, at most the 10,000 series, and `by (job)` at 10. -#[test] -fn a_grouping_is_priced_at_the_product_of_its_labels_counts() { - let run = priced_exact(&workload(serde_json::json!({"host": 1000, "job": 10}))); - assert_eq!( - priced_groups(&run, Q3_BY_HOST_JOB), - BTreeSet::from([10_000]) - ); - assert_eq!(priced_groups(&run, Q4_BY_JOB), BTreeSet::from([10])); -} - -/// A product above `series_count` is capped at it: each group holds at -/// least one series. -#[test] -fn a_grouping_has_at_most_one_group_per_series() { - let run = priced_exact(&workload(serde_json::json!({"host": 5000, "job": 10}))); - assert_eq!( - priced_groups(&run, Q3_BY_HOST_JOB), - BTreeSet::from([10_000]) - ); -} - -/// A grouping with an undeclared label gets no invented count: each of the -/// 10,000 series is in one group, and each of the 10,000 samples the -/// instant selector reads in at most one, so `by (host, job)` with only -/// `host` declared, and both groupings with nothing declared, are priced at -/// 10,000 groups. -#[test] -fn a_grouping_with_an_undeclared_label_is_bound_by_the_series_read() { - let run = priced_exact(&workload(serde_json::json!({"host": 1000}))); - assert_eq!( - priced_groups(&run, Q3_BY_HOST_JOB), - BTreeSet::from([10_000]) - ); - assert_eq!(priced_groups(&run, Q4_BY_JOB), BTreeSet::from([10_000])); - let run = priced_exact(&workload(serde_json::json!({}))); - assert_eq!( - priced_groups(&run, Q3_BY_HOST_JOB), - BTreeSet::from([10_000]) - ); -} diff --git a/crates/integration-tests/tests/deployment_capabilities.rs b/crates/integration-tests/tests/deployment_capabilities.rs deleted file mode 100644 index 482ca9ac5..000000000 --- a/crates/integration-tests/tests/deployment_capabilities.rs +++ /dev/null @@ -1,271 +0,0 @@ -//! The deployment's capabilities (#509's execution-capability input) over -//! the planner-layering examples: the summaries the executor supports -//! restrict Stage 1 (Pass 1, and so Pass 2). - -use crate::planner_layering_common; - -use asap_cost::CostInputs; -use asap_devtools::planning_stages::lower_sql_roots; -use asap_logical_asap_optimization::{DeploymentCapabilities, SupportedSummary}; -use asap_types::ir::flat::FlatDag; -use asap_types::ir::schema::{SketchAlgorithm, StatisticKind}; -use asap_types::ir::{ASAPOp, Operator}; -use asap_types::physical::Materialization; -use planner_layering_common::*; - -/// A deployment whose executor supports only `supported`: each family with -/// its readouts. -fn supporting(supported: &[(SketchAlgorithm, &[StatisticKind])]) -> DeploymentCapabilities { - DeploymentCapabilities { - supported_summaries: Some( - supported - .iter() - .map(|(family, readouts)| SupportedSummary { - family: *family, - readouts: readouts.to_vec(), - }) - .collect(), - ), - ..DeploymentCapabilities::default() - } -} - -/// Every (family, readout) a logical candidate's DAG uses: each sketch -/// build with each statistic estimated from it. -fn summaries_used(dag: &FlatDag) -> Vec<(SketchAlgorithm, StatisticKind)> { - sketch_builds(dag) - .into_iter() - .flat_map(|(build, family, _)| { - estimates_of(dag, build) - .into_iter() - .map(move |(_, statistic)| (family, statistic.kind())) - }) - .collect() -} - -/// Capabilities filter each local computation before physical selection. -/// Compare explicit one-rule applications, not a timed search's archive. -#[test] -fn restricting_summaries_removes_exactly_the_candidates_that_use_them() { - let heap = supporting(&[(SketchAlgorithm::CountSketchWithHeap, &[StatisticKind::TopK])]); - assert_removes_exactly("example 1", lower_promql(&example1()), &heap); - let distinct_and_l2 = supporting(&[ - ( - SketchAlgorithm::UnivMon, - &[StatisticKind::Distinct, StatisticKind::FrequencyL2], - ), - (SketchAlgorithm::Hll, &[StatisticKind::Distinct]), - ]); - let (workload, catalog) = example2(); - assert_removes_exactly( - "example 2", - lower_sql_roots(&workload, &catalog).unwrap(), - &distinct_and_l2, - ); - let kll = supporting(&[(SketchAlgorithm::Kll, &[StatisticKind::Quantile])]); - assert_removes_exactly("pattern B", lower_promql(&pattern_b()), &kll); -} - -fn assert_removes_exactly( - name: &str, - roots: Vec, - deployment: &DeploymentCapabilities, -) { - use asap_logical_asap_optimization::test_support::{ - compose_logical_candidate, generate_per_computation_candidates, - }; - let local = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&roots), - ) - .unwrap(); - let enumerate_local = - |inventory: &asap_logical_asap_optimization::test_support::PerComputationCandidates| { - let mut result = Vec::new(); - for (target, options) in inventory.targets.iter().enumerate() { - for choice in 0..options.choice_count() { - let mut choices = vec![0; inventory.targets.len()]; - choices[target] = choice; - if let Ok(roots) = compose_logical_candidate(inventory, &choices) { - result.push(asap_types::ir::flat::flatten(&roots).0); - } - } - } - result - }; - let unrestricted = enumerate_local(&local); - let restricted = enumerate_local( - &local.without_unsupported_summaries(deployment.supported_summaries.as_deref()), - ); - let expected = unrestricted - .iter() - .filter(|dag| { - summaries_used(dag) - .iter() - .all(|(family, readout)| deployment.supports(family, *readout)) - }) - .collect::>(); - assert!( - expected.len() < unrestricted.len(), - "{name}: unsupported candidates removed" - ); - assert!( - restricted.iter().any(|dag| !summaries_used(dag).is_empty()), - "{name}: supported summary retained" - ); - assert_eq!(restricted.iter().collect::>(), expected, "{name}"); -} - -/// Pattern B with a p50 panel beside its p99, both every minute over the -/// same 5-min window. One 1-min tumbling KLL chain serves both panels. -fn pattern_b_with_p50() -> asap_types::workload::PlanningWorkload { - let mut workload = pattern_b(); - let repeating = workload - .query_workload - .repeating_queries - .as_mut() - .expect("pattern B repeats"); - let mut p50 = repeating[0].clone(); - p50.query = asap_types::workload::Query("quantile_over_time(0.5, latency_ms[5m])".into()); - repeating.push(p50); - workload -} - -/// One explicitly shared KLL tumbling-window chain, passed through normal physical rules. -fn shared_tumbling_window_configurations(deployment: &DeploymentCapabilities) -> Run { - use asap_logical_asap_optimization::test_support::{ - compose_logical_candidate, generate_per_computation_candidates, - }; - use asap_logical_asap_optimization::test_support::{ - tumbling_window_candidates, window_composition_pass2_candidates, - }; - use asap_types::ir::QueryRoot; - use asap_types::stages::HowToComputeAggregateMeasure; - let workload = pattern_b_with_p50(); - let original = lower_promql(&workload); - let local = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&original), - ) - .unwrap(); - let choices = local - .targets - .iter() - .map(|target| { - target - .ways_to_compute - .iter() - .position(|candidate| { - matches!( - candidate, - HowToComputeAggregateMeasure::Summary(SketchAlgorithm::Kll) - ) - }) - .unwrap() - }) - .collect::>(); - let composed = compose_logical_candidate(&local, &choices).unwrap(); - let shared = asap_logical_asap_optimization::test_support::share_common_sub_dags( - composed - .into_iter() - .enumerate() - .map(|(id, root)| match root { - QueryRoot::Operator(node) => (id, node), - _ => unreachable!(), - }) - .collect(), - ) - .into_iter() - .map(|(_, node)| QueryRoot::Operator(node)) - .collect::>(); - let tumbling_windows = tumbling_window_candidates(&shared, &[Some(60_000), Some(60_000)]); - assert_eq!(tumbling_windows.len(), 1, "both estimators share one build"); - let tumbling_window_roots = window_composition_pass2_candidates(&shared, &tumbling_windows) - .unwrap() - .remove(0) - .roots - .unwrap(); - let settings = CostInputs::default(); - run_explicit_candidates( - &workload, - original, - vec![tumbling_window_roots], - asap_planner::PlanningModels::builtin() - .with_cost_inputs(&settings) - .with_deployment_capabilities(deployment), - ) -} - -/// Physical candidate ids with their cost per second. -type CandidateCosts = Vec<(String, f64)>; - -/// The per-second cost of every physical candidate whose tumbling-window chain both -/// panels read is materialized at query time: shared within one evaluation -/// (`MatAtQuery`), and kept across evaluations -/// (`MatAtQueryKeptAcrossEvaluations`, which Stage 3 prices as such). -fn shared_chain_costs(run: &Run) -> (CandidateCosts, CandidateCosts) { - let (mut within_one, mut kept) = (Vec::new(), Vec::new()); - for p in &run.physical { - for n in &p.dag().nodes { - let shared_chain = matches!( - n.payload, - Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) - ) && readers(p.dag(), &p.query_roots, n.id).len() == 2; - let Some(cost) = run.selection.costs.get(&p.id).filter(|_| shared_chain) else { - continue; - }; - let entry = (p.id.clone(), cost.cost_per_second.expect("cost model 1")); - let kept_detail = cost.per_node[&n.id].detail.contains("across evaluations"); - match n.materialization { - Materialization::MatAtQuery(_) => within_one.push(entry), - Materialization::MatAtQueryKeptAcrossEvaluations(_) if kept_detail => { - kept.push(entry) - } - Materialization::MatAtQueryKeptAcrossEvaluations(_) => { - panic!("{}: a kept chain priced as not kept", p.id) - } - _ => {} - } - } - } - (within_one, kept) -} - -/// With query-time results retained, a chain two panels read is offered -/// both ways: shared within one evaluation (rebuilt from the 5-min window -/// every minute, then reused by both panels) and kept across evaluations -/// (B3: one new tumbling window per evaluation, the 4 older tumbling windows held in memory -/// between evaluations). Holding 1M per-series KLLs between evaluations -/// costs far more than rebuilding them, so the shared option is cheaper -/// and the kept one is not selected. Without query-time retention, the -/// kept option is not offered. -#[test] -fn query_time_results_retained_still_offers_the_chain_shared_within_one_evaluation() { - let keeping = DeploymentCapabilities { - query_time_results_retained: true, - ..DeploymentCapabilities::default() - }; - let run = shared_tumbling_window_configurations(&keeping); - let (within_one, kept) = shared_chain_costs(&run); - assert!(!within_one.is_empty(), "no shared option: kept {kept:?}"); - assert!(!kept.is_empty(), "no kept option: shared {within_one:?}"); - let cheapest = |options: &[(String, f64)]| { - options - .iter() - .map(|(_, cost)| *cost) - .fold(f64::INFINITY, f64::min) - }; - assert!( - cheapest(&within_one) < cheapest(&kept), - "shared {within_one:?}, kept {kept:?}" - ); - assert!( - kept.iter().all(|(id, _)| *id != run.selection.selected), - "{}", - run.selection.selected - ); - assert_selects_cheapest_valid(&run); - - let (_, kept) = shared_chain_costs(&shared_tumbling_window_configurations( - &DeploymentCapabilities::default(), - )); - assert!(kept.is_empty(), "{kept:?}"); -} diff --git a/crates/integration-tests/tests/exact_composition.rs b/crates/integration-tests/tests/exact_composition.rs index 63d051298..4ed6ff21a 100644 --- a/crates/integration-tests/tests/exact_composition.rs +++ b/crates/integration-tests/tests/exact_composition.rs @@ -9,17 +9,15 @@ use std::rc::Rc; -use asap_integration_tests::physical_export::{at_ingestion, physical_asap_dag, timed}; +use asap_integration_tests::physical_export::physical_asap_dag; use asap_integration_tests::planning::explicit_local_candidates; use asap_types::ir::operator::agg_intent::{default_quantile, AggIntent}; use asap_types::ir::operator::operator_properties::{Reduction, Source}; use asap_types::ir::physical_export::PhysicalASAPOperatorPayload; use asap_types::ir::properties::ExecutionTiming; use asap_types::ir::schema::{DataType, Field, Schema}; -use asap_types::ir::schema::{ExactKind, FieldDataType, SummaryUpdate}; -use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode, TimeRangeKind}; -use asap_types::ir::properties::AccuracyTarget; +use asap_types::ir::{ASAPOp, NonASAPOp, OperatorNode}; // ── fixtures ──────────────────────────────────────────────────────────── @@ -58,17 +56,6 @@ fn agg(by: Vec, intent: AggIntent, child: Rc) -> Rc) -> Rc { - node(NonASAPOp::Aggregate { - reduction: Reduction::PerEntity, - measures: vec![intent], - output_names: vec![], - filters: vec![], - having: None, - child, - }) -} - /// `quantile by (zone, host) (latency)` — the fine-grained inner summary. fn fine_quantile() -> Rc { agg( @@ -110,210 +97,12 @@ fn summary_evaluation(node: &Rc) -> Option<&Rc> { node.children().into_iter().find_map(summary_evaluation) } -fn is_plain(node: &OperatorNode) -> bool { - node.schema - .fields - .iter() - .all(|f| matches!(f.dtype, FieldDataType::Plain(_))) -} - -fn names(node: &OperatorNode) -> Vec<&str> { - node.schema.fields.iter().map(|f| f.name.as_str()).collect() -} - // ── step 1: pin every already-supported exact-accumulator nesting ─────── -#[test] -fn every_exact_accumulator_is_finalized_before_an_outer_sketch() { - use std::time::Duration; - let cases: Vec<(Rc, ExactKind)> = vec![ - ( - agg( - vec![2], - AggIntent::Sum { col: None }, - metric_scan(&["zone"]), - ), - ExactKind::Sum, - ), - ( - agg( - vec![2], - AggIntent::Count { - accuracy: AccuracyTarget::exact(), - }, - metric_scan(&["zone"]), - ), - ExactKind::Count, - ), - ( - agg( - vec![2], - AggIntent::Min { col: None }, - metric_scan(&["zone"]), - ), - ExactKind::Min, - ), - ( - agg( - vec![2], - AggIntent::Max { col: None }, - metric_scan(&["zone"]), - ), - ExactKind::Max, - ), - ( - per_entity( - AggIntent::Rate, - node(NonASAPOp::TimeRange { - range: Duration::from_secs(300), - kind: TimeRangeKind::Range, - child: metric_scan(&["zone"]), - }), - ), - ExactKind::Rate, - ), - ( - per_entity( - AggIntent::Increase, - node(NonASAPOp::TimeRange { - range: Duration::from_secs(300), - kind: TimeRangeKind::Range, - child: metric_scan(&["zone"]), - }), - ), - ExactKind::Increase, - ), - ]; - for (inner, kind) in cases { - let outer = agg(vec![], default_quantile(0.9), inner); - // The KLL candidate over the inner exact accumulator. - let root = explicit_local_candidates(Rc::clone(&outer)) - .into_iter() - .map(|candidate| Rc::clone(below_projects(&candidate))) - .find(|candidate| { - matches!(candidate.asap(), Some(ASAPOp::SummaryEstimate { summary_input, .. }) - if matches!(summary_input.asap(), Some(ASAPOp::SummaryAgg { child, .. }) - if matches!(child.asap(), Some(ASAPOp::FinalizeExactAccumulator { .. })))) - }) - .unwrap_or_else(|| panic!("{kind:?}: a KLL over the exact accumulator")); - // Timing is not stored on the plan: time it with the outer summary - // maintained (which also validates every edge) and inspect the copy. - let root = at_ingestion(&root); - let Operator::ASAP(ASAPOp::SummaryEstimate { summary_input, .. }) = &root.operator else { - panic!("expected KLL evaluation, got {:?}", root.operator); - }; - let Operator::ASAP(ASAPOp::SummaryAgg { child, .. }) = &summary_input.operator else { - panic!("expected outer SummaryAgg"); - }; - let Operator::ASAP(ASAPOp::FinalizeExactAccumulator { child: finalized }) = &child.operator - else { - panic!("{kind:?}: missing maintenance finalization"); - }; - assert_eq!( - child.timing, - Some(ExecutionTiming::IngestionTime), - "{kind:?}: finalization runs at maintenance time" - ); - assert!( - matches!( - &finalized.operator, - Operator::ASAP(ASAPOp::SummaryAgg { family: FieldDataType::ExactAggregate(k, _), .. }) if *k == kind - ), - "{kind:?}: expected the exact accumulator under its finalization, got {:?}", - finalized.operator - ); - } -} - // ── direction 1: outer exact fold over an inner summary evaluation ──────── -/// `max`/`avg` over a quantile composes as an exact fold at query time over -/// the inner quantile's evaluation, with the pre-ASAP target's own schema. -#[test] -fn max_and_avg_over_quantile_compose_at_query_time() { - for intent in [AggIntent::Max { col: None }, AggIntent::Avg { col: None }] { - let root = agg(vec![0], intent.clone(), fine_quantile()); - let composed = composed(&root); - let child = summary_evaluation(&composed).unwrap(); - assert!( - child.guarantee.is_some(), - "child has its KLL rank guarantee" - ); - assert!( - composed.guarantee.is_none(), - "rank error has no definition-backed conversion through max/average" - ); - assert!(is_plain(&composed)); - assert_eq!( - names(&composed), - root.schema - .fields - .iter() - .map(|c| c.name.as_str()) - .collect::>(), - "the composed plan's schema is the pre-ASAP target's own" - ); - assert_eq!( - timed(&composed).timing, - Some(ExecutionTiming::QueryTime), - "{intent:?}: the exact fold runs at query time" - ); - } -} - -/// Grouped fine-to-coarse fold (`by (zone)` over `by (zone, host)`) and the -/// identity fold (`by (zone)` over `by (zone)`) both compose; the operator -/// is the same, only the fold's row multiplicity differs. -#[test] -fn identity_and_genuine_multi_row_folds_both_compose() { - let identity_inner = agg(vec![2], default_quantile(0.99), metric_scan(&["zone"])); - for (label, inner) in [ - ("identity", identity_inner), - ("fine-to-coarse", fine_quantile()), - ] { - let root = agg(vec![0], AggIntent::Max { col: None }, inner); - let composed = composed(&root); - assert_eq!( - timed(&composed).timing, - Some(ExecutionTiming::QueryTime), - "{label}" - ); - assert_eq!(names(&composed), vec!["zone", "max"], "{label}"); - } -} - // ── summary construction phase ────────────────────────────────────────── -/// Summary construction can consume query-time values without pretending -/// they are available to an ingestion-time consumer. -#[test] -fn summary_construction_follows_its_value_input_phase() { - let root = agg(vec![0], AggIntent::Max { col: None }, fine_quantile()); - let post = composed(&root); - let illegal = std::rc::Rc::new( - OperatorNode::with_schema( - asap_types::ir::Operator::ASAP(ASAPOp::SummaryAgg { - child: post, - family: FieldDataType::ExactAggregate( - ExactKind::Max, - asap_types::ir::schema::ExactParams::Max, - ), - input: SummaryUpdate::column(asap_types::ir::scalar::ColumnRef::SampleValue), - reduction: Reduction::by(vec![]), - grouping: Default::default(), - filter: None, - }), - Schema::lifted(vec![], None), - ) - .with_guarantee(None), - ); - // Even under an ingestion-time consumer the state is built at query - // time, because a evaluation sits below it. - let state = asap_types::ir::planned_data_state(&illegal, ExecutionTiming::IngestionTime); - assert_eq!(state.timing, ExecutionTiming::QueryTime); - asap_types::ir::validate_at_ingestion(&illegal, state.timing).unwrap(); -} - // ── Physical export: explicit timing ───────────────────────────────────── #[test] diff --git a/crates/integration-tests/tests/filtered_aggregates.rs b/crates/integration-tests/tests/filtered_aggregates.rs index 47dfd4efd..40c52db98 100644 --- a/crates/integration-tests/tests/filtered_aggregates.rs +++ b/crates/integration-tests/tests/filtered_aggregates.rs @@ -1,7 +1,6 @@ //! Filtered aggregates (`FILTER (WHERE …)`, a per-measure filter, or a //! filtered `SummaryAgg`) execute with SQL semantics: a row that fails the //! filter contributes nothing, but its group is kept. -use crate::physical_common; use std::rc::Rc; use std::sync::Arc; @@ -57,7 +56,7 @@ fn rows() -> Vec> { /// Run the logical DAG `root` over [`rows`]; rows come back shown and sorted. fn execute(root: &Rc) -> Vec { - let dag = physical_common::compile_physical_asap_dag(root).unwrap(); + let dag = asap_types::test_support::physical_asap_dag_at_query_time(root).unwrap(); let scan = dag .nodes .iter() @@ -259,7 +258,7 @@ async fn hand_filtered_kll_compiles_and_executes() { .await; let roots = hand_filtered( "SELECT g, approx_percentile_cont(x, 0.5) AS q FROM events GROUP BY g", - |f| matches!(f, FieldDataType::Sketch(kind, _) if kind.algorithm() == &SketchAlgorithm::Kll), + |f| matches!(f, FieldDataType::Sketch(kind, _) if kind == &SketchAlgorithm::Kll), &filter, ) .await; @@ -278,7 +277,7 @@ async fn kll_candidates(sql: &str) -> Vec> { .filter(|root| { OperatorNode::reachable(root).iter().any(|n| { matches!(n.asap(), Some(ASAPOp::SummaryAgg { family: FieldDataType::Sketch(kind, _), .. }) - if kind.algorithm() == &SketchAlgorithm::Kll) + if kind == &SketchAlgorithm::Kll) }) }) .collect(); diff --git a/crates/integration-tests/tests/finer_aggregate_reuse.rs b/crates/integration-tests/tests/finer_aggregate_reuse.rs index ed8a1b06e..443181401 100644 --- a/crates/integration-tests/tests/finer_aggregate_reuse.rs +++ b/crates/integration-tests/tests/finer_aggregate_reuse.rs @@ -2,18 +2,10 @@ //! when its groups are coarser, a rollup of values or states (#950). use asap_devtools::planner_fuzz::{read_seed_file, FuzzSettings, FuzzWorkload, Planner}; use asap_devtools::planning_stages::run_search_over; -use asap_executor::values::Value; -use asap_logical_asap_optimization::generate_dag_of_logical_memo_groups; -use asap_logical_asap_optimization::test_support::{ - generate_per_computation_candidates, try_populate_pass1, -}; -use asap_types::ir::{ASAPOp, NonASAPOp, OperatorNode}; -use asap_types::stages::{CandidateForMemoGroup, Memo, OriginOfCandidateForMemoGroup}; + use serde_json::json; use std::path::Path; -use crate::physical_common; - const P90: &str = "percentile_cont(0.9) WITHIN GROUP (ORDER BY value)"; const WINDOW: &str = "ts > now() - INTERVAL '5' MINUTE"; @@ -41,257 +33,6 @@ fn workload(queries: &[String]) -> FuzzWorkload { w } -/// The expanded candidate memo of `queries`, as plan selection receives it. -fn expanded(queries: &[String]) -> Memo { - let roots: Vec<_> = Planner::new(FuzzSettings::default()) - .lower(&workload(queries)) - .unwrap() - .into_iter() - .collect(); - let initial = generate_dag_of_logical_memo_groups(&roots); - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&roots), - ) - .unwrap(); - let memo = try_populate_pass1(initial, &inventory).unwrap(); - asap_logical_asap_optimization::test_support::sharing_rules_and_shared_state_groups(memo) - .unwrap() -} - -/// The aggregate group of query `query` alone and its rollup candidates. -fn rollups(memo: &Memo, query: usize) -> (usize, Vec<&CandidateForMemoGroup>) { - let owner = memo - .groups - .iter() - .position(|group| { - group.reached_by_queries() == [query] - && matches!( - group.logical_node.non_asap(), - Some(NonASAPOp::Aggregate { .. }) - ) - }) - .expect("the query's own aggregate group"); - let candidates = memo.candidates[owner] - .iter() - .filter(|c| c.origin == OriginOfCandidateForMemoGroup::Rollup) - .collect(); - (owner, candidates) -} - -/// Whether `candidate` reads another group's result through a `Filter` -/// (the value path), rather than merging summary states. -fn reads_finer_result(candidate: &CandidateForMemoGroup) -> bool { - !OperatorNode::reachable(&candidate.root_node) - .iter() - .any(|node| matches!(node.asap(), Some(ASAPOp::SummaryMerge { .. }))) - && OperatorNode::reachable(&candidate.root_node) - .iter() - .any(|node| matches!(node.non_asap(), Some(NonASAPOp::Filter { .. }))) -} - -fn row(city: Option<&str>, host: &str, value: Option) -> Vec { - vec![ - Value::Timestamp(0), - city.map_or(Value::Null, |city| Value::Utf8(city.into())), - Value::Utf8(host.into()), - value.map_or(Value::Null, Value::Float64), - ] -} - -/// Row sets covering an empty input, an absent selected group, NULL samples, -/// several hosts per city and other cities. -fn row_sets() -> Vec>> { - vec![ - vec![], - vec![row(Some("LA"), "h1", Some(5.0))], - vec![row(Some("NYC"), "h1", None)], - vec![ - row(Some("NYC"), "h1", Some(10.0)), - row(Some("NYC"), "h1", Some(30.0)), - row(Some("NYC"), "h2", Some(20.0)), - row(Some("SF"), "h1", Some(7.0)), - row(Some("LA"), "h1", Some(90.0)), - row(None, "h1", Some(1.0)), - ], - ] -} - -/// The candidate gives the original query's rows on every row set. -fn assert_same_rows(memo: &Memo, owner: usize, candidate: &CandidateForMemoGroup) { - let sorted = |rows: Vec>| { - let mut rows: Vec<_> = rows.iter().map(|row| format!("{row:?}")).collect(); - rows.sort(); - rows - }; - for rows in row_sets() { - let before = physical_common::execute_raw_rows_at( - &memo.groups[owner].logical_node, - rows.clone(), - 1000, - ); - let after = physical_common::execute_raw_rows_at(&candidate.root_node, rows.clone(), 1000); - assert_eq!(sorted(after), sorted(before), "rows {rows:?}"); - } -} - -/// Example A: both dropped keys are pinned, so the quantile query reads one -/// row of the finer quantile result; no quantile rollup is needed. -#[test] -fn pinned_keys_read_one_finer_quantile_row() { - let memo = expanded(&[ - format!("SELECT city, host, {P90} FROM metrics WHERE {WINDOW} GROUP BY city, host"), - format!("SELECT {P90} FROM metrics WHERE {WINDOW} AND city = 'NYC' AND host = 'h1'"), - ]); - let (owner, candidates) = rollups(&memo, 1); - let read = candidates - .into_iter() - .find(|c| reads_finer_result(c)) - .expect("a read of the finer quantile result"); - assert_same_rows(&memo, owner, read); -} - -/// Example B: a filtered global sum keeps the `city = 'NYC'` rows of the -/// finer sums and sums them. -#[test] -fn filtered_sum_rolls_up_finer_sums() { - let memo = expanded(&[ - format!("SELECT city, host, SUM(value) FROM metrics WHERE {WINDOW} GROUP BY city, host"), - format!("SELECT SUM(value) FROM metrics WHERE {WINDOW} AND city = 'NYC'"), - ]); - let (owner, candidates) = rollups(&memo, 1); - let read = candidates - .into_iter() - .find(|c| reads_finer_result(c)) - .expect("a rollup of the finer sums"); - assert_same_rows(&memo, owner, read); -} - -/// A grouped query whose filter keeps several groups: filter, then roll the -/// finer `(city, host)` sums and maxima up to `city`. -#[test] -fn filtered_grouped_aggregates_roll_up_finer_values() { - let memo = expanded(&[ - format!( - "SELECT city, host, SUM(value), MAX(value) FROM metrics WHERE {WINDOW} GROUP BY city, host" - ), - format!( - "SELECT city, SUM(value), MAX(value) FROM metrics WHERE {WINDOW} AND city IN ('NYC', 'SF') GROUP BY city" - ), - ]); - let (owner, candidates) = rollups(&memo, 1); - let read = candidates - .into_iter() - .find(|c| reads_finer_result(c)) - .expect("a filtered rollup of the finer values"); - assert_same_rows(&memo, owner, read); -} - -/// Example C: two cities' p90 values cannot be combined, but their KLL -/// states can: keep the NYC and SF states, merge them, estimate. The merged -/// read gives the coarse KLL read's rows, SQL's one NULL row over an empty -/// selection included; the query's own read keeps the rows the `OR` form -/// keeps. -#[test] -fn filtered_kll_states_merge() { - let memo = expanded(&[ - format!("SELECT city, {P90} FROM metrics WHERE {WINDOW} GROUP BY city"), - format!("SELECT {P90} FROM metrics WHERE {WINDOW} AND city IN ('NYC', 'SF')"), - ]); - let (owner, candidates) = rollups(&memo, 1); - let merged = candidates - .into_iter() - .find(|candidate| { - OperatorNode::reachable(&candidate.root_node).iter().any(|node| { - matches!(node.asap(), Some(ASAPOp::SummaryMerge { children, group_by: Some(_), .. }) - if matches!(children[0].non_asap(), Some(NonASAPOp::Filter { .. }))) - }) - }) - .expect("a candidate merging the filtered finer states"); - let own_kll = |memo: &Memo, owner: usize| { - memo.candidates[owner] - .iter() - .find(|candidate| { - candidate.origin == OriginOfCandidateForMemoGroup::Pass1 - && OperatorNode::reachable(&candidate.root_node) - .iter() - .any(|node| { - matches!(node.asap(), Some(ASAPOp::SummaryAgg { family, .. }) - if format!("{family:?}").contains("Kll")) - }) - }) - .expect("the query's own KLL candidate") - .clone() - }; - let direct = own_kll(&memo, owner); - let or_memo = expanded(&[format!( - "SELECT {P90} FROM metrics WHERE {WINDOW} AND (city = 'NYC' OR city = 'SF')" - )]); - let or_direct = own_kll(&or_memo, rollups(&or_memo, 0).0); - for rows in row_sets() { - let before = physical_common::execute_raw_rows_at(&direct.root_node, rows.clone(), 1000); - let or_form = - physical_common::execute_raw_rows_at(&or_direct.root_node, rows.clone(), 1000); - assert_eq!( - format!("{before:?}"), - format!("{or_form:?}"), - "rows {rows:?}" - ); - let after = physical_common::execute_raw_rows_at(&merged.root_node, rows.clone(), 1000); - assert_eq!(format!("{after:?}"), format!("{before:?}"), "rows {rows:?}"); - assert_eq!(after.len(), 1); - } -} - -/// The same groups in another key order are one computation: neither -/// aggregate reads the other (two such reads would be a cycle). -#[test] -fn permuted_group_by_is_not_a_finer_read() { - let memo = expanded(&[ - format!("SELECT city, host, SUM(value) FROM metrics WHERE {WINDOW} GROUP BY city, host"), - format!("SELECT host, city, SUM(value) FROM metrics WHERE {WINDOW} GROUP BY host, city"), - ]); - for query in 0..2 { - let (_, candidates) = rollups(&memo, query); - assert!(!candidates.into_iter().any(reads_finer_result)); - } -} - -/// A conjunct on a non-key column cannot be applied to finer sums. -#[test] -fn value_conjunct_has_no_finer_read() { - let memo = expanded(&[ - format!("SELECT city, host, SUM(value) FROM metrics WHERE {WINDOW} GROUP BY city, host"), - format!("SELECT SUM(value) FROM metrics WHERE {WINDOW} AND city = 'NYC' AND value > 3"), - ]); - let (_, candidates) = rollups(&memo, 1); - assert!(!candidates.into_iter().any(reads_finer_result)); -} - -/// A dropped key that is not pinned leaves several quantile values per -/// group, and quantile values do not roll up. -#[test] -fn unpinned_key_has_no_quantile_value_read() { - let memo = expanded(&[ - format!("SELECT city, host, {P90} FROM metrics WHERE {WINDOW} GROUP BY city, host"), - format!("SELECT {P90} FROM metrics WHERE {WINDOW} AND city = 'NYC'"), - ]); - let (_, candidates) = rollups(&memo, 1); - assert!(!candidates.into_iter().any(reads_finer_result)); -} - -/// The finer result lacks rows the query needs: compensation only removes rows. -#[test] -fn narrower_finer_aggregate_has_no_read() { - let memo = expanded(&[ - format!( - "SELECT city, host, SUM(value) FROM metrics WHERE {WINDOW} AND city = 'SF' GROUP BY city, host" - ), - format!("SELECT SUM(value) FROM metrics WHERE {WINDOW} AND city = 'NYC'"), - ]); - let (_, candidates) = rollups(&memo, 1); - assert!(!candidates.into_iter().any(reads_finer_result)); -} - /// The selected plan's cost per second for `queries`. fn cost(queries: &[String]) -> f64 { use asap_cost::CostInputs; @@ -311,7 +52,6 @@ fn cost(queries: &[String]) -> f64 { &w.data, models, PlanSearchSettings::default(), - None, ) .unwrap(); let selection = run.run.selection.as_ref().unwrap(); diff --git a/crates/integration-tests/tests/frequency_execution.rs b/crates/integration-tests/tests/frequency_execution.rs deleted file mode 100644 index 24c083091..000000000 --- a/crates/integration-tests/tests/frequency_execution.rs +++ /dev/null @@ -1,148 +0,0 @@ -//! #509 Example 2's PromQL stand-in executes natively: the distinct count, -//! entropy and L2 norm of one series' sample values over 1 min, through the -//! exact candidate and through the one that builds a UnivMon, at query time. - -use std::collections::BTreeMap; -use std::rc::Rc; - -use asap_executor::physical_planner::{compile, promql_rows}; -use asap_executor::runtime::Scope; -use asap_executor::values::Value; -use asap_integration_tests::fixtures::lower_promql; -use asap_integration_tests::planning::explicit_local_candidates; -use asap_types::ir::physical_export::{compile_physical_asap_dag, PhysicalASAPOperatorPayload}; -use asap_types::ir::properties::AccuracyTarget; -use asap_types::ir::schema::{FieldDataType, SketchAlgorithm}; -use asap_types::ir::{ - apply_materialization_timings, ASAPOp, MaterializationAssignment, NodeToTimedNodeCache, - NonASAPOp, OperatorNode, -}; - -use crate::physical_common; - -const EVALUATION_MS: i64 = 300_000; -/// `flows_src_ip` samples: source 1 twice, 2 three times, 3 once. -const SAMPLES: [f64; 6] = [1.0, 2.0, 1.0, 2.0, 3.0, 2.0]; - -/// The summary family a candidate builds: `None` for the exact plan. -fn built_family(candidate: &Rc) -> Option { - OperatorNode::reachable(candidate) - .iter() - .find_map(|node| match node.asap() { - Some(ASAPOp::SummaryAgg { family, .. }) => Some(family.clone()), - _ => None, - }) -} - -/// Execute `candidate` at query time over `SAMPLES`, all inside the window. -fn execute(candidate: &Rc) -> Vec { - let timed = apply_materialization_timings( - candidate, - &MaterializationAssignment::default(), - &mut NodeToTimedNodeCache::new(), - ) - .unwrap(); - let dag = compile_physical_asap_dag(&timed).unwrap(); - let ranges = dag - .nodes - .iter() - .filter(|node| { - matches!( - node.payload, - PhysicalASAPOperatorPayload::NonASAP(NonASAPOp::TimeRange { .. }) - ) - }) - .count(); - assert_eq!(ranges, 1, "one range selector"); - // The whole plan runs at query time over the raw samples. - let compiled = compile(&dag, &[], &[dag.roots[0] as u64]).unwrap(); - let labels = BTreeMap::from([("__name__".to_string(), "flows_src_ip".to_string())]); - let store = physical_common::raw_data_store(&dag, |_, schema| { - SAMPLES - .iter() - .enumerate() - .map(|(i, value)| { - promql_rows::series_row(schema, &labels, 290_000 + i as i64, *value).unwrap() - }) - .collect() - }); - let result = physical_common::execute( - &compiled, - store, - Scope::Query { - evaluation_time_ms: EVALUATION_MS, - }, - ); - result[0] - .iter() - .flat_map(|batch| batch.to_rows()) - .flatten() - .filter_map(|value| match value { - Value::Float64(value) => Some(value), - Value::Int64(value) => Some(value as f64), - _ => None, - }) - .collect() -} - -// The exact candidate and the UnivMon candidate of each statistic both -// execute and give the statistic of the window's sample values (entropy in -// bits). -#[test] -fn standin_exact_and_univmon_candidates_execute() { - let entropy = -[2.0_f64, 3.0, 1.0] - .iter() - .map(|count| count / 6.0 * (count / 6.0).log2()) - .sum::(); - // A UnivMon sized for a distinct count at ε = 0.01 holds a heap of - // millions of keys on each of its 19 layers, past the default 64 MiB. - for (query, expected, epsilon, delta) in [ - ("distinct_over_time(flows_src_ip[1m])", 3.0, 0.2, 0.1), - ("entropy_over_time(flows_src_ip[1m])", entropy, 0.01, 0.01), - ( - "l2_over_time(flows_src_ip[1m])", - 14.0_f64.sqrt(), - 0.01, - 0.01, - ), - ] { - let accuracy = AccuracyTarget::max_error(epsilon).failure_probability(delta); - let root = lower_promql(query, accuracy.clone()).unwrap(); - let root = promql_rows::with_series_identity(&root).unwrap(); - let mut executed = Vec::new(); - // This is an operator execution test. Enumerate each local rule once, - // not an arbitrary sized configuration visited by a budgeted search. - for candidate in explicit_local_candidates(root) { - let family = built_family(&candidate); - let name = match &family { - None => "exact".to_string(), - Some(FieldDataType::Sketch(kind, _)) - if kind.algorithm() == &SketchAlgorithm::UnivMon => - { - "UnivMon".to_string() - } - _ => continue, - }; - if query.starts_with("distinct_over_time") { - if let Some(FieldDataType::Sketch(kind, _)) = &family { - let asap_types::ir::schema::SketchParams::UnivMon { - distinct_failure_probability, - .. - } = kind.params() - else { - unreachable!("family filtered above") - }; - assert_eq!(*distinct_failure_probability, Some(delta)); - } - } - let values = execute(&candidate); - assert!( - matches!(values.as_slice(), [value] if (value - expected).abs() < 1e-9), - "{query} {name}: {values:?}, expected {expected}" - ); - executed.push(name); - } - executed.sort(); - assert_eq!(executed, ["UnivMon", "exact"], "{query}"); - } -} diff --git a/crates/integration-tests/tests/frontend_timestamps.rs b/crates/integration-tests/tests/frontend_timestamps.rs index 5a1fd282b..4890c85fb 100644 --- a/crates/integration-tests/tests/frontend_timestamps.rs +++ b/crates/integration-tests/tests/frontend_timestamps.rs @@ -20,7 +20,7 @@ async fn promql_eval_time_and_sql_current_timestamp_remain_distinct() { "expected a bare evaluation-time scalar, got {promql:?}" ); assert_eq!( - ScalarExpr::EvalTimestamp + ScalarExpr::>::EvalTimestamp .scalar_type(&Schema::default()) .unwrap() .0, diff --git a/crates/integration-tests/tests/having_shared_aggregate.rs b/crates/integration-tests/tests/having_shared_aggregate.rs deleted file mode 100644 index 554e63d38..000000000 --- a/crates/integration-tests/tests/having_shared_aggregate.rs +++ /dev/null @@ -1,115 +0,0 @@ -//! A key-only `HAVING` of one query is not pushed below an aggregate it -//! shares with another query (#1089): every candidate of the shared -//! aggregate's memo group returns the group's own groups. -use asap_devtools::planner_fuzz::{read_seed_file, FuzzSettings, FuzzWorkload, Planner}; -use asap_executor::values::Value; -use asap_logical_asap_optimization::generate_dag_of_logical_memo_groups; -use asap_logical_asap_optimization::test_support::{ - generate_per_computation_candidates, try_populate_pass1, -}; -use asap_types::ir::NonASAPOp; -use asap_types::stages::Memo; -use serde_json::json; -use std::path::Path; - -use crate::physical_common; - -const P90: &str = "percentile_cont(0.9) WITHIN GROUP (ORDER BY value)"; -const WINDOW: &str = "ts > now() - INTERVAL '1' HOUR"; - -/// `queries` over `metrics(ts, city, host, value)`, each repeated with -/// `max_error` 0.01 and `failure_probability` 0.001. -fn workload(queries: &[String]) -> FuzzWorkload { - let mut w = read_seed_file( - &Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/sharing_issue_fixtures/902.json"), - ) - .unwrap(); - w.tables = vec![json!({"name":"metrics", "time_index":0, "columns":[ - {"name":"ts","type":"timestamp","nullable":false}, - {"name":"city","type":"utf8","nullable":true}, - {"name":"host","type":"utf8","nullable":true}, - {"name":"value","type":"float64","nullable":true} - ]})]; - let template = w.queries[0].clone(); - w.queries = queries - .iter() - .map(|query| { - let mut entry = template.clone(); - entry.entry.query.0 = query.clone(); - entry - }) - .collect(); - w -} - -/// The Pass 1 memo of `queries` with every Pass 2 sharing rule applied. -fn expanded(queries: &[String]) -> Memo { - let roots: Vec<_> = Planner::new(FuzzSettings::default()) - .lower(&workload(queries)) - .unwrap() - .into_iter() - .collect(); - let dag = generate_dag_of_logical_memo_groups(&roots); - let inventory = generate_per_computation_candidates(&dag).unwrap(); - let memo = try_populate_pass1(dag, &inventory).unwrap(); - asap_logical_asap_optimization::test_support::sharing_rules_and_shared_state_groups(memo) - .unwrap() -} - -fn row(city: &str, host: &str, value: f64) -> Vec { - vec![ - Value::Timestamp(0), - Value::Utf8(city.into()), - Value::Utf8(host.into()), - Value::Float64(value), - ] -} - -/// The cities `node` returns over the #1089 rows, sorted. -fn cities(node: &std::rc::Rc) -> Vec { - let rows = vec![ - row("NYC", "h1", 10.0), - row("NYC", "h2", 20.0), - row("LA", "h1", 5.0), - row("LA", "h2", 7.0), - row("SF", "h1", 100.0), - ]; - let mut cities: Vec = physical_common::execute_raw_rows_at(node, rows, 1000) - .into_iter() - .map(|row| format!("{:?}", row[0])) - .collect(); - cities.sort(); - cities -} - -/// #1089: A (`HAVING city = 'NYC'`) listed before B (the same query without -/// it) share one aggregate group. Every candidate of that group returns all -/// three cities, as B needs; A filters above it. -#[test] -fn key_only_having_of_one_reader_stays_above_a_shared_aggregate() { - let grouped = format!("SELECT city, {P90} FROM metrics WHERE {WINDOW} GROUP BY city"); - let memo = expanded(&[format!("{grouped} HAVING city = 'NYC'"), grouped]); - let shared = memo - .groups - .iter() - .position(|group| { - group.reached_by_queries() == [0, 1] - && matches!( - group.logical_node.non_asap(), - Some(NonASAPOp::Aggregate { .. }) - ) - }) - .expect("the aggregate both queries read"); - let expected = cities(&memo.groups[shared].logical_node); - assert_eq!(expected.len(), 3, "{expected:?}"); - assert!(memo.candidates[shared].len() > 1, "summary candidates too"); - for candidate in &memo.candidates[shared] { - assert_eq!( - cities(&candidate.root_node), - expected, - "{:?} candidate {:?}", - candidate.origin, - candidate.root_node - ); - } -} diff --git a/crates/integration-tests/tests/identical_exact_queries.rs b/crates/integration-tests/tests/identical_exact_queries.rs index 55fb53010..9a31850a3 100644 --- a/crates/integration-tests/tests/identical_exact_queries.rs +++ b/crates/integration-tests/tests/identical_exact_queries.rs @@ -35,7 +35,6 @@ fn cost(w: &FuzzWorkload) -> f64 { &w.data, models, PlanSearchSettings::default(), - None, ) .unwrap(); let selection = run.run.selection.as_ref().unwrap(); diff --git a/crates/integration-tests/tests/independent_plan_seed.rs b/crates/integration-tests/tests/independent_plan_seed.rs index 7dcee170e..77767994d 100644 --- a/crates/integration-tests/tests/independent_plan_seed.rs +++ b/crates/integration-tests/tests/independent_plan_seed.rs @@ -51,7 +51,6 @@ fn cost(queries: &[String]) -> f64 { &w.data, models, PlanSearchSettings::default(), - None, ) .unwrap(); let selection = run.run.selection.as_ref().unwrap(); diff --git a/crates/integration-tests/tests/kll_tumbling_window_execution.rs b/crates/integration-tests/tests/kll_tumbling_window_execution.rs index bc0ad0698..217f15cfe 100644 --- a/crates/integration-tests/tests/kll_tumbling_window_execution.rs +++ b/crates/integration-tests/tests/kll_tumbling_window_execution.rs @@ -13,18 +13,13 @@ use asap_executor::{ use asap_types::ir::materialized_view::MaterializedView; use asap_types::ir::operator::operator_properties::TimeShift; use asap_types::ir::operator::{GroupKeys, Reduction, Source}; -use asap_types::ir::physical_export::{ - compile_physical_asap_workload_with_node_ids, PhysicalASAPDAG, -}; +use asap_types::ir::physical_export::PhysicalASAPDAG; use asap_types::ir::scalar::ColumnRef; use asap_types::ir::schema::{ - DataType, Field, FieldDataType, GroupingStrategy, Schema, SketchAlgorithm, SketchKind, - SketchParams, SketchStatistic, SummaryUpdate, -}; -use asap_types::ir::{ - apply_materialization_timings_to_roots, ASAPOp, MaterializationAssignment, NonASAPOp, - OperatorNode, QueryRoot, TimeRangeKind, + DataType, Field, FieldDataType, GroupingStrategy, Schema, SketchAlgorithm, SketchStatistic, + SummaryUpdate, }; +use asap_types::ir::{ASAPOp, NonASAPOp, OperatorNode, QueryRoot, TimeRangeKind}; use futures::{executor::block_on, StreamExt}; use std::{ rc::Rc, @@ -36,17 +31,8 @@ use std::{ const TUMBLING_WINDOW_WIDTH_MS: i64 = 60_000; -fn family(k: u32) -> FieldDataType { - FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k, - failure_probability: None, - }, - ), - Default::default(), - ) +fn family(_k: u32) -> FieldDataType { + FieldDataType::Sketch(SketchAlgorithm::Kll, Default::default()) } /// `t(ts, value)`, time index `ts`. fn table_schema() -> Schema { @@ -120,20 +106,17 @@ fn fixture() -> Fixture { query: SketchStatistic::Quantile { q }, })) }; - let roots = apply_materialization_timings_to_roots( - &[ - QueryRoot::Operator(estimate(0.5)), - QueryRoot::Operator(estimate(0.99)), - ], - &MaterializationAssignment::default(), - ) - .unwrap(); - let compiled = compile_physical_asap_workload_with_node_ids(&roots).unwrap(); + let roots = [ + QueryRoot::Operator(estimate(0.5)), + QueryRoot::Operator(estimate(0.99)), + ]; + let (dag, operator_node_of_id) = + asap_types::test_support::physical_asap_dag_at_query_time_with_node_ids(&roots).unwrap(); let mut tumbling_window_ids = vec![0; 5]; let (mut merge, mut p50, mut p99) = (0, 0, 0); - for node in &compiled.dag.nodes { + for node in &dag.nodes { let id = node.id as u64; - match &compiled.node_ids.operator_node(node.id).unwrap().operator { + match &operator_node_of_id[node.id].operator { asap_types::ir::Operator::ASAP(ASAPOp::SummaryAgg { child, .. }) => { let shift = match child.children()[0].non_asap() { Some(NonASAPOp::TimeShift { shift, .. }) => shift.offset_ms, @@ -152,7 +135,7 @@ fn fixture() -> Fixture { _ => {} } } - let mut dag = compiled.dag; + let mut dag = dag; let views = views_at(&mut dag, &tumbling_window_ids).unwrap(); Fixture { dag, diff --git a/crates/integration-tests/tests/main.rs b/crates/integration-tests/tests/main.rs index bf81ee35d..dfbe200d2 100644 --- a/crates/integration-tests/tests/main.rs +++ b/crates/integration-tests/tests/main.rs @@ -8,52 +8,30 @@ mod planner_layering_common; mod aggregate; mod binary_op; mod crate_graph; -mod declared_group_counts; -mod deployment_capabilities; mod exact_composition; mod filtered_aggregates; mod finer_aggregate_reuse; -mod frequency_execution; mod frontend_timestamps; -mod having_shared_aggregate; mod identical_exact_queries; mod independent_plan_seed; mod joint_search_equivalence; mod kll_tumbling_window_execution; -mod materialization_e2e; mod materialized_views; -mod memo_enumeration; mod nested; mod operator_design_examples; mod operator_sharing; mod per_computation_candidates_sql; mod per_series_exact_execution; -mod planner_layering_example1; -mod planner_layering_example2; -mod planner_layering_example3; -mod planner_layering_example4; -mod precompute_raw_samples; mod production_rollup_workloads; mod promql_numeric_regressions; mod promql_to_post_asap; -mod range_evaluation; -mod reuse_rules; mod rollup_controlled; -mod runtime_handoff; mod scan; mod schema; mod sharing_issue_regressions; mod sketch_rollup; -mod sketch_rollup_planning; mod sql_cardinality; mod sql_frequency; -mod sql_group_bound; -mod sql_to_physical; mod sql_to_post_asap; -mod sql_trailing_window; -mod sql_two_percentiles; mod stage1_logical_asap_optimization; -mod stage2_physical_asap_optimization_sql; -mod summary_coverage_e2e; mod time_range; -mod tumbling_window_merge; diff --git a/crates/integration-tests/tests/materialization_e2e.rs b/crates/integration-tests/tests/materialization_e2e.rs deleted file mode 100644 index 1751d3e95..000000000 --- a/crates/integration-tests/tests/materialization_e2e.rs +++ /dev/null @@ -1,356 +0,0 @@ -//! Materialization decides placement, not answers: the same selected DAG -//! timed with its states `NotMaterialized` (everything at query time) and -//! `MatAtIngestion` (materialized views refreshed at ingestion time) -//! executes to the same result. - -use std::collections::{BTreeMap, HashMap}; -use std::rc::Rc; - -use asap_executor::physical_planner::{compile, promql_rows}; -use asap_executor::runtime::Scope; -use asap_executor::values::{Batch, SchemaRef, Value}; -use asap_integration_tests::fixtures::lower_promql; -use asap_integration_tests::planning::{ - explicit_local_candidates, explicit_summary_candidate, selected_dag, -}; -use asap_physical_asap_optimization::test_support::split_shared_by_timing; -use asap_types::ir::operator::Source; -use asap_types::ir::physical_export::{ - compile_physical_asap_dag, PhysicalASAPDAG, PhysicalASAPOperatorPayload, -}; -use asap_types::ir::properties::ExecutionTiming; -use asap_types::ir::properties::{AccuracyMetric, AccuracyTarget}; -use asap_types::ir::{ - apply_materialization_timings, ASAPOp, MaterializationAssignment, NodeToTimedNodeCache, - NonASAPOp, OperatorNode, -}; -use asap_types::physical::{Materialization, Medium}; - -use crate::physical_common; - -const AT_INGESTION: Materialization = Materialization::MatAtIngestion(Medium::Memory); - -/// The logical DAG the planner pipeline selects for `query`, optionally with -/// a PromQL series identity (per-entity state needs one). -fn selected(query: &str, series_identity: bool) -> Rc { - selected_with(query, series_identity, AccuracyTarget::max_error(0.01)) -} - -/// [`selected`] for a quantile query, whose target names its metric (a -/// rank error). -fn selected_quantile(query: &str, series_identity: bool) -> Rc { - selected_with( - query, - series_identity, - AccuracyTarget::max_error(0.01).accuracy_metric(AccuracyMetric::Rank), - ) -} - -fn selected_with(query: &str, series_identity: bool, accuracy: AccuracyTarget) -> Rc { - let mut root = lower_promql(query, accuracy.clone()).unwrap(); - if series_identity { - root = promql_rows::with_series_identity(&root).unwrap(); - } - selected_dag(root, accuracy) -} - -/// The first logical candidate for `query` that builds a sketch: plan -/// selection picks the exact plan while every summary runs at query time. -fn sketch_candidate(query: &str) -> Rc { - let accuracy = AccuracyTarget::max_error(0.01); - let root = lower_promql(query, accuracy.clone()).unwrap(); - explicit_local_candidates(root) - .into_iter() - .find(|candidate| { - OperatorNode::reachable(candidate).iter().any(|node| { - matches!( - node.asap(), - Some(ASAPOp::SummaryAgg { - family: asap_types::ir::schema::FieldDataType::Sketch(..), - .. - }) - ) - }) - }) - .expect("a sketch candidate") -} - -/// The two placements under test: nothing stored, and every state stored at -/// ingestion time where the DAG allows it. -fn placements() -> [MaterializationAssignment; 2] { - [ - MaterializationAssignment::default(), - MaterializationAssignment::with_default(AT_INGESTION), - ] -} - -/// `root` timed under `assignment`, as Stage 2 times a plan: a node read at -/// two timings (Pass 1 composes identical inputs as one memo group's node, -/// e.g. the scan of two quantiles of one metric) is split first, one copy -/// per timing. -fn timed_dag(root: &Rc, assignment: &MaterializationAssignment) -> PhysicalASAPDAG { - let (split, assignment) = split_shared_by_timing(std::slice::from_ref(root), assignment); - let timed = - apply_materialization_timings(&split[0], &assignment, &mut NodeToTimedNodeCache::new()) - .unwrap(); - compile_physical_asap_dag(&timed).unwrap() -} - -fn states(dag: &PhysicalASAPDAG) -> Vec { - dag.nodes - .iter() - .filter(|node| node.materialization == AT_INGESTION) - .map(|node| node.id as u64) - .collect() -} - -fn floats(result: &[Vec]) -> Vec { - result[0] - .iter() - .flat_map(|batch| batch.to_rows()) - .flatten() - .filter_map(|value| match value { - Value::Float64(value) => Some(value), - _ => None, - }) - .collect() -} - -/// Execute a timed DAG over one raw-source batch per `Scan` -/// (`batch(schema, metric)`): each refresh group of materialized views -/// refreshed in one ingestion run over `ingestion`, then the query at `evaluation_time_ms` -/// reading the views' states. Returns the root's values and the views. -fn execute( - dag: &PhysicalASAPDAG, - batch: &dyn Fn(&SchemaRef, &str) -> Batch, - ingestion: Scope, - evaluation_time_ms: i64, -) -> (Vec, Vec) { - let mut dag = dag.clone(); - let views = dag.set_retention(&HashMap::new(), &HashMap::new()).unwrap(); - let raw = |dag: &PhysicalASAPDAG| { - physical_common::raw_data_store(dag, |source, schema| { - let Source::TimeSeries { metric } = source else { - panic!("a time series source") - }; - batch(schema, metric).to_rows() - }) - }; - // Each refresh group's ingestion run refreshes its views from the raw - // data, compiled once with all of them as roots. - let mut store = raw(&dag); - for group in dag.ingestion_refresh_groups().unwrap() { - let roots: Vec = group.views.iter().map(|view| view.node_id as u64).collect(); - let refresh = compile(&dag, &[], &roots).unwrap(); - let states = physical_common::execute(&refresh, raw(&dag), ingestion.clone()); - for (view, states) in group.views.iter().zip(states) { - store = store.with_materialized_view(view.node_id, states).unwrap(); - } - } - let query = compile(&dag, &views, &[dag.roots[0] as u64]).unwrap(); - let result = physical_common::execute(&query, store, Scope::Query { evaluation_time_ms }); - ( - floats(&result), - views.iter().map(|view| view.node_id as u64).collect(), - ) -} - -/// `(metric, job, seconds, value)` samples as one batch per metric. -fn job_samples<'a>( - samples: &'a [(&'a str, &'a str, i64, f64)], -) -> impl Fn(&SchemaRef, &str) -> Batch + 'a { - move |schema, name| { - let rows = samples - .iter() - .filter(|sample| sample.0 == name) - .map(|(metric, job, seconds, value)| { - let labels = BTreeMap::from([ - ("__name__".to_string(), metric.to_string()), - ("job".to_string(), job.to_string()), - ]); - promql_rows::series_row(schema, &labels, seconds * 1000, *value).unwrap() - }) - .collect(); - Batch::try_new(schema.clone(), rows).unwrap() - } -} - -/// Values 1..=100 at 300 s, one row each, in every Float64 column of -/// `schema`. -fn values_1_to_100(schema: &SchemaRef, _metric: &str) -> Batch { - let rows = (1..=100) - .map(|value| { - schema - .fields - .iter() - .map(|field| match field.dtype { - asap_types::ir::schema::FieldDataType::Plain( - asap_types::ir::schema::DataType::Float64, - ) => Value::Float64(f64::from(value)), - asap_types::ir::schema::FieldDataType::Plain( - asap_types::ir::schema::DataType::Timestamp, - ) => Value::Timestamp(300_000), - _ => panic!("unexpected field {field:?}"), - }) - .collect() - }) - .collect(); - Batch::try_new(schema.clone(), rows).unwrap() -} - -/// `MatAtIngestion` puts the quantile's state in a materialized view; -/// `NotMaterialized` has no view and reads the raw source at -/// query time; both answer alike. -#[test] -fn materialization_decides_precompute_contents() { - let root = sketch_candidate("quantile(0.99, latency)"); - let mut answers = Vec::new(); - for assignment in placements() { - let dag = timed_dag(&root, &assignment); - let ingestion = Scope::Ingestion { - window_start_ms: 0, - window_end_ms: 300_000, - }; - let (values, frontier) = execute(&dag, &values_1_to_100, ingestion, 300_000); - assert_eq!(frontier, states(&dag)); - answers.push(values); - } - assert_eq!(answers[1].len(), 1); - assert!(!states(&timed_dag(&root, &placements()[1])).is_empty()); - assert_eq!(answers[0], answers[1]); -} - -/// Mixed placement in one DAG: one summary `MatAtIngestion` (a materialized view) -/// and the other `NotMaterialized` (built from the raw source at query -/// time) answer like the all-query-time placement. -#[test] -fn mixed_placement_answers_like_query_time() { - let accuracy = AccuracyTarget::max_error(0.01); - let root = lower_promql( - "quantile(0.99, latency) + quantile(0.5, latency)", - accuracy.clone(), - ) - .unwrap(); - let sketches = |candidate: &Rc| -> Vec> { - OperatorNode::reachable(candidate) - .into_iter() - .filter(|node| { - matches!( - node.asap(), - Some(ASAPOp::SummaryAgg { - family: asap_types::ir::schema::FieldDataType::Sketch(..), - .. - }) - ) - }) - .collect() - }; - let root = explicit_summary_candidate(root, asap_types::ir::schema::SketchAlgorithm::Kll); - assert_eq!( - sketches(&root).len(), - 2, - "two explicitly selected KLL builds" - ); - let mut mixed = MaterializationAssignment::default(); - mixed.set(&sketches(&root)[0], AT_INGESTION); - let mut answers = Vec::new(); - for assignment in [MaterializationAssignment::default(), mixed] { - let dag = timed_dag(&root, &assignment); - let ingestion = Scope::Ingestion { - window_start_ms: 0, - window_end_ms: 300_000, - }; - let (values, frontier) = execute(&dag, &values_1_to_100, ingestion, 300_000); - assert_eq!(frontier, states(&dag)); - answers.push((values, frontier.len())); - } - // Only the assigned sketch is stored. - assert_eq!(answers[0].1, 0); - assert_eq!(answers[1].1, 1); - assert_eq!(answers[1].0.len(), 1); - assert_eq!(answers[0].0, answers[1].0); -} - -/// Whether the DAG has an ingestion-time `BinaryOp`. -fn ingestion_binary(dag: &PhysicalASAPDAG) -> bool { - dag.nodes.iter().any(|node| { - matches!( - node.payload, - PhysicalASAPOperatorPayload::NonASAP(NonASAPOp::BinaryOp { .. }) - ) && node.output_state.timing == ExecutionTiming::IngestionTime - }) -} - -const INGESTION_WINDOW: Scope = Scope::Ingestion { - window_start_ms: 240_000, - window_end_ms: 300_000, -}; - -/// Prometheus drops series without a match: arithmetic over different -/// selectors keeps only label sets present on both sides (none when -/// disjoint), at either placement, and never becomes aligned maintenance. -#[test] -fn arithmetic_over_different_selectors_matches_prometheus() { - // The selected root reads the outer `sum`'s exact state through its - // finalize. - let root = selected("sum(sum_over_time(m[1m]) + sum_over_time(n[1m]))", true); - assert!(matches!( - root.asap(), - Some(ASAPOp::FinalizeExactAccumulator { .. }) - )); - let disjoint = [ - ("m", "a", 250, 1.0), - ("m", "a", 290, 2.0), - ("n", "b", 250, 5.0), - ]; - // Only job a is on both sides: m_a + n_a = (1 + 2) + 7; m{job="b"} is dropped. - let overlapping = [ - ("m", "a", 250, 1.0), - ("m", "a", 290, 2.0), - ("m", "b", 250, 5.0), - ("n", "a", 250, 7.0), - ]; - for assignment in placements() { - let dag = timed_dag(&root, &assignment); - assert!(!ingestion_binary(&dag)); - let (values, _) = execute(&dag, &job_samples(&disjoint), INGESTION_WINDOW, 300_000); - assert!(values.is_empty(), "{values:?}"); - let (values, _) = execute(&dag, &job_samples(&overlapping), INGESTION_WINDOW, 300_000); - assert_eq!(values, [10.0]); - } - let root = selected_quantile( - "quantile(0.9, sum_over_time(m[1m]) + sum_over_time(n[1m]))", - true, - ); - assert!(!ingestion_binary(&timed_dag(&root, &placements()[1]))); -} - -/// Arithmetic over one selector is aligned maintenance when stored at -/// ingestion time, and adds each series' two evaluations before the -/// quantile at either placement. -#[test] -#[ignore = "the two identical selectors stay separate until Pass 2 merges identical sub-DAGs (#580)"] -fn arithmetic_over_one_selector_executes() { - let root = selected_quantile( - "quantile(0.9, sum_over_time(m[1m]) + sum_over_time(m[1m]))", - true, - ); - let samples = [ - ("m", "a", 250, 1.0), - ("m", "a", 290, 2.0), - ("m", "b", 250, 5.0), - ]; - let [not_materialized, at_ingestion] = placements().map(|a| timed_dag(&root, &a)); - assert!(!ingestion_binary(¬_materialized)); - assert!( - ingestion_binary(&at_ingestion), - "one selector shares its key set" - ); - for dag in [not_materialized, at_ingestion] { - let (values, _) = execute(&dag, &job_samples(&samples), INGESTION_WINDOW, 300_000); - // job a: 3 + 3 = 6; job b: 5 + 5 = 10 (mispairing a with b gives 8 and 8). - // KLL at epsilon 0.01 returns an input value within 0.01 of rank 0.9; - // of two values only the larger is. - assert_eq!(values, [10.0]); - } -} diff --git a/crates/integration-tests/tests/materialized_view_fixtures/plan_wire_version_10.json b/crates/integration-tests/tests/materialized_view_fixtures/plan_wire_version_10.json index 515db4302..d2e26ea78 100644 --- a/crates/integration-tests/tests/materialized_view_fixtures/plan_wire_version_10.json +++ b/crates/integration-tests/tests/materialized_view_fixtures/plan_wire_version_10.json @@ -1 +1 @@ -{"dag":{"schema_version":10,"dag":{"edges":[{"consumer":1,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":2,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":1,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":3,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"producer":2,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":4,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"producer":3,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":5,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"producer":4,"role":"Input","window":"NotApplicable"},{"consumer":6,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"producer":2,"role":"Input","window":"NotApplicable"}],"nodes":[{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":0,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"source":{"TimeSeries":{"metric":"m"}}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":1,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":2,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":1,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[2,3]}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":3,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":2}}}},{"guarantee":null,"id":4,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":3,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":{"Named":"sum"}},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[1]}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":5,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":4}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":6,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":2}}}}],"roots":[5,6],"scalar_roots":[]}},"queries":[{"query":"sum by (job) (sum_over_time(m[1m]))","root_node_id":5},{"query":"sum by (job, instance) (sum_over_time(m[1m]))","root_node_id":6}],"stored_outputs":[{"built_at":"ingestion_run","coverage":{"definition_id":"db0c9241a79202b6925e84aef010e89a","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":0,"scan_node_id":0,"time_range_node_id":1,"width_ms":60000}],"stored_outputs":[]},"node_id":2,"retention_ms":60000},{"built_at":"ingestion_run","coverage":{"definition_id":"3c90a6b5ff3ecaa445ea9bf285e1dc0b","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[],"stored_outputs":[2]},"node_id":4,"retention_ms":60000}],"query_time_raw_windows":[],"selection":null} +{"dag": {"schema_version": 10, "dag": {"edges": [{"consumer": 1, "data_state": {"primitive": "Raw", "timing": "ingestion_time"}, "grouping": "NotApplicable", "intermediate_schema": {"closed": true, "fields": [{"dtype": {"Plain": "timestamp"}, "name": "ts", "nullable": false, "table": null}, {"dtype": {"Plain": "float64"}, "name": "value", "nullable": false, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "$promql_series_identity", "nullable": false, "table": null}], "time_index": 0, "unique_keys": [[4, 0]]}, "producer": 0, "role": "Input", "window": "RequiresAlignedPanePhaseOrExactBoundaryResidual"}, {"consumer": 2, "data_state": {"primitive": "Raw", "timing": "ingestion_time"}, "grouping": "NotApplicable", "intermediate_schema": {"closed": true, "fields": [{"dtype": {"Plain": "timestamp"}, "name": "ts", "nullable": false, "table": null}, {"dtype": {"Plain": "float64"}, "name": "value", "nullable": false, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "$promql_series_identity", "nullable": false, "table": null}], "time_index": 0, "unique_keys": [[4, 0]]}, "producer": 1, "role": "Input", "window": "RequiresAlignedPanePhaseOrExactBoundaryResidual"}, {"consumer": 3, "data_state": {"primitive": "SummaryState", "timing": "ingestion_time"}, "grouping": "NotApplicable", "intermediate_schema": {"closed": true, "fields": [{"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"ExactAggregate": ["Sum", "Sum"]}, "name": "state", "nullable": false, "table": null}], "time_index": null, "unique_keys": [[0, 1]]}, "producer": 2, "role": "Input", "window": "RequiresAlignedPanePhaseOrExactBoundaryResidual"}, {"consumer": 4, "data_state": {"primitive": "Raw", "timing": "ingestion_time"}, "grouping": "NotApplicable", "intermediate_schema": {"closed": true, "fields": [{"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"Plain": "float64"}, "name": "sum", "nullable": false, "table": null}], "time_index": null, "unique_keys": [[0, 1]]}, "producer": 3, "role": "Input", "window": "RequiresAlignedPanePhaseOrExactBoundaryResidual"}, {"consumer": 5, "data_state": {"primitive": "SummaryState", "timing": "ingestion_time"}, "grouping": "NotApplicable", "intermediate_schema": {"closed": true, "fields": [{"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"ExactAggregate": ["Sum", "Sum"]}, "name": "state", "nullable": false, "table": null}], "time_index": null, "unique_keys": [[0]]}, "producer": 4, "role": "Input", "window": "NotApplicable"}, {"consumer": 6, "data_state": {"primitive": "SummaryState", "timing": "ingestion_time"}, "grouping": "NotApplicable", "intermediate_schema": {"closed": true, "fields": [{"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"ExactAggregate": ["Sum", "Sum"]}, "name": "state", "nullable": false, "table": null}], "time_index": null, "unique_keys": [[0, 1]]}, "producer": 2, "role": "Input", "window": "NotApplicable"}], "nodes": [{"guarantee": {"provenance": [{"kind": "exact", "reason": "exact over exact inputs"}], "spec": {"accuracy_metric": "absolute", "accuracy_scope": "per_value", "error": {"op": "zero"}, "failure_probability": {"op": "zero"}}}, "id": 0, "lifetime": "None", "output_schema": {"closed": true, "fields": [{"dtype": {"Plain": "timestamp"}, "name": "ts", "nullable": false, "table": null}, {"dtype": {"Plain": "float64"}, "name": "value", "nullable": false, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "$promql_series_identity", "nullable": false, "table": null}], "time_index": 0, "unique_keys": [[4, 0]]}, "output_state": {"primitive": "Raw", "timing": "ingestion_time"}, "payload": {"NonASAP": {"Scan": {"predicates": [], "schema": {"closed": true, "fields": [{"dtype": {"Plain": "timestamp"}, "name": "ts", "nullable": false, "table": null}, {"dtype": {"Plain": "float64"}, "name": "value", "nullable": false, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "$promql_series_identity", "nullable": false, "table": null}], "time_index": 0, "unique_keys": [[4, 0]]}, "source": {"TimeSeries": {"metric": "m"}}}}}}, {"guarantee": {"provenance": [{"kind": "exact", "reason": "exact over exact inputs"}], "spec": {"accuracy_metric": "absolute", "accuracy_scope": "per_value", "error": {"op": "zero"}, "failure_probability": {"op": "zero"}}}, "id": 1, "lifetime": "None", "output_schema": {"closed": true, "fields": [{"dtype": {"Plain": "timestamp"}, "name": "ts", "nullable": false, "table": null}, {"dtype": {"Plain": "float64"}, "name": "value", "nullable": false, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "$promql_series_identity", "nullable": false, "table": null}], "time_index": 0, "unique_keys": [[4, 0]]}, "output_state": {"primitive": "Raw", "timing": "ingestion_time"}, "payload": {"NonASAP": {"TimeRange": {"child": 0, "kind": "range", "range": {"nanos": 0, "secs": 60}}}}}, {"guarantee": null, "id": 2, "lifetime": {"AcrossEvaluations": "Memory"}, "output_schema": {"closed": true, "fields": [{"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"ExactAggregate": ["Sum", "Sum"]}, "name": "state", "nullable": false, "table": null}], "time_index": null, "unique_keys": [[0, 1]]}, "output_state": {"primitive": "SummaryState", "timing": "ingestion_time"}, "payload": {"ASAP": {"SummaryAgg": {"child": 1, "family": {"ExactAggregate": ["Sum", "Sum"]}, "filter": null, "grouping": "PerSubpopulationInstance", "input": {"item": null, "weight": {"Column": "SampleValue"}, "weight_domain": {"kind": "unknown_or_signed"}}, "reduction": {"Reduce": [2, 3]}}}}}, {"guarantee": {"provenance": [{"kind": "exact", "reason": "exact accumulator over exact input"}], "spec": {"accuracy_metric": "absolute", "accuracy_scope": "per_value", "error": {"op": "zero"}, "failure_probability": {"op": "zero"}}}, "id": 3, "lifetime": "None", "output_schema": {"closed": true, "fields": [{"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"Plain": "float64"}, "name": "sum", "nullable": false, "table": null}], "time_index": null, "unique_keys": [[0, 1]]}, "output_state": {"primitive": "Raw", "timing": "ingestion_time"}, "payload": {"ASAP": {"FinalizeExactAccumulator": {"child": 2}}}}, {"guarantee": null, "id": 4, "lifetime": {"AcrossEvaluations": "Memory"}, "output_schema": {"closed": true, "fields": [{"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"ExactAggregate": ["Sum", "Sum"]}, "name": "state", "nullable": false, "table": null}], "time_index": null, "unique_keys": [[0]]}, "output_state": {"primitive": "SummaryState", "timing": "ingestion_time"}, "payload": {"ASAP": {"SummaryAgg": {"child": 3, "family": {"ExactAggregate": ["Sum", "Sum"]}, "filter": null, "grouping": "PerSubpopulationInstance", "input": {"item": null, "weight": {"Column": {"Named": "sum"}}, "weight_domain": {"kind": "unknown_or_signed"}}, "reduction": {"Reduce": [1]}}}}}, {"guarantee": {"provenance": [{"kind": "exact", "reason": "exact accumulator over exact input"}], "spec": {"accuracy_metric": "absolute", "accuracy_scope": "per_value", "error": {"op": "zero"}, "failure_probability": {"op": "zero"}}}, "id": 5, "lifetime": "None", "output_schema": {"closed": true, "fields": [{"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"Plain": "float64"}, "name": "sum", "nullable": false, "table": null}], "time_index": null, "unique_keys": [[0]]}, "output_state": {"primitive": "Raw", "timing": "query_time"}, "payload": {"ASAP": {"FinalizeExactAccumulator": {"child": 4}}}}, {"guarantee": {"provenance": [{"kind": "exact", "reason": "exact accumulator over exact input"}], "spec": {"accuracy_metric": "absolute", "accuracy_scope": "per_value", "error": {"op": "zero"}, "failure_probability": {"op": "zero"}}}, "id": 6, "lifetime": "None", "output_schema": {"closed": true, "fields": [{"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"Plain": "float64"}, "name": "sum", "nullable": false, "table": null}], "time_index": null, "unique_keys": [[0, 1]]}, "output_state": {"primitive": "Raw", "timing": "query_time"}, "payload": {"ASAP": {"FinalizeExactAccumulator": {"child": 2}}}}], "roots": [5, 6], "scalar_roots": []}}, "queries": [{"query": "sum by (job) (sum_over_time(m[1m]))", "root_node_id": 5}, {"query": "sum by (job, instance) (sum_over_time(m[1m]))", "root_node_id": 6}], "stored_outputs": [{"built_at": "ingestion_run", "coverage": {"definition_id": "db0c9241a79202b6925e84aef010e89a", "selection": [{"columns": [], "relative_time": [{"excluded": -60000}, {"included": 0}]}]}, "inputs": {"raw_sample_windows": [{"end_offset_ms": 0, "scan_node_id": 0, "time_range_node_id": 1, "width_ms": 60000}], "stored_outputs": []}, "node_id": 2, "retention_ms": 60000}, {"built_at": "ingestion_run", "coverage": {"definition_id": "3c90a6b5ff3ecaa445ea9bf285e1dc0b", "selection": [{"columns": [], "relative_time": [{"excluded": -60000}, {"included": 0}]}]}, "inputs": {"raw_sample_windows": [], "stored_outputs": [2]}, "node_id": 4, "retention_ms": 60000}], "query_time_raw_windows": [], "selection": null} diff --git a/crates/integration-tests/tests/materialized_view_fixtures/stored_outputs_on_main.json b/crates/integration-tests/tests/materialized_view_fixtures/stored_outputs_on_main.json index 3a73c4fcf..f822d6b70 100644 --- a/crates/integration-tests/tests/materialized_view_fixtures/stored_outputs_on_main.json +++ b/crates/integration-tests/tests/materialized_view_fixtures/stored_outputs_on_main.json @@ -1,10 +1,10 @@ [ -{"name":"repeating p99-60m/no_raw","dag":{"edges":[{"consumer":1,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":2,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":1,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":3,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":4,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":3,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":5,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":4,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":6,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":7,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":6,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":8,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":7,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":9,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":10,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":9,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":11,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":10,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":12,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":13,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":12,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":14,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":13,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":15,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":16,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":15,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":17,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":16,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":18,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":19,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":18,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":20,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":19,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":21,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":22,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":21,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":23,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":22,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":24,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":25,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":24,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":26,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":25,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":27,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":28,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":27,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":29,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":28,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":30,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":31,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":30,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":32,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":31,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":33,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":34,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":33,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":35,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":34,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":36,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":37,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":36,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":38,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":37,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":39,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":40,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":39,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":41,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":40,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":42,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":43,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":42,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":44,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":43,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":45,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":46,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":45,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":47,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":46,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":48,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":49,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":48,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":50,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":49,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":51,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":52,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":51,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":53,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":52,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":2,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":5,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":8,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":11,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":14,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":17,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":20,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":23,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":26,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":29,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":32,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":35,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":38,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":41,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":44,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":47,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":50,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":53,"role":"Input","window":"NotApplicable"},{"consumer":55,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":54,"role":"Input","window":"NotApplicable"},{"consumer":56,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"quantile","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":55,"role":"Input","window":"NotApplicable"}],"nodes":[{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":0,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"source":{"TimeSeries":{"metric":"latency_ms"}}}}}},{"guarantee":null,"id":1,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":2,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":1,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":3,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":200000}}}}},{"guarantee":null,"id":4,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":3,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":5,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":4,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":6,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":400000}}}}},{"guarantee":null,"id":7,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":6,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":8,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":7,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":9,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":600000}}}}},{"guarantee":null,"id":10,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":9,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":11,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":10,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":12,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":800000}}}}},{"guarantee":null,"id":13,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":12,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":14,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":13,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":15,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":1000000}}}}},{"guarantee":null,"id":16,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":15,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":17,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":16,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":18,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":1200000}}}}},{"guarantee":null,"id":19,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":18,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":20,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":19,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":21,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":1400000}}}}},{"guarantee":null,"id":22,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":21,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":23,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":22,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":24,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":1600000}}}}},{"guarantee":null,"id":25,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":24,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":26,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":25,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":27,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":1800000}}}}},{"guarantee":null,"id":28,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":27,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":29,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":28,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":30,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":2000000}}}}},{"guarantee":null,"id":31,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":30,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":32,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":31,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":33,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":2200000}}}}},{"guarantee":null,"id":34,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":33,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":35,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":34,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":36,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":2400000}}}}},{"guarantee":null,"id":37,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":36,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":38,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":37,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":39,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":2600000}}}}},{"guarantee":null,"id":40,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":39,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":41,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":40,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":42,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":2800000}}}}},{"guarantee":null,"id":43,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":42,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":44,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":43,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":45,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":3000000}}}}},{"guarantee":null,"id":46,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":45,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":47,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":46,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":48,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":3200000}}}}},{"guarantee":null,"id":49,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":48,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":50,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":49,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":51,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":3400000}}}}},{"guarantee":null,"id":52,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":51,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":53,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":52,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":54,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryMerge":{"children":[2,5,8,11,14,17,20,23,26,29,32,35,38,41,44,47,50,53]}}}},{"guarantee":{"provenance":[{"algorithm":"Kll","contract":"apache_datasketches_kll_empirical_99_delta_scaled_a9b42755072b","kind":"sketch_evaluation","params":{"Kll":{"failure_probability":1e-05,"k":469}},"query":"Quantile { q: 0.99 }"}],"spec":{"accuracy_metric":"rank","accuracy_scope":"per_value","error":{"op":"constant","value":0.009954464416322084},"failure_probability":{"op":"constant","value":1e-05}}},"id":55,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"quantile","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"SummaryEstimate":{"query":{"Quantile":{"q":0.99}},"summary_input":54}}}},{"guarantee":{"provenance":[{"algorithm":"Kll","contract":"apache_datasketches_kll_empirical_99_delta_scaled_a9b42755072b","kind":"sketch_evaluation","params":{"Kll":{"failure_probability":1e-05,"k":469}},"query":"Quantile { q: 0.99 }"}],"spec":{"accuracy_metric":"rank","accuracy_scope":"per_value","error":{"op":"constant","value":0.009954464416322084},"failure_probability":{"op":"constant","value":1e-05}}},"id":56,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"Project":{"child":55,"cols":[{"alias":"ts","expr":{"Column":0}},{"alias":"value","expr":{"Column":1}},{"alias":"$promql_series_identity","expr":{"Column":2}}],"qualifier":null}}}}],"roots":[56],"scalar_roots":[]},"stored_outputs":[{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-200000},{"included":0}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":0,"scan_node_id":0,"time_range_node_id":1,"width_ms":200000}],"stored_outputs":[]},"node_id":2,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-400000},{"included":-200000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":200000,"scan_node_id":0,"time_range_node_id":4,"width_ms":200000}],"stored_outputs":[]},"node_id":5,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-600000},{"included":-400000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":400000,"scan_node_id":0,"time_range_node_id":7,"width_ms":200000}],"stored_outputs":[]},"node_id":8,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-800000},{"included":-600000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":600000,"scan_node_id":0,"time_range_node_id":10,"width_ms":200000}],"stored_outputs":[]},"node_id":11,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-1000000},{"included":-800000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":800000,"scan_node_id":0,"time_range_node_id":13,"width_ms":200000}],"stored_outputs":[]},"node_id":14,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-1200000},{"included":-1000000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":1000000,"scan_node_id":0,"time_range_node_id":16,"width_ms":200000}],"stored_outputs":[]},"node_id":17,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-1400000},{"included":-1200000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":1200000,"scan_node_id":0,"time_range_node_id":19,"width_ms":200000}],"stored_outputs":[]},"node_id":20,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-1600000},{"included":-1400000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":1400000,"scan_node_id":0,"time_range_node_id":22,"width_ms":200000}],"stored_outputs":[]},"node_id":23,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-1800000},{"included":-1600000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":1600000,"scan_node_id":0,"time_range_node_id":25,"width_ms":200000}],"stored_outputs":[]},"node_id":26,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-2000000},{"included":-1800000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":1800000,"scan_node_id":0,"time_range_node_id":28,"width_ms":200000}],"stored_outputs":[]},"node_id":29,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-2200000},{"included":-2000000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":2000000,"scan_node_id":0,"time_range_node_id":31,"width_ms":200000}],"stored_outputs":[]},"node_id":32,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-2400000},{"included":-2200000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":2200000,"scan_node_id":0,"time_range_node_id":34,"width_ms":200000}],"stored_outputs":[]},"node_id":35,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-2600000},{"included":-2400000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":2400000,"scan_node_id":0,"time_range_node_id":37,"width_ms":200000}],"stored_outputs":[]},"node_id":38,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-2800000},{"included":-2600000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":2600000,"scan_node_id":0,"time_range_node_id":40,"width_ms":200000}],"stored_outputs":[]},"node_id":41,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-3000000},{"included":-2800000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":2800000,"scan_node_id":0,"time_range_node_id":43,"width_ms":200000}],"stored_outputs":[]},"node_id":44,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-3200000},{"included":-3000000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":3000000,"scan_node_id":0,"time_range_node_id":46,"width_ms":200000}],"stored_outputs":[]},"node_id":47,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-3400000},{"included":-3200000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":3200000,"scan_node_id":0,"time_range_node_id":49,"width_ms":200000}],"stored_outputs":[]},"node_id":50,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-3600000},{"included":-3400000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":3400000,"scan_node_id":0,"time_range_node_id":52,"width_ms":200000}],"stored_outputs":[]},"node_id":53,"retention_ms":3600000}]}, -{"name":"repeating offset-chain/no_raw","dag":{"edges":[{"consumer":1,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":2,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":1,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":3,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":2,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":4,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":5,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":4,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":6,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":5,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":7,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":8,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":7,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":9,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":8,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":10,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":11,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":10,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":12,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":11,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":13,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":14,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":13,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":15,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":14,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":16,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":17,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":16,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":18,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":17,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":19,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":20,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":19,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":21,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":20,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":22,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":23,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":22,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":24,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":23,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":25,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":26,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":25,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":27,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":26,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":28,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":29,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":28,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":30,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":29,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":3,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":6,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":9,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":12,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":15,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":18,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":21,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":24,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":27,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":30,"role":"Input","window":"NotApplicable"},{"consumer":32,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":31,"role":"Input","window":"NotApplicable"},{"consumer":33,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"quantile","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":32,"role":"Input","window":"NotApplicable"}],"nodes":[{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":0,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"source":{"TimeSeries":{"metric":"latency_ms"}}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":1,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":300000}}}}},{"guarantee":null,"id":2,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":1,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":3,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":2,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":4,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":360000}}}}},{"guarantee":null,"id":5,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":4,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":6,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":5,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":7,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":420000}}}}},{"guarantee":null,"id":8,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":7,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":9,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":8,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":10,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":480000}}}}},{"guarantee":null,"id":11,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":10,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":12,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":11,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":13,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":540000}}}}},{"guarantee":null,"id":14,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":13,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":15,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":14,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":16,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":600000}}}}},{"guarantee":null,"id":17,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":16,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":18,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":17,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":19,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":660000}}}}},{"guarantee":null,"id":20,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":19,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":21,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":20,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":22,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":720000}}}}},{"guarantee":null,"id":23,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":22,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":24,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":23,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":25,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":780000}}}}},{"guarantee":null,"id":26,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":25,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":27,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":26,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":28,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":840000}}}}},{"guarantee":null,"id":29,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":28,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":30,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":29,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":31,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryMerge":{"children":[3,6,9,12,15,18,21,24,27,30]}}}},{"guarantee":{"provenance":[{"algorithm":"Kll","contract":"apache_datasketches_kll_empirical_99_delta_scaled_a9b42755072b","kind":"sketch_evaluation","params":{"Kll":{"failure_probability":1e-05,"k":469}},"query":"Quantile { q: 0.99 }"}],"spec":{"accuracy_metric":"rank","accuracy_scope":"per_value","error":{"op":"constant","value":0.009954464416322084},"failure_probability":{"op":"constant","value":1e-05}}},"id":32,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"quantile","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"SummaryEstimate":{"query":{"Quantile":{"q":0.99}},"summary_input":31}}}},{"guarantee":{"provenance":[{"algorithm":"Kll","contract":"apache_datasketches_kll_empirical_99_delta_scaled_a9b42755072b","kind":"sketch_evaluation","params":{"Kll":{"failure_probability":1e-05,"k":469}},"query":"Quantile { q: 0.99 }"}],"spec":{"accuracy_metric":"rank","accuracy_scope":"per_value","error":{"op":"constant","value":0.009954464416322084},"failure_probability":{"op":"constant","value":1e-05}}},"id":33,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"Project":{"child":32,"cols":[{"alias":"ts","expr":{"Column":0}},{"alias":"value","expr":{"Column":1}},{"alias":"$promql_series_identity","expr":{"Column":2}}],"qualifier":null}}}}],"roots":[33],"scalar_roots":[]},"stored_outputs":[{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-360000},{"included":-300000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":300000,"scan_node_id":0,"time_range_node_id":2,"width_ms":60000}],"stored_outputs":[]},"node_id":3,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-420000},{"included":-360000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":360000,"scan_node_id":0,"time_range_node_id":5,"width_ms":60000}],"stored_outputs":[]},"node_id":6,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-480000},{"included":-420000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":420000,"scan_node_id":0,"time_range_node_id":8,"width_ms":60000}],"stored_outputs":[]},"node_id":9,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-540000},{"included":-480000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":480000,"scan_node_id":0,"time_range_node_id":11,"width_ms":60000}],"stored_outputs":[]},"node_id":12,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-600000},{"included":-540000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":540000,"scan_node_id":0,"time_range_node_id":14,"width_ms":60000}],"stored_outputs":[]},"node_id":15,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-660000},{"included":-600000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":600000,"scan_node_id":0,"time_range_node_id":17,"width_ms":60000}],"stored_outputs":[]},"node_id":18,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-720000},{"included":-660000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":660000,"scan_node_id":0,"time_range_node_id":20,"width_ms":60000}],"stored_outputs":[]},"node_id":21,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-780000},{"included":-720000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":720000,"scan_node_id":0,"time_range_node_id":23,"width_ms":60000}],"stored_outputs":[]},"node_id":24,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-840000},{"included":-780000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":780000,"scan_node_id":0,"time_range_node_id":26,"width_ms":60000}],"stored_outputs":[]},"node_id":27,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-900000},{"included":-840000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":840000,"scan_node_id":0,"time_range_node_id":29,"width_ms":60000}],"stored_outputs":[]},"node_id":30,"retention_ms":900000}]}, -{"name":"repeating rollup/no_raw","dag":{"edges":[{"consumer":1,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":2,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":1,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":3,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":4,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":3,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":5,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":4,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":6,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":7,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":6,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":8,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":7,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":9,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":10,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":9,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":11,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":10,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":12,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":13,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":12,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":14,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":13,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":15,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":16,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":15,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":17,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":16,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":18,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":19,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":18,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":20,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":19,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":21,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":22,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":21,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":23,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":22,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":24,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":25,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":24,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":26,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":25,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":27,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":28,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":27,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":29,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":28,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":2,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":5,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":8,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":11,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":14,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":17,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":20,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":23,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":26,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":29,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":30,"role":"Input","window":"NotApplicable"},{"consumer":32,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":31,"role":"Input","window":"NotApplicable"},{"consumer":33,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"producer":32,"role":"Input","window":"NotApplicable"},{"consumer":34,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"producer":33,"role":"Input","window":"NotApplicable"},{"consumer":35,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"producer":34,"role":"Input","window":"NotApplicable"}],"nodes":[{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":0,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"source":{"TimeSeries":{"metric":"latency_ms"}}}}}},{"guarantee":null,"id":1,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":2,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":1,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":3,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":60000}}}}},{"guarantee":null,"id":4,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":3,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":5,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":4,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":6,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":120000}}}}},{"guarantee":null,"id":7,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":6,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":8,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":7,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":9,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":180000}}}}},{"guarantee":null,"id":10,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":9,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":11,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":10,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":12,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":240000}}}}},{"guarantee":null,"id":13,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":12,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":14,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":13,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":15,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":300000}}}}},{"guarantee":null,"id":16,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":15,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":17,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":16,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":18,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":360000}}}}},{"guarantee":null,"id":19,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":18,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":20,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":19,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":21,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":420000}}}}},{"guarantee":null,"id":22,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":21,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":23,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":22,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":24,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":480000}}}}},{"guarantee":null,"id":25,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":24,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":26,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":25,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":27,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":540000}}}}},{"guarantee":null,"id":28,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":27,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":29,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":28,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":30,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryMerge":{"children":[2,5,8,11,14,17,20,23,26,29]}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":31,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":30}}}},{"guarantee":null,"id":32,"materialization":{"MatAtQuery":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":31,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[2]}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":33,"materialization":{"MatAtQuery":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":32}}}},{"guarantee":null,"id":34,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":33,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":{"Named":"sum"}},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[]}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":35,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":34}}}}],"roots":[33,35],"scalar_roots":[]},"stored_outputs":[{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":0,"scan_node_id":0,"time_range_node_id":1,"width_ms":60000}],"stored_outputs":[]},"node_id":2,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-120000},{"included":-60000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":60000,"scan_node_id":0,"time_range_node_id":4,"width_ms":60000}],"stored_outputs":[]},"node_id":5,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-180000},{"included":-120000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":120000,"scan_node_id":0,"time_range_node_id":7,"width_ms":60000}],"stored_outputs":[]},"node_id":8,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-240000},{"included":-180000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":180000,"scan_node_id":0,"time_range_node_id":10,"width_ms":60000}],"stored_outputs":[]},"node_id":11,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-300000},{"included":-240000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":240000,"scan_node_id":0,"time_range_node_id":13,"width_ms":60000}],"stored_outputs":[]},"node_id":14,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-360000},{"included":-300000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":300000,"scan_node_id":0,"time_range_node_id":16,"width_ms":60000}],"stored_outputs":[]},"node_id":17,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-420000},{"included":-360000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":360000,"scan_node_id":0,"time_range_node_id":19,"width_ms":60000}],"stored_outputs":[]},"node_id":20,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-480000},{"included":-420000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":420000,"scan_node_id":0,"time_range_node_id":22,"width_ms":60000}],"stored_outputs":[]},"node_id":23,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-540000},{"included":-480000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":480000,"scan_node_id":0,"time_range_node_id":25,"width_ms":60000}],"stored_outputs":[]},"node_id":26,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-600000},{"included":-540000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":540000,"scan_node_id":0,"time_range_node_id":28,"width_ms":60000}],"stored_outputs":[]},"node_id":29,"retention_ms":600000}]}, -{"name":"repeating rate-5m/no_raw","dag":{"edges":[{"consumer":1,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":2,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":1,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":3,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":4,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":3,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":5,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":4,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":6,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":7,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":6,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":8,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":7,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":9,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":10,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":9,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":11,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":10,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":12,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":13,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":12,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":14,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":13,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":2,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":5,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":8,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":11,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":14,"role":"Input","window":"NotApplicable"},{"consumer":16,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":15,"role":"Input","window":"NotApplicable"},{"consumer":17,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":16,"role":"Input","window":"NotApplicable"},{"consumer":18,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"producer":17,"role":"Input","window":"NotApplicable"},{"consumer":19,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":20,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":19,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":21,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":20,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":22,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":23,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":22,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":24,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":23,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":25,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":26,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":25,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":27,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":26,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":28,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":29,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":28,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":30,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":29,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":31,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":32,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":31,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":33,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":32,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":34,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":21,"role":"Input","window":"NotApplicable"},{"consumer":34,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":24,"role":"Input","window":"NotApplicable"},{"consumer":34,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":27,"role":"Input","window":"NotApplicable"},{"consumer":34,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":30,"role":"Input","window":"NotApplicable"},{"consumer":34,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":33,"role":"Input","window":"NotApplicable"},{"consumer":35,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":34,"role":"Input","window":"NotApplicable"},{"consumer":36,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":35,"role":"Input","window":"NotApplicable"},{"consumer":37,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"producer":36,"role":"Input","window":"NotApplicable"}],"nodes":[{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":0,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"source":{"TimeSeries":{"metric":"latency_ms"}}}}}},{"guarantee":null,"id":1,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":2,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":1,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":3,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":60000}}}}},{"guarantee":null,"id":4,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":3,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":5,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":4,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":6,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":120000}}}}},{"guarantee":null,"id":7,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":6,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":8,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":7,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":9,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":180000}}}}},{"guarantee":null,"id":10,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":9,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":11,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":10,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":12,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":240000}}}}},{"guarantee":null,"id":13,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":12,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":14,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":13,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":15,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryMerge":{"children":[2,5,8,11,14]}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":16,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":15}}}},{"guarantee":null,"id":17,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":16,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[]}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":18,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":17}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":19,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":3600000}}}}},{"guarantee":null,"id":20,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":19,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":21,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":20,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":22,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":3660000}}}}},{"guarantee":null,"id":23,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":22,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":24,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":23,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":25,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":3720000}}}}},{"guarantee":null,"id":26,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":25,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":27,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":26,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":28,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":3780000}}}}},{"guarantee":null,"id":29,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":28,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":30,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":29,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":31,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":3840000}}}}},{"guarantee":null,"id":32,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":31,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":33,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":32,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":34,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryMerge":{"children":[21,24,27,30,33]}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":35,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":34}}}},{"guarantee":null,"id":36,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":35,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[]}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":37,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":36}}}}],"roots":[18,37],"scalar_roots":[]},"stored_outputs":[{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":0,"scan_node_id":0,"time_range_node_id":1,"width_ms":60000}],"stored_outputs":[]},"node_id":2,"retention_ms":300000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-120000},{"included":-60000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":60000,"scan_node_id":0,"time_range_node_id":4,"width_ms":60000}],"stored_outputs":[]},"node_id":5,"retention_ms":300000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-180000},{"included":-120000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":120000,"scan_node_id":0,"time_range_node_id":7,"width_ms":60000}],"stored_outputs":[]},"node_id":8,"retention_ms":300000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-240000},{"included":-180000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":180000,"scan_node_id":0,"time_range_node_id":10,"width_ms":60000}],"stored_outputs":[]},"node_id":11,"retention_ms":300000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-300000},{"included":-240000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":240000,"scan_node_id":0,"time_range_node_id":13,"width_ms":60000}],"stored_outputs":[]},"node_id":14,"retention_ms":300000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-3660000},{"included":-3600000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":3600000,"scan_node_id":0,"time_range_node_id":20,"width_ms":60000}],"stored_outputs":[]},"node_id":21,"retention_ms":3900000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-3720000},{"included":-3660000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":3660000,"scan_node_id":0,"time_range_node_id":23,"width_ms":60000}],"stored_outputs":[]},"node_id":24,"retention_ms":3900000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-3780000},{"included":-3720000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":3720000,"scan_node_id":0,"time_range_node_id":26,"width_ms":60000}],"stored_outputs":[]},"node_id":27,"retention_ms":3900000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-3840000},{"included":-3780000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":3780000,"scan_node_id":0,"time_range_node_id":29,"width_ms":60000}],"stored_outputs":[]},"node_id":30,"retention_ms":3900000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-3900000},{"included":-3840000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":3840000,"scan_node_id":0,"time_range_node_id":32,"width_ms":60000}],"stored_outputs":[]},"node_id":33,"retention_ms":3900000}]}, -{"name":"runtime_install a_coarse_sum_rolls_up_from_the_stored_fine_sum","dag":{"edges":[{"consumer":1,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":2,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":1,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":3,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"producer":2,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":4,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"producer":3,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":5,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"producer":4,"role":"Input","window":"NotApplicable"},{"consumer":6,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"producer":2,"role":"Input","window":"NotApplicable"}],"nodes":[{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":0,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"source":{"TimeSeries":{"metric":"m"}}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":1,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":2,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":1,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[2,3]}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":3,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":2}}}},{"guarantee":null,"id":4,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":3,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":{"Named":"sum"}},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[1]}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":5,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":4}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":6,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":2}}}}],"roots":[5,6],"scalar_roots":[]},"stored_outputs":[{"built_at":"ingestion_run","coverage":{"definition_id":"db0c9241a79202b6925e84aef010e89a","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":0,"scan_node_id":0,"time_range_node_id":1,"width_ms":60000}],"stored_outputs":[]},"node_id":2,"retention_ms":60000},{"built_at":"ingestion_run","coverage":{"definition_id":"3c90a6b5ff3ecaa445ea9bf285e1dc0b","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[],"stored_outputs":[2]},"node_id":4,"retention_ms":60000}]}, -{"name":"runtime_install a_kept_sliding_window_merge_is_listed_and_built","dag":{"edges":[{"consumer":1,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"NotApplicable"},{"consumer":2,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":1,"role":"Input","window":"NotApplicable"},{"consumer":3,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"NotApplicable"},{"consumer":4,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":3,"role":"Input","window":"NotApplicable"},{"consumer":5,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":4,"role":"Input","window":"NotApplicable"},{"consumer":6,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"NotApplicable"},{"consumer":7,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":6,"role":"Input","window":"NotApplicable"},{"consumer":8,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":7,"role":"Input","window":"NotApplicable"},{"consumer":9,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"NotApplicable"},{"consumer":10,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":9,"role":"Input","window":"NotApplicable"},{"consumer":11,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":10,"role":"Input","window":"NotApplicable"},{"consumer":12,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"NotApplicable"},{"consumer":13,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":12,"role":"Input","window":"NotApplicable"},{"consumer":14,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":13,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":2,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":5,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":8,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":11,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":14,"role":"Input","window":"NotApplicable"},{"consumer":16,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":15,"role":"Input","window":"NotApplicable"}],"nodes":[{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":0,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"source":{"TimeSeries":{"metric":"m"}}}}}},{"guarantee":null,"id":1,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":2,"materialization":{"MatAtQueryKeptAcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":1,"family":{"ExactAggregate":["Max","Max"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":3,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":60000}}}}},{"guarantee":null,"id":4,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeRange":{"child":3,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":5,"materialization":{"MatAtQueryKeptAcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":4,"family":{"ExactAggregate":["Max","Max"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":6,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":120000}}}}},{"guarantee":null,"id":7,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeRange":{"child":6,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":8,"materialization":{"MatAtQueryKeptAcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":7,"family":{"ExactAggregate":["Max","Max"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":9,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":180000}}}}},{"guarantee":null,"id":10,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeRange":{"child":9,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":11,"materialization":{"MatAtQueryKeptAcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":10,"family":{"ExactAggregate":["Max","Max"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":12,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":240000}}}}},{"guarantee":null,"id":13,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeRange":{"child":12,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":14,"materialization":{"MatAtQueryKeptAcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":13,"family":{"ExactAggregate":["Max","Max"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":15,"materialization":{"MatAtQueryKeptAcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryMerge":{"children":[2,5,8,11,14]}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":16,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":15}}}}],"roots":[16],"scalar_roots":[]},"stored_outputs":[{"built_at":"query_run","coverage":{"definition_id":"71610a93a8af5ae7ee6ecc0839e28d3b","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":0,"scan_node_id":0,"time_range_node_id":1,"width_ms":60000}],"stored_outputs":[]},"node_id":2,"retention_ms":3900000},{"built_at":"query_run","coverage":{"definition_id":"71610a93a8af5ae7ee6ecc0839e28d3b","selection":[{"columns":[],"relative_time":[{"excluded":-120000},{"included":-60000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":60000,"scan_node_id":0,"time_range_node_id":4,"width_ms":60000}],"stored_outputs":[]},"node_id":5,"retention_ms":3900000},{"built_at":"query_run","coverage":{"definition_id":"71610a93a8af5ae7ee6ecc0839e28d3b","selection":[{"columns":[],"relative_time":[{"excluded":-180000},{"included":-120000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":120000,"scan_node_id":0,"time_range_node_id":7,"width_ms":60000}],"stored_outputs":[]},"node_id":8,"retention_ms":3900000},{"built_at":"query_run","coverage":{"definition_id":"71610a93a8af5ae7ee6ecc0839e28d3b","selection":[{"columns":[],"relative_time":[{"excluded":-240000},{"included":-180000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":180000,"scan_node_id":0,"time_range_node_id":10,"width_ms":60000}],"stored_outputs":[]},"node_id":11,"retention_ms":3900000},{"built_at":"query_run","coverage":{"definition_id":"71610a93a8af5ae7ee6ecc0839e28d3b","selection":[{"columns":[],"relative_time":[{"excluded":-300000},{"included":-240000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":240000,"scan_node_id":0,"time_range_node_id":13,"width_ms":60000}],"stored_outputs":[]},"node_id":14,"retention_ms":3900000},{"built_at":"query_run","coverage":{"definition_id":"71610a93a8af5ae7ee6ecc0839e28d3b","selection":[{"columns":[],"relative_time":[{"excluded":-300000},{"included":0}]}]},"inputs":{"raw_sample_windows":[],"stored_outputs":[2,5,8,11,14]},"node_id":15,"retention_ms":3900000}]}, -{"name":"runtime_install a_summary_across_series_from_per_series_summaries_is_listed_and_built","dag":{"edges":[{"consumer":1,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":2,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":1,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":3,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":2,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":4,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":3,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":5,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"producer":4,"role":"Input","window":"NotApplicable"},{"consumer":6,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":2,"role":"Input","window":"NotApplicable"},{"consumer":7,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":6,"role":"Input","window":"NotApplicable"}],"nodes":[{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":0,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"source":{"TimeSeries":{"metric":"m"}}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":1,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":2,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":1,"family":{"ExactAggregate":["Max","Max"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":3,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":2}}}},{"guarantee":null,"id":4,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":3,"family":{"ExactAggregate":["Max","Max"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[2]}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":5,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"max","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":4}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":6,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":2}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":7,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"Project":{"child":6,"cols":[{"alias":"ts","expr":{"Column":0}},{"alias":"value","expr":{"Column":1}},{"alias":"$promql_series_identity","expr":{"Column":3}}],"qualifier":null}}}}],"roots":[5,7],"scalar_roots":[]},"stored_outputs":[{"built_at":"ingestion_run","coverage":{"definition_id":"5555de6812a18c62345c199d5ba8f06e","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":0,"scan_node_id":0,"time_range_node_id":1,"width_ms":60000}],"stored_outputs":[]},"node_id":2,"retention_ms":60000},{"built_at":"ingestion_run","coverage":{"definition_id":"f35ee4e847802c130cbd5221a81f7ea1","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[],"stored_outputs":[2]},"node_id":4,"retention_ms":60000}]}, -{"name":"runtime_install group_outputs_from_one_stored_per_series_sum_are_listed_and_built","dag":{"edges":[{"consumer":1,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":2,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":1,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":3,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":2,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":4,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":3,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":5,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":4,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":6,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"producer":5,"role":"Input","window":"NotApplicable"},{"consumer":7,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":2,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":8,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":7,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":9,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":8,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":10,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"producer":9,"role":"Input","window":"NotApplicable"}],"nodes":[{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":0,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"source":{"TimeSeries":{"metric":"m"}}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":1,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":2,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":1,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":3,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":2}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":4,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Project":{"child":3,"cols":[{"alias":"ts","expr":{"Column":0}},{"alias":"value","expr":{"Column":1}},{"alias":"job","expr":{"Column":2}},{"alias":"$promql_series_identity","expr":{"Column":4}}],"qualifier":null}}}},{"guarantee":null,"id":5,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":4,"family":{"ExactAggregate":["Max","Max"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[2]}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":6,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"max","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":5}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":7,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":2}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":8,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Project":{"child":7,"cols":[{"alias":"ts","expr":{"Column":0}},{"alias":"value","expr":{"Column":1}},{"alias":"instance","expr":{"Column":3}},{"alias":"job","expr":{"Column":2}},{"alias":"$promql_series_identity","expr":{"Column":4}}],"qualifier":null}}}},{"guarantee":null,"id":9,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":8,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[2,3]}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":10,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":9}}}}],"roots":[6,10],"scalar_roots":[]},"stored_outputs":[{"built_at":"ingestion_run","coverage":{"definition_id":"672f965318f2bf2ba350748044719b9b","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":0,"scan_node_id":0,"time_range_node_id":1,"width_ms":60000}],"stored_outputs":[]},"node_id":2,"retention_ms":60000},{"built_at":"ingestion_run","coverage":{"definition_id":"7976839b25e74a9cfac2ab1d6d01a63a","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[],"stored_outputs":[2]},"node_id":5,"retention_ms":60000},{"built_at":"ingestion_run","coverage":{"definition_id":"84209f44520311bced823b9e98eaf85c","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[],"stored_outputs":[2]},"node_id":9,"retention_ms":60000}]} +{"name":"repeating p99-60m/no_raw","dag":{"edges":[{"consumer":1,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":2,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":1,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":3,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":4,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":3,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":5,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":4,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":6,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":7,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":6,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":8,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":7,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":9,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":10,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":9,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":11,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":10,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":12,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":13,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":12,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":14,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":13,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":15,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":16,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":15,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":17,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":16,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":18,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":19,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":18,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":20,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":19,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":21,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":22,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":21,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":23,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":22,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":24,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":25,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":24,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":26,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":25,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":27,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":28,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":27,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":29,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":28,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":30,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":31,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":30,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":32,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":31,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":33,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":34,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":33,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":35,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":34,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":36,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":37,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":36,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":38,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":37,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":39,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":40,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":39,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":41,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":40,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":42,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":43,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":42,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":44,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":43,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":45,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":46,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":45,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":47,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":46,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":48,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":49,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":48,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":50,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":49,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":51,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":52,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":51,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":53,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":52,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":2,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":5,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":8,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":11,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":14,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":17,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":20,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":23,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":26,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":29,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":32,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":35,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":38,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":41,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":44,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":47,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":50,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":53,"role":"Input","window":"NotApplicable"},{"consumer":55,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":54,"role":"Input","window":"NotApplicable"},{"consumer":56,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"quantile","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":55,"role":"Input","window":"NotApplicable"}],"nodes":[{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":0,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"source":{"TimeSeries":{"metric":"latency_ms"}}}}}},{"guarantee":null,"id":1,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":2,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":1,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":3,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":200000}}}}},{"guarantee":null,"id":4,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":3,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":5,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":4,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":6,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":400000}}}}},{"guarantee":null,"id":7,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":6,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":8,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":7,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":9,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":600000}}}}},{"guarantee":null,"id":10,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":9,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":11,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":10,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":12,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":800000}}}}},{"guarantee":null,"id":13,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":12,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":14,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":13,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":15,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":1000000}}}}},{"guarantee":null,"id":16,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":15,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":17,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":16,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":18,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":1200000}}}}},{"guarantee":null,"id":19,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":18,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":20,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":19,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":21,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":1400000}}}}},{"guarantee":null,"id":22,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":21,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":23,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":22,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":24,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":1600000}}}}},{"guarantee":null,"id":25,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":24,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":26,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":25,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":27,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":1800000}}}}},{"guarantee":null,"id":28,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":27,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":29,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":28,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":30,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":2000000}}}}},{"guarantee":null,"id":31,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":30,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":32,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":31,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":33,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":2200000}}}}},{"guarantee":null,"id":34,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":33,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":35,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":34,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":36,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":2400000}}}}},{"guarantee":null,"id":37,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":36,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":38,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":37,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":39,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":2600000}}}}},{"guarantee":null,"id":40,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":39,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":41,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":40,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":42,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":2800000}}}}},{"guarantee":null,"id":43,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":42,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":44,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":43,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":45,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":3000000}}}}},{"guarantee":null,"id":46,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":45,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":47,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":46,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":48,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":3200000}}}}},{"guarantee":null,"id":49,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":48,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":50,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":49,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":51,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":3400000}}}}},{"guarantee":null,"id":52,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":51,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":53,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":52,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":54,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryMerge":{"children":[2,5,8,11,14,17,20,23,26,29,32,35,38,41,44,47,50,53]}}}},{"guarantee":{"provenance":[{"algorithm":"Kll","contract":"apache_datasketches_kll_empirical_99_delta_scaled_a9b42755072b","kind":"sketch_evaluation","params":{"Kll":{"failure_probability":1e-05,"k":469}},"query":"Quantile { q: 0.99 }"}],"spec":{"accuracy_metric":"rank","accuracy_scope":"per_value","error":{"op":"constant","value":0.009954464416322084},"failure_probability":{"op":"constant","value":1e-05}}},"id":55,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"quantile","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"SummaryEstimate":{"query":{"Quantile":{"q":0.99}},"summary_input":54}}}},{"guarantee":{"provenance":[{"algorithm":"Kll","contract":"apache_datasketches_kll_empirical_99_delta_scaled_a9b42755072b","kind":"sketch_evaluation","params":{"Kll":{"failure_probability":1e-05,"k":469}},"query":"Quantile { q: 0.99 }"}],"spec":{"accuracy_metric":"rank","accuracy_scope":"per_value","error":{"op":"constant","value":0.009954464416322084},"failure_probability":{"op":"constant","value":1e-05}}},"id":56,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"Project":{"child":55,"cols":[{"alias":"ts","expr":{"Column":0}},{"alias":"value","expr":{"Column":1}},{"alias":"$promql_series_identity","expr":{"Column":2}}],"qualifier":null}}}}],"roots":[56],"scalar_roots":[]},"stored_outputs":[{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-200000},{"included":0}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":0,"scan_node_id":0,"time_range_node_id":1,"width_ms":200000}],"stored_outputs":[]},"node_id":2,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-400000},{"included":-200000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":200000,"scan_node_id":0,"time_range_node_id":4,"width_ms":200000}],"stored_outputs":[]},"node_id":5,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-600000},{"included":-400000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":400000,"scan_node_id":0,"time_range_node_id":7,"width_ms":200000}],"stored_outputs":[]},"node_id":8,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-800000},{"included":-600000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":600000,"scan_node_id":0,"time_range_node_id":10,"width_ms":200000}],"stored_outputs":[]},"node_id":11,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-1000000},{"included":-800000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":800000,"scan_node_id":0,"time_range_node_id":13,"width_ms":200000}],"stored_outputs":[]},"node_id":14,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-1200000},{"included":-1000000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":1000000,"scan_node_id":0,"time_range_node_id":16,"width_ms":200000}],"stored_outputs":[]},"node_id":17,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-1400000},{"included":-1200000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":1200000,"scan_node_id":0,"time_range_node_id":19,"width_ms":200000}],"stored_outputs":[]},"node_id":20,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-1600000},{"included":-1400000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":1400000,"scan_node_id":0,"time_range_node_id":22,"width_ms":200000}],"stored_outputs":[]},"node_id":23,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-1800000},{"included":-1600000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":1600000,"scan_node_id":0,"time_range_node_id":25,"width_ms":200000}],"stored_outputs":[]},"node_id":26,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-2000000},{"included":-1800000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":1800000,"scan_node_id":0,"time_range_node_id":28,"width_ms":200000}],"stored_outputs":[]},"node_id":29,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-2200000},{"included":-2000000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":2000000,"scan_node_id":0,"time_range_node_id":31,"width_ms":200000}],"stored_outputs":[]},"node_id":32,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-2400000},{"included":-2200000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":2200000,"scan_node_id":0,"time_range_node_id":34,"width_ms":200000}],"stored_outputs":[]},"node_id":35,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-2600000},{"included":-2400000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":2400000,"scan_node_id":0,"time_range_node_id":37,"width_ms":200000}],"stored_outputs":[]},"node_id":38,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-2800000},{"included":-2600000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":2600000,"scan_node_id":0,"time_range_node_id":40,"width_ms":200000}],"stored_outputs":[]},"node_id":41,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-3000000},{"included":-2800000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":2800000,"scan_node_id":0,"time_range_node_id":43,"width_ms":200000}],"stored_outputs":[]},"node_id":44,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-3200000},{"included":-3000000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":3000000,"scan_node_id":0,"time_range_node_id":46,"width_ms":200000}],"stored_outputs":[]},"node_id":47,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-3400000},{"included":-3200000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":3200000,"scan_node_id":0,"time_range_node_id":49,"width_ms":200000}],"stored_outputs":[]},"node_id":50,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-3600000},{"included":-3400000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":3400000,"scan_node_id":0,"time_range_node_id":52,"width_ms":200000}],"stored_outputs":[]},"node_id":53,"retention_ms":3600000}]}, +{"name":"repeating offset-chain/no_raw","dag":{"edges":[{"consumer":1,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":2,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":1,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":3,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":2,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":4,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":5,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":4,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":6,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":5,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":7,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":8,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":7,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":9,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":8,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":10,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":11,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":10,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":12,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":11,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":13,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":14,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":13,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":15,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":14,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":16,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":17,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":16,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":18,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":17,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":19,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":20,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":19,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":21,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":20,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":22,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":23,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":22,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":24,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":23,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":25,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":26,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":25,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":27,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":26,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":28,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":29,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":28,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":30,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":29,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":3,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":6,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":9,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":12,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":15,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":18,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":21,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":24,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":27,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":30,"role":"Input","window":"NotApplicable"},{"consumer":32,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":31,"role":"Input","window":"NotApplicable"},{"consumer":33,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"quantile","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":32,"role":"Input","window":"NotApplicable"}],"nodes":[{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":0,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"source":{"TimeSeries":{"metric":"latency_ms"}}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":1,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":300000}}}}},{"guarantee":null,"id":2,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":1,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":3,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":2,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":4,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":360000}}}}},{"guarantee":null,"id":5,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":4,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":6,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":5,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":7,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":420000}}}}},{"guarantee":null,"id":8,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":7,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":9,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":8,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":10,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":480000}}}}},{"guarantee":null,"id":11,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":10,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":12,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":11,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":13,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":540000}}}}},{"guarantee":null,"id":14,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":13,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":15,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":14,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":16,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":600000}}}}},{"guarantee":null,"id":17,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":16,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":18,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":17,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":19,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":660000}}}}},{"guarantee":null,"id":20,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":19,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":21,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":20,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":22,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":720000}}}}},{"guarantee":null,"id":23,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":22,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":24,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":23,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":25,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":780000}}}}},{"guarantee":null,"id":26,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":25,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":27,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":26,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":28,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":840000}}}}},{"guarantee":null,"id":29,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":28,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":30,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":29,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":31,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryMerge":{"children":[3,6,9,12,15,18,21,24,27,30]}}}},{"guarantee":{"provenance":[{"algorithm":"Kll","contract":"apache_datasketches_kll_empirical_99_delta_scaled_a9b42755072b","kind":"sketch_evaluation","params":{"Kll":{"failure_probability":1e-05,"k":469}},"query":"Quantile { q: 0.99 }"}],"spec":{"accuracy_metric":"rank","accuracy_scope":"per_value","error":{"op":"constant","value":0.009954464416322084},"failure_probability":{"op":"constant","value":1e-05}}},"id":32,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"quantile","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"SummaryEstimate":{"query":{"Quantile":{"q":0.99}},"summary_input":31}}}},{"guarantee":{"provenance":[{"algorithm":"Kll","contract":"apache_datasketches_kll_empirical_99_delta_scaled_a9b42755072b","kind":"sketch_evaluation","params":{"Kll":{"failure_probability":1e-05,"k":469}},"query":"Quantile { q: 0.99 }"}],"spec":{"accuracy_metric":"rank","accuracy_scope":"per_value","error":{"op":"constant","value":0.009954464416322084},"failure_probability":{"op":"constant","value":1e-05}}},"id":33,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"Project":{"child":32,"cols":[{"alias":"ts","expr":{"Column":0}},{"alias":"value","expr":{"Column":1}},{"alias":"$promql_series_identity","expr":{"Column":2}}],"qualifier":null}}}}],"roots":[33],"scalar_roots":[]},"stored_outputs":[{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-360000},{"included":-300000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":300000,"scan_node_id":0,"time_range_node_id":2,"width_ms":60000}],"stored_outputs":[]},"node_id":3,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-420000},{"included":-360000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":360000,"scan_node_id":0,"time_range_node_id":5,"width_ms":60000}],"stored_outputs":[]},"node_id":6,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-480000},{"included":-420000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":420000,"scan_node_id":0,"time_range_node_id":8,"width_ms":60000}],"stored_outputs":[]},"node_id":9,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-540000},{"included":-480000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":480000,"scan_node_id":0,"time_range_node_id":11,"width_ms":60000}],"stored_outputs":[]},"node_id":12,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-600000},{"included":-540000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":540000,"scan_node_id":0,"time_range_node_id":14,"width_ms":60000}],"stored_outputs":[]},"node_id":15,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-660000},{"included":-600000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":600000,"scan_node_id":0,"time_range_node_id":17,"width_ms":60000}],"stored_outputs":[]},"node_id":18,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-720000},{"included":-660000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":660000,"scan_node_id":0,"time_range_node_id":20,"width_ms":60000}],"stored_outputs":[]},"node_id":21,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-780000},{"included":-720000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":720000,"scan_node_id":0,"time_range_node_id":23,"width_ms":60000}],"stored_outputs":[]},"node_id":24,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-840000},{"included":-780000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":780000,"scan_node_id":0,"time_range_node_id":26,"width_ms":60000}],"stored_outputs":[]},"node_id":27,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-900000},{"included":-840000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":840000,"scan_node_id":0,"time_range_node_id":29,"width_ms":60000}],"stored_outputs":[]},"node_id":30,"retention_ms":900000}]}, +{"name":"repeating rollup/no_raw","dag":{"edges":[{"consumer":1,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":2,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":1,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":3,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":4,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":3,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":5,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":4,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":6,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":7,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":6,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":8,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":7,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":9,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":10,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":9,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":11,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":10,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":12,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":13,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":12,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":14,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":13,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":15,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":16,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":15,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":17,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":16,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":18,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":19,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":18,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":20,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":19,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":21,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":22,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":21,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":23,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":22,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":24,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":25,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":24,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":26,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":25,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":27,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":28,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":27,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":29,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":28,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":2,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":5,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":8,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":11,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":14,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":17,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":20,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":23,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":26,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":29,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":30,"role":"Input","window":"NotApplicable"},{"consumer":32,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":31,"role":"Input","window":"NotApplicable"},{"consumer":33,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"producer":32,"role":"Input","window":"NotApplicable"},{"consumer":34,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"producer":33,"role":"Input","window":"NotApplicable"},{"consumer":35,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"producer":34,"role":"Input","window":"NotApplicable"}],"nodes":[{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":0,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"source":{"TimeSeries":{"metric":"latency_ms"}}}}}},{"guarantee":null,"id":1,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":2,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":1,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":3,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":60000}}}}},{"guarantee":null,"id":4,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":3,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":5,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":4,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":6,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":120000}}}}},{"guarantee":null,"id":7,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":6,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":8,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":7,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":9,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":180000}}}}},{"guarantee":null,"id":10,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":9,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":11,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":10,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":12,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":240000}}}}},{"guarantee":null,"id":13,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":12,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":14,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":13,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":15,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":300000}}}}},{"guarantee":null,"id":16,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":15,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":17,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":16,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":18,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":360000}}}}},{"guarantee":null,"id":19,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":18,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":20,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":19,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":21,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":420000}}}}},{"guarantee":null,"id":22,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":21,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":23,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":22,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":24,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":480000}}}}},{"guarantee":null,"id":25,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":24,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":26,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":25,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":27,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":540000}}}}},{"guarantee":null,"id":28,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":27,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":29,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":28,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":30,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryMerge":{"children":[2,5,8,11,14,17,20,23,26,29]}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":31,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":30}}}},{"guarantee":null,"id":32,"lifetime":{"OneEvaluation":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":31,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[2]}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":33,"lifetime":{"OneEvaluation":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":32}}}},{"guarantee":null,"id":34,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":33,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":{"Named":"sum"}},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[]}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":35,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":34}}}}],"roots":[33,35],"scalar_roots":[]},"stored_outputs":[{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":0,"scan_node_id":0,"time_range_node_id":1,"width_ms":60000}],"stored_outputs":[]},"node_id":2,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-120000},{"included":-60000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":60000,"scan_node_id":0,"time_range_node_id":4,"width_ms":60000}],"stored_outputs":[]},"node_id":5,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-180000},{"included":-120000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":120000,"scan_node_id":0,"time_range_node_id":7,"width_ms":60000}],"stored_outputs":[]},"node_id":8,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-240000},{"included":-180000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":180000,"scan_node_id":0,"time_range_node_id":10,"width_ms":60000}],"stored_outputs":[]},"node_id":11,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-300000},{"included":-240000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":240000,"scan_node_id":0,"time_range_node_id":13,"width_ms":60000}],"stored_outputs":[]},"node_id":14,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-360000},{"included":-300000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":300000,"scan_node_id":0,"time_range_node_id":16,"width_ms":60000}],"stored_outputs":[]},"node_id":17,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-420000},{"included":-360000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":360000,"scan_node_id":0,"time_range_node_id":19,"width_ms":60000}],"stored_outputs":[]},"node_id":20,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-480000},{"included":-420000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":420000,"scan_node_id":0,"time_range_node_id":22,"width_ms":60000}],"stored_outputs":[]},"node_id":23,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-540000},{"included":-480000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":480000,"scan_node_id":0,"time_range_node_id":25,"width_ms":60000}],"stored_outputs":[]},"node_id":26,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-600000},{"included":-540000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":540000,"scan_node_id":0,"time_range_node_id":28,"width_ms":60000}],"stored_outputs":[]},"node_id":29,"retention_ms":600000}]}, +{"name":"repeating rate-5m/no_raw","dag":{"edges":[{"consumer":1,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":2,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":1,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":3,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":4,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":3,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":5,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":4,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":6,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":7,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":6,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":8,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":7,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":9,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":10,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":9,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":11,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":10,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":12,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":13,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":12,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":14,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":13,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":2,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":5,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":8,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":11,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":14,"role":"Input","window":"NotApplicable"},{"consumer":16,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":15,"role":"Input","window":"NotApplicable"},{"consumer":17,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":16,"role":"Input","window":"NotApplicable"},{"consumer":18,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"producer":17,"role":"Input","window":"NotApplicable"},{"consumer":19,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":20,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":19,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":21,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":20,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":22,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":23,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":22,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":24,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":23,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":25,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":26,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":25,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":27,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":26,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":28,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":29,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":28,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":30,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":29,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":31,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":32,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":31,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":33,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":32,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":34,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":21,"role":"Input","window":"NotApplicable"},{"consumer":34,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":24,"role":"Input","window":"NotApplicable"},{"consumer":34,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":27,"role":"Input","window":"NotApplicable"},{"consumer":34,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":30,"role":"Input","window":"NotApplicable"},{"consumer":34,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":33,"role":"Input","window":"NotApplicable"},{"consumer":35,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":34,"role":"Input","window":"NotApplicable"},{"consumer":36,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":35,"role":"Input","window":"NotApplicable"},{"consumer":37,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"producer":36,"role":"Input","window":"NotApplicable"}],"nodes":[{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":0,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"source":{"TimeSeries":{"metric":"latency_ms"}}}}}},{"guarantee":null,"id":1,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":2,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":1,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":3,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":60000}}}}},{"guarantee":null,"id":4,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":3,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":5,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":4,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":6,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":120000}}}}},{"guarantee":null,"id":7,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":6,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":8,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":7,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":9,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":180000}}}}},{"guarantee":null,"id":10,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":9,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":11,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":10,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":12,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":240000}}}}},{"guarantee":null,"id":13,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":12,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":14,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":13,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":15,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryMerge":{"children":[2,5,8,11,14]}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":16,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":15}}}},{"guarantee":null,"id":17,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":16,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[]}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":18,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":17}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":19,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":3600000}}}}},{"guarantee":null,"id":20,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":19,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":21,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":20,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":22,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":3660000}}}}},{"guarantee":null,"id":23,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":22,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":24,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":23,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":25,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":3720000}}}}},{"guarantee":null,"id":26,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":25,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":27,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":26,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":28,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":3780000}}}}},{"guarantee":null,"id":29,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":28,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":30,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":29,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":31,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":3840000}}}}},{"guarantee":null,"id":32,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":31,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":33,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":32,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":34,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryMerge":{"children":[21,24,27,30,33]}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":35,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":34}}}},{"guarantee":null,"id":36,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":35,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[]}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":37,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":36}}}}],"roots":[18,37],"scalar_roots":[]},"stored_outputs":[{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":0,"scan_node_id":0,"time_range_node_id":1,"width_ms":60000}],"stored_outputs":[]},"node_id":2,"retention_ms":300000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-120000},{"included":-60000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":60000,"scan_node_id":0,"time_range_node_id":4,"width_ms":60000}],"stored_outputs":[]},"node_id":5,"retention_ms":300000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-180000},{"included":-120000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":120000,"scan_node_id":0,"time_range_node_id":7,"width_ms":60000}],"stored_outputs":[]},"node_id":8,"retention_ms":300000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-240000},{"included":-180000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":180000,"scan_node_id":0,"time_range_node_id":10,"width_ms":60000}],"stored_outputs":[]},"node_id":11,"retention_ms":300000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-300000},{"included":-240000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":240000,"scan_node_id":0,"time_range_node_id":13,"width_ms":60000}],"stored_outputs":[]},"node_id":14,"retention_ms":300000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-3660000},{"included":-3600000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":3600000,"scan_node_id":0,"time_range_node_id":20,"width_ms":60000}],"stored_outputs":[]},"node_id":21,"retention_ms":3900000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-3720000},{"included":-3660000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":3660000,"scan_node_id":0,"time_range_node_id":23,"width_ms":60000}],"stored_outputs":[]},"node_id":24,"retention_ms":3900000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-3780000},{"included":-3720000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":3720000,"scan_node_id":0,"time_range_node_id":26,"width_ms":60000}],"stored_outputs":[]},"node_id":27,"retention_ms":3900000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-3840000},{"included":-3780000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":3780000,"scan_node_id":0,"time_range_node_id":29,"width_ms":60000}],"stored_outputs":[]},"node_id":30,"retention_ms":3900000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-3900000},{"included":-3840000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":3840000,"scan_node_id":0,"time_range_node_id":32,"width_ms":60000}],"stored_outputs":[]},"node_id":33,"retention_ms":3900000}]}, +{"name":"runtime_install a_coarse_sum_rolls_up_from_the_stored_fine_sum","dag":{"edges":[{"consumer":1,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":2,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":1,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":3,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"producer":2,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":4,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"producer":3,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":5,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"producer":4,"role":"Input","window":"NotApplicable"},{"consumer":6,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"producer":2,"role":"Input","window":"NotApplicable"}],"nodes":[{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":0,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"source":{"TimeSeries":{"metric":"m"}}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":1,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":2,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":1,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[2,3]}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":3,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":2}}}},{"guarantee":null,"id":4,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":3,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":{"Named":"sum"}},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[1]}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":5,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":4}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":6,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":2}}}}],"roots":[5,6],"scalar_roots":[]},"stored_outputs":[{"built_at":"ingestion_run","coverage":{"definition_id":"db0c9241a79202b6925e84aef010e89a","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":0,"scan_node_id":0,"time_range_node_id":1,"width_ms":60000}],"stored_outputs":[]},"node_id":2,"retention_ms":60000},{"built_at":"ingestion_run","coverage":{"definition_id":"3c90a6b5ff3ecaa445ea9bf285e1dc0b","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[],"stored_outputs":[2]},"node_id":4,"retention_ms":60000}]}, +{"name":"runtime_install a_materialized_sliding_window_merge_is_listed_and_built","dag":{"edges":[{"consumer":1,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"NotApplicable"},{"consumer":2,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":1,"role":"Input","window":"NotApplicable"},{"consumer":3,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"NotApplicable"},{"consumer":4,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":3,"role":"Input","window":"NotApplicable"},{"consumer":5,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":4,"role":"Input","window":"NotApplicable"},{"consumer":6,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"NotApplicable"},{"consumer":7,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":6,"role":"Input","window":"NotApplicable"},{"consumer":8,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":7,"role":"Input","window":"NotApplicable"},{"consumer":9,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"NotApplicable"},{"consumer":10,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":9,"role":"Input","window":"NotApplicable"},{"consumer":11,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":10,"role":"Input","window":"NotApplicable"},{"consumer":12,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"NotApplicable"},{"consumer":13,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":12,"role":"Input","window":"NotApplicable"},{"consumer":14,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":13,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":2,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":5,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":8,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":11,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":14,"role":"Input","window":"NotApplicable"},{"consumer":16,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":15,"role":"Input","window":"NotApplicable"}],"nodes":[{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":0,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"source":{"TimeSeries":{"metric":"m"}}}}}},{"guarantee":null,"id":1,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":2,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":1,"family":{"ExactAggregate":["Max","Max"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":3,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":60000}}}}},{"guarantee":null,"id":4,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeRange":{"child":3,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":5,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":4,"family":{"ExactAggregate":["Max","Max"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":6,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":120000}}}}},{"guarantee":null,"id":7,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeRange":{"child":6,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":8,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":7,"family":{"ExactAggregate":["Max","Max"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":9,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":180000}}}}},{"guarantee":null,"id":10,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeRange":{"child":9,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":11,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":10,"family":{"ExactAggregate":["Max","Max"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":12,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":240000}}}}},{"guarantee":null,"id":13,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeRange":{"child":12,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":14,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":13,"family":{"ExactAggregate":["Max","Max"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":15,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryMerge":{"children":[2,5,8,11,14]}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":16,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":15}}}}],"roots":[16],"scalar_roots":[]},"stored_outputs":[{"built_at":"query_run","coverage":{"definition_id":"71610a93a8af5ae7ee6ecc0839e28d3b","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":0,"scan_node_id":0,"time_range_node_id":1,"width_ms":60000}],"stored_outputs":[]},"node_id":2,"retention_ms":3900000},{"built_at":"query_run","coverage":{"definition_id":"71610a93a8af5ae7ee6ecc0839e28d3b","selection":[{"columns":[],"relative_time":[{"excluded":-120000},{"included":-60000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":60000,"scan_node_id":0,"time_range_node_id":4,"width_ms":60000}],"stored_outputs":[]},"node_id":5,"retention_ms":3900000},{"built_at":"query_run","coverage":{"definition_id":"71610a93a8af5ae7ee6ecc0839e28d3b","selection":[{"columns":[],"relative_time":[{"excluded":-180000},{"included":-120000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":120000,"scan_node_id":0,"time_range_node_id":7,"width_ms":60000}],"stored_outputs":[]},"node_id":8,"retention_ms":3900000},{"built_at":"query_run","coverage":{"definition_id":"71610a93a8af5ae7ee6ecc0839e28d3b","selection":[{"columns":[],"relative_time":[{"excluded":-240000},{"included":-180000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":180000,"scan_node_id":0,"time_range_node_id":10,"width_ms":60000}],"stored_outputs":[]},"node_id":11,"retention_ms":3900000},{"built_at":"query_run","coverage":{"definition_id":"71610a93a8af5ae7ee6ecc0839e28d3b","selection":[{"columns":[],"relative_time":[{"excluded":-300000},{"included":-240000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":240000,"scan_node_id":0,"time_range_node_id":13,"width_ms":60000}],"stored_outputs":[]},"node_id":14,"retention_ms":3900000},{"built_at":"query_run","coverage":{"definition_id":"71610a93a8af5ae7ee6ecc0839e28d3b","selection":[{"columns":[],"relative_time":[{"excluded":-300000},{"included":0}]}]},"inputs":{"raw_sample_windows":[],"stored_outputs":[2,5,8,11,14]},"node_id":15,"retention_ms":3900000}]}, +{"name":"runtime_install a_summary_across_series_from_per_series_summaries_is_listed_and_built","dag":{"edges":[{"consumer":1,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":2,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":1,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":3,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":2,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":4,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":3,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":5,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"producer":4,"role":"Input","window":"NotApplicable"},{"consumer":6,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":2,"role":"Input","window":"NotApplicable"},{"consumer":7,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":6,"role":"Input","window":"NotApplicable"}],"nodes":[{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":0,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"source":{"TimeSeries":{"metric":"m"}}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":1,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":2,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":1,"family":{"ExactAggregate":["Max","Max"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":3,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":2}}}},{"guarantee":null,"id":4,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":3,"family":{"ExactAggregate":["Max","Max"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[2]}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":5,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"max","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":4}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":6,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":2}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":7,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"Project":{"child":6,"cols":[{"alias":"ts","expr":{"Column":0}},{"alias":"value","expr":{"Column":1}},{"alias":"$promql_series_identity","expr":{"Column":3}}],"qualifier":null}}}}],"roots":[5,7],"scalar_roots":[]},"stored_outputs":[{"built_at":"ingestion_run","coverage":{"definition_id":"5555de6812a18c62345c199d5ba8f06e","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":0,"scan_node_id":0,"time_range_node_id":1,"width_ms":60000}],"stored_outputs":[]},"node_id":2,"retention_ms":60000},{"built_at":"ingestion_run","coverage":{"definition_id":"f35ee4e847802c130cbd5221a81f7ea1","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[],"stored_outputs":[2]},"node_id":4,"retention_ms":60000}]}, +{"name":"runtime_install group_outputs_from_one_stored_per_series_sum_are_listed_and_built","dag":{"edges":[{"consumer":1,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":2,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":1,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":3,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":2,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":4,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":3,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":5,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":4,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":6,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"producer":5,"role":"Input","window":"NotApplicable"},{"consumer":7,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":2,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":8,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":7,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":9,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":8,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":10,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"producer":9,"role":"Input","window":"NotApplicable"}],"nodes":[{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":0,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"source":{"TimeSeries":{"metric":"m"}}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":1,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":2,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":1,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":3,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":2}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":4,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Project":{"child":3,"cols":[{"alias":"ts","expr":{"Column":0}},{"alias":"value","expr":{"Column":1}},{"alias":"job","expr":{"Column":2}},{"alias":"$promql_series_identity","expr":{"Column":4}}],"qualifier":null}}}},{"guarantee":null,"id":5,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":4,"family":{"ExactAggregate":["Max","Max"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[2]}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":6,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"max","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":5}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":7,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":2}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":8,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Project":{"child":7,"cols":[{"alias":"ts","expr":{"Column":0}},{"alias":"value","expr":{"Column":1}},{"alias":"instance","expr":{"Column":3}},{"alias":"job","expr":{"Column":2}},{"alias":"$promql_series_identity","expr":{"Column":4}}],"qualifier":null}}}},{"guarantee":null,"id":9,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":8,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[2,3]}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":10,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":9}}}}],"roots":[6,10],"scalar_roots":[]},"stored_outputs":[{"built_at":"ingestion_run","coverage":{"definition_id":"672f965318f2bf2ba350748044719b9b","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":0,"scan_node_id":0,"time_range_node_id":1,"width_ms":60000}],"stored_outputs":[]},"node_id":2,"retention_ms":60000},{"built_at":"ingestion_run","coverage":{"definition_id":"7976839b25e74a9cfac2ab1d6d01a63a","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[],"stored_outputs":[2]},"node_id":5,"retention_ms":60000},{"built_at":"ingestion_run","coverage":{"definition_id":"84209f44520311bced823b9e98eaf85c","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[],"stored_outputs":[2]},"node_id":9,"retention_ms":60000}]} ] diff --git a/crates/integration-tests/tests/materialized_views.rs b/crates/integration-tests/tests/materialized_views.rs index c6fad2f29..090b07416 100644 --- a/crates/integration-tests/tests/materialized_views.rs +++ b/crates/integration-tests/tests/materialized_views.rs @@ -3,153 +3,21 @@ use std::collections::HashSet; -use asap_logical_asap_optimization::DeploymentCapabilities; use asap_planner::{ - e2e_plan, installable_plan_of, plan_installable, FrontendInput, InstallablePlan, - InstallablePlanError, MaterializedViewRefresh, PlanningModels, RawDataRetention, UserInput, - INSTALLABLE_PLAN_WIRE_VERSION, + InstallablePlan, InstallablePlanError, MaterializedViewRefresh, INSTALLABLE_PLAN_WIRE_VERSION, }; use asap_types::ir::operator::Source; -use asap_types::ir::physical_export::{compile_physical_asap_workload, PhysicalASAPDAG}; -use asap_types::ir::properties::ExecutionTiming; -use asap_types::ir::{ASAPOp, NonASAPOp, Operator}; -use asap_types::physical::Materialization; -use asap_types::workload::PlanningWorkload; -use serde_json::Value; +use asap_types::ir::physical_export::PhysicalASAPDAG; -/// A repeating PromQL workload over ingesting `m`, each query every minute. -fn repeating(queries: &[&str]) -> PlanningWorkload { - repeating_with(queries, "") -} +use asap_types::ir::{NonASAPOp, Operator}; -/// [`repeating`], each entry with `extra` fields (e.g. a range evaluation). -fn repeating_with(queries: &[&str], extra: &str) -> PlanningWorkload { - let entries: Vec = queries - .iter() - .map(|q| { - format!( - r#"{{"query": {q:?}, "demand": {{"fixed_interval": 60000}}, - "requirements": {{"accuracy": {{"max_error": 0.01, "failure_probability": 0.01}}}}, - "predictability": {{"predictable": {{"known_at": null}}}}{extra}}}"# - ) - }) - .collect(); - serde_json::from_str(&format!( - r#"{{"query_workload": {{"language": "promql", "repeating_queries": [{}]}}, - "data_workload": {{"arrival": {{"ingesting": {{"rows_per_second": 1000.0}}}}, - "sources": {{"m": {{"series_count": 1000, "sample_interval": 1000}}}}}}}}"#, - entries.join(",") - )) - .unwrap() -} - -/// Stage 3's priced raw lookback of `dag` (the selected candidate's -/// physical DAG, tumbling-window chains unexpanded), per source: each query-time -/// scan's `PhysicalASAPDAG::scan_window` with every chain kept across -/// evaluations reading its newest tumbling window, as `asap-plan-selection` prices -/// it, the longest per source. -fn stage3_raw_lookback_ms(dag: &PhysicalASAPDAG) -> Vec<(Source, Option)> { - let kept_chains: HashSet<_> = dag - .nodes - .iter() - .filter(|node| { - matches!( - (&node.payload, node.materialization), - ( - Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }), - Materialization::MatAtQueryKeptAcrossEvaluations(_) - ) - ) - }) - .map(|node| node.id) - .collect(); - let mut lookback: Vec<(Source, Option)> = Vec::new(); - for node in &dag.nodes { - let Operator::NonASAP(NonASAPOp::Scan { source, .. }) = &node.payload else { - continue; - }; - if node.output_state.timing != ExecutionTiming::QueryTime { - continue; - } - let scan = dag.scan_window(node.id, &kept_chains).lookback_ms(); - match lookback.iter_mut().find(|(kept, _)| kept == source) { - Some((_, kept)) => *kept = kept.zip(scan).map(|(a, b)| a.max(b)), - None => lookback.push((source.clone(), scan)), - } - } - lookback -} - -/// `rate(m[5m])` and `rate(m[5m] offset 1h)`, both at query time: the -/// runtime keeps 65 minutes of `m`, the raw lookback Stage 3 prices for the -/// same plan. With no raw data kept, both run from views refreshed at -/// ingestion and no source is read at query time. -#[tokio::test(flavor = "multi_thread")] -async fn raw_data_retention_is_the_lookback_stage3_prices() { - let workload = repeating(&["rate(m[5m])", "rate(m[5m] offset 1h)"]); - let input = || UserInput::new(&workload, FrontendInput::Promql, PlanningModels::builtin()); - let (plan, _) = plan_installable(input()).await.unwrap(); - let m = Source::TimeSeries { metric: "m".into() }; - assert_eq!( - plan.raw_data_retention().unwrap(), - [RawDataRetention::new(m.clone(), Some(65 * 60_000))] - ); - let output = e2e_plan(input()).await.unwrap(); - let selected = compile_physical_asap_workload(&output.roots()).unwrap(); - assert_eq!(stage3_raw_lookback_ms(&selected), [(m, Some(65 * 60_000))]); - - let capabilities = DeploymentCapabilities { - raw_data_retained: false, - ..DeploymentCapabilities::default() - }; - let (plan, _) = plan_installable(UserInput::new( - &workload, - FrontendInput::Promql, - PlanningModels::builtin().with_deployment_capabilities(&capabilities), - )) - .await - .unwrap(); - assert!(!plan.materialized_views().unwrap().is_empty()); - assert_eq!(plan.raw_data_retention().unwrap(), []); -} - -/// A range-evaluated query (`max_over_time(m[5m])` over the last hour at -/// 1-minute steps) installed with every node at query time (its frontend -/// DAG, nothing materialized): its oldest step reads `m` 5 minutes before an -/// hour ago, so the runtime keeps 65 minutes of `m`, the scan's window plus -/// the range span, on the scan's node. (Stage 3 prices one evaluation's -/// window, 5 minutes.) -#[test] -fn a_range_evaluated_query_time_scan_keeps_its_window_plus_the_span() { - let workload = repeating_with( - &["max_over_time(m[5m])"], - r#", "range_evaluation": {"step_ms": 60000, "span_ms": 3600000}"#, - ); - let roots = asap_devtools::planning_stages::lower_promql_roots(&workload).unwrap(); - let entries: Vec<_> = workload.query_workload.entries().collect(); - let (plan, _) = installable_plan_of(&roots, None, &Default::default(), &entries).unwrap(); - plan.validate().unwrap(); - assert!(plan.materialized_views().unwrap().is_empty()); - assert_eq!( - plan.raw_data_retention().unwrap(), - [RawDataRetention::new( - Source::TimeSeries { metric: "m".into() }, - Some(5 * 60_000 + 3_600_000) - )] - ); - let scan = plan - .dag - .nodes - .iter() - .find(|node| matches!(node.payload, Operator::NonASAP(NonASAPOp::Scan { .. }))) - .unwrap(); - assert_eq!(scan.retention_ms, Some(3_900_000)); -} +use serde_json::Value; /// A plan file of wire version 10 (assembled from the parts of a plan -/// written on `main` before #970: the versioned DAG envelope, `queries`, -/// `stored_outputs`, `query_time_raw_windows`, `selection`) is refused for -/// its version, not for a missing field; a file of this version reads back. +/// written before #970: the versioned DAG envelope, `queries`, +/// `stored_outputs`, `query_time_raw_windows`, `selection`; its DAG nodes +/// carry today's node fields, e.g. `lifetime`) is refused for its version, +/// not for a missing field; its DAG with today's version reads back. #[test] fn a_plan_file_of_an_older_wire_version_is_refused_by_version() { let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) diff --git a/crates/integration-tests/tests/memo_enumeration.rs b/crates/integration-tests/tests/memo_enumeration.rs deleted file mode 100644 index 6566b08e1..000000000 --- a/crates/integration-tests/tests/memo_enumeration.rs +++ /dev/null @@ -1,481 +0,0 @@ -//! The memo the production planner builds, measured on the planner-layering -//! doc's example (`avg by (job) (quantile_over_time(0.5, data[5m]) / -//! quantile_over_time(0.5, data[5m]) + quantile_over_time(0.9, data[5m]))`, -//! the stage-pipeline workload -//! `tools/dag-viewer/memo-examples/promql/ratio-of-quantiles-5m.json`). -//! -//! Lowering is production's: the PromQL frontend, then each root with the -//! series-identity column, as `Stage1ToStage3Pass` does. The memo is read from -//! the production run's `PlanningRun::per_computation_candidates` (after -//! Pass 1) and `PlanningRun::logical_asap_candidates` (after Pass 2). -//! -//! **The memo and the whole-workload plans.** After Pass 1 the memo's -//! choices whose candidates are all sized for the whole accuracy budget -//! (`accuracy_budget_parts` 1) and the plans `compose_logical_candidate` builds from -//! the Pass 1 inventory's assignments are not equal sets: -//! - A whole-workload plan is a memo DAG exactly when targets that compute -//! the same thing take the same choice and it takes no KLL. With series -//! identity the two `quantile 0.5` are one memo group, which takes one -//! candidate; the inventory keeps a target for each and may give them -//! different summaries. The inventory sizes a KLL per value; Pass 1 sizes -//! every memo KLL for the whole result (a failure probability below the -//! query's, divided over the source's series), so no KLL plan is in both. -//! - A whole-budget memo DAG is a whole-workload plan exactly when it takes -//! no KLL sized for the whole result, that is, no KLL. -//! -//! The memo's other choices size an operand's summary for a share of the -//! budget (a ratio's operands); the whole-workload path has no such plans. -use asap_logical_asap_optimization::generate_dag_of_logical_memo_groups; -use std::rc::Rc; - -use asap_devtools::planning_stages::lower_promql_roots; -use asap_integration_tests::planning::expanded_memo; -use asap_logical_asap_optimization::test_support::{ - compose_logical_candidate, generate_per_computation_candidates, PerComputationCandidates, -}; -use asap_logical_asap_optimization::{ - build_logical_asap_dags_from_chosen_candidates, enumerate, ChosenCandidatePerMemoGroup, -}; -use asap_planner::{run_planning, PlanSearchSettings, PlanningModels, PlanningRun}; -use asap_types::ir::schema::{FieldDataType, SketchParams}; -use asap_types::ir::{ASAPOp, OperatorNode, QueryRoot}; -use asap_types::stages::{queries_rooted_at, MemoGroupId}; -use asap_types::stages::{CandidateForMemoGroup, Memo, OriginOfCandidateForMemoGroup}; -use asap_types::workload::{PlanningWorkload, Query, RepeatedDemand, RepetitionInterval}; - -const RATIO_OF_QUANTILES: &str = concat!( - env!("CARGO_MANIFEST_DIR"), - "/../../tools/dag-viewer/memo-examples/promql/ratio-of-quantiles-5m.json" -); - -/// The example's failure probability, as in its workload file. -const FAILURE_PROBABILITY: f64 = 0.001; - -/// One root per query, in query order. -type Plan = Vec>; - -fn ratio_of_quantiles() -> PlanningWorkload { - serde_json::from_str(&std::fs::read_to_string(RATIO_OF_QUANTILES).unwrap()).unwrap() -} - -/// The example's workload with `queries` in place of its query, each with -/// its requirements, repeating every `interval_ms`. -fn with_queries(queries: &[&str], interval_ms: u32) -> PlanningWorkload { - let mut workload = ratio_of_quantiles(); - let repeating = workload.query_workload.repeating_queries.as_mut().unwrap(); - let mut entry = repeating[0].clone(); - entry.demand = RepeatedDemand::FixedInterval(RepetitionInterval(interval_ms)); - *repeating = queries - .iter() - .map(|query| { - let mut entry = entry.clone(); - entry.query = Query((*query).into()); - entry - }) - .collect(); - workload -} - -/// The frontend's roots, each with the series-identity column, as -/// `Stage1ToStage3Pass` and `asap_planner` lower a PromQL workload. -fn production_roots(workload: &PlanningWorkload) -> Vec { - lower_promql_roots(workload).unwrap() -} - -/// The production run of Stages 1 to 3 over `workload`. -fn production_run(workload: &PlanningWorkload) -> PlanningRun { - production_run_with(workload, PlanningModels::builtin()) -} - -/// [`production_run`] with `models`, keeping every stage's memo. -fn production_run_with(workload: &PlanningWorkload, models: PlanningModels<'_>) -> PlanningRun { - let search = PlanSearchSettings { inspect: true }; - let entries: Vec<_> = workload.query_workload.entries().collect(); - let data = workload.data_workload.clone().unwrap_or_default(); - let roots = production_roots(workload); - run_planning(roots, &entries, &data, models, search).unwrap() -} - -/// A deployment model that may accept an output whatever its guarantee, so -/// Pass 1 prunes no summary for its target, and accepts none: Stage 3 -/// validates no plan, and the run still records its memo. -struct AcceptsNoOutput; - -impl asap_accuracy::AccuracyModel for AcceptsNoOutput { - fn satisfies( - &self, - _guarantee: &asap_types::ir::properties::AccuracyGuarantee, - _target: &asap_types::ir::properties::AccuracyTarget, - ) -> bool { - false - } -} - -/// The memo after Pass 2. -fn pass2_memo(run: &PlanningRun) -> &Memo { - &run.logical_asap_candidates - .as_ref() - .expect("the run recorded its Pass 2 memo") - .memo -} - -/// The memo after Pass 1. -fn pass1_memo(run: &PlanningRun) -> &Memo { - &run.per_computation_candidates - .as_ref() - .expect("Pass 1's output") - .memo -} - -/// Pass 1's target list for `workload`, pruned as the facade's Pass 1 -/// prunes it under `models`. -fn pass1_inventory( - workload: &PlanningWorkload, - models: &PlanningModels<'_>, -) -> PerComputationCandidates { - let roots = production_roots(workload); - let inventory = - generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&roots)) - .unwrap() - .without_unsupported_summaries( - models - .deployment_capabilities - .supported_summaries - .as_deref(), - ); - let accuracy = models.accuracy_check_inputs(); - if accuracy.model.may_accept_regardless_of_guarantee() - || accuracy.evidence.may_provide_evidence() - { - return inventory; - } - let targets: Vec<_> = workload - .query_workload - .entries() - .map(|entry| entry.requirements.accuracy.clone()) - .collect(); - let data = workload.data_workload.clone().unwrap_or_default(); - inventory.without_summaries_no_query_can_use(&targets, &data, accuracy.model) -} - -fn candidate_count(memo: &Memo) -> usize { - memo.candidates.iter().map(Vec::len).sum() -} - -/// Every plan the whole-workload path composes from the inventory: one per -/// assignment of each target's choices, with the assignment. -fn whole_workload_plans(inventory: &PerComputationCandidates) -> Vec<(Vec, Plan)> { - let counts: Vec<_> = inventory.targets.iter().map(|t| t.choice_count()).collect(); - let total: usize = counts.iter().product(); - (0..total) - .map(|mut n| { - counts - .iter() - .map(|&count| { - let digit = n % count; - n /= count; - digit - }) - .collect::>() - }) - .filter(|choice| !inventory.absorbed_choice_varies(choice)) - .filter_map(|choice| { - let roots = compose_logical_candidate(inventory, &choice).ok()?; - let plan = roots - .into_iter() - .map(|root| match root { - QueryRoot::Operator(node) => node, - QueryRoot::Scalar(_) => panic!("operator roots only"), - }) - .collect(); - Some((choice, plan)) - }) - .collect() -} - -/// `choice`'s DAG, one root per query. -fn composed(memo: &Memo, choice: &ChosenCandidatePerMemoGroup) -> Plan { - let mut roots = build_logical_asap_dags_from_chosen_candidates(memo, choice) - .expect("an enumerated choice composes"); - roots.sort_by_key(|(group, _)| { - queries_rooted_at(&memo.query_roots, *group) - .first() - .copied() - }); - roots.into_iter().map(|(_, root)| root).collect() -} - -fn chosen<'a>( - memo: &'a Memo, - choice: &'a ChosenCandidatePerMemoGroup, -) -> impl Iterator { - choice - .iter() - .map(|(group, &index)| &memo.candidates[group.0 as usize][index]) -} - -/// The KLL parameters of the summaries `candidate` builds. -fn kll(candidate: &CandidateForMemoGroup) -> Vec<(u32, Option)> { - OperatorNode::reachable(&candidate.root_node) - .iter() - .filter_map(|node| match node.asap() { - Some(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), - .. - }) => match kind.params() { - SketchParams::Kll { - k, - failure_probability, - } => Some((*k, *failure_probability)), - _ => None, - }, - _ => None, - }) - .collect() -} - -/// A KLL Pass 1 sized for the whole result, at the query's failure -/// probability divided over the source's series (1e-7 here), far below what -/// a per-value KLL is sized for (the query's, or half of it). -fn sized_for_whole_result(candidate: &CandidateForMemoGroup) -> bool { - kll(candidate) - .iter() - .any(|(_, p)| p.is_some_and(|p| p < FAILURE_PROBABILITY / 10.0)) -} - -/// Checks the relation of the module doc between the Pass 1 memo's -/// whole-budget DAGs and the whole-workload plans; returns (memo choices, -/// whole-budget ones, whole-workload plans, plans in both). -fn assert_pass1_memo_and_whole_workload_plans( - run: &PlanningRun, - inventory: &PerComputationCandidates, -) -> (usize, usize, usize, usize) { - let memo = pass1_memo(run); - let enumeration = enumerate(memo, 100_000); - assert!(enumeration.complete); - let plans: Vec<_> = enumeration - .all_chosen_candidates - .iter() - .map(|choice| composed(memo, choice)) - .collect(); - for (i, plan) in plans.iter().enumerate() { - assert!(!plans[..i].contains(plan), "each choice is a different DAG"); - } - let whole_budget: Vec<_> = enumeration - .all_chosen_candidates - .iter() - .zip(&plans) - .filter(|(choice, _)| chosen(memo, choice).all(|c| c.accuracy_budget_parts == 1)) - .collect(); - let whole = whole_workload_plans(inventory); - // Targets that compute the same thing. - let same: Vec<(usize, usize)> = (0..inventory.targets.len()) - .flat_map(|a| (a + 1..inventory.targets.len()).map(move |b| (a, b))) - .filter(|&(a, b)| { - let (a, b) = (&inventory.targets[a], &inventory.targets[b]); - inventory.aggregate(a) == inventory.aggregate(b) && a.measure_index == b.measure_index - }) - .collect(); - let takes_kll = |plan: &Plan| { - plan.iter().flat_map(OperatorNode::reachable).any(|node| { - matches!(node.asap(), Some(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), .. - }) if matches!(kind.params(), SketchParams::Kll { .. })) - }) - }; - for (assignment, plan) in &whole { - assert_eq!( - whole_budget.iter().any(|(_, p)| *p == plan), - same.iter().all(|&(a, b)| assignment[a] == assignment[b]) && !takes_kll(plan), - "a whole-workload plan is a memo DAG when equal targets choose alike \ - and it takes no (per-value) KLL" - ); - assert!( - !plans.iter().any(|p| p == plan) || whole_budget.iter().any(|(_, p)| *p == plan), - "a whole-workload plan is never a budget-share DAG" - ); - } - for (choice, plan) in &whole_budget { - assert_eq!( - whole.iter().any(|(_, p)| p == *plan), - !chosen(memo, choice).any(sized_for_whole_result), - "a whole-budget memo DAG is a whole-workload plan unless it takes a \ - KLL sized for the whole result" - ); - } - let both = whole - .iter() - .filter(|(_, plan)| whole_budget.iter().any(|(_, p)| p == &plan)) - .count(); - (plans.len(), whole_budget.len(), whole.len(), both) -} - -/// The example's memo after each production step, and the `accuracy_budget_parts` -/// of the state-configuration rule's rewrites (one test: one planning run). -#[test] -fn ratio_of_quantiles_production_memo() { - // With the built-in model, Pass 1 prunes every summary of the quantile - // groups: no query has a guarantee through `/` and `+` over a KLL rank - // error or a DDSketch relative one. The memo below is planned with a - // model that prunes none and accepts no output, so its summaries stay - // and its Pass 2 memo is recorded. - let enforced = production_run(&ratio_of_quantiles()); - assert_eq!(candidate_count(pass1_memo(&enforced)), 7); - let models = PlanningModels::builtin().with_accuracy(&AcceptsNoOutput); - let run = production_run_with(&ratio_of_quantiles(), models); - // Stage 0 (Pass 1's groups): series identity makes the two - // `quantile 0.5` one group. - // Pass 1: the two quantile groups have 5 candidates each, 3 for the - // whole budget (exact, DDSketch, KLL sized for the whole result) and 2 - // for half of it; the other 5 groups one each. - assert_eq!(pass1_memo(&run).groups.len(), 7); - assert_eq!(candidate_count(pass1_memo(&run)), 15); - // 5 × 5 choices, 3 × 3 of them for the whole budget. The inventory has - // 3 × 3 assignments (one target per quantile group and measure, so the - // two `quantile 0.5` share one, 3 choices each): the 2 × 2 that take no - // KLL are memo DAGs. - assert_eq!( - assert_pass1_memo_and_whole_workload_plans( - &run, - &pass1_inventory(&ratio_of_quantiles(), &models) - ), - (25, 9, 9, 4) - ); - // The last step, over the complete tumbling-window-width domain. - let last = pass2_memo(&run); - assert_eq!(last.groups.len(), 255); - assert_eq!(candidate_count(last), 1243); - let enumeration = enumerate(last, 2_000); - assert!(!enumeration.complete); - assert_eq!(enumeration.all_chosen_candidates.len(), 2_000); - for (group, candidates) in last.candidates.iter().enumerate() { - for index in 0..candidates.len() { - let choice = [(MemoGroupId(group as u32), index)].into(); - build_logical_asap_dags_from_chosen_candidates(last, &choice) - .unwrap_or_else(|e| panic!("G{group} candidate {index}: {e}")); - } - } - // The state-configuration rule rewrites a candidate's KLL to the size - // the readers of a state share; the rewrite is a copy of the candidate - // (`derive` in `pass2/memo/rules/shared_configuration_readers.rs`) with - // the `accuracy_budget_parts` its new size was made for. In each quantile - // group the `accuracy_budget_parts` 1 KLL `k` 567 (sized for the whole - // result) gets copies at `k` 567 (parts 1) and 1184 (parts 2, the size of - // the half-budget KLL); the `accuracy_budget_parts` 2 KLL `k` 1184 keeps - // 2. The `k` 1184 copy composes the same DAG as the original - // `accuracy_budget_parts` 2 candidate: one sketch, sized for half the - // budget. - let pass1 = pass1_memo(&run); - let quantile_groups: Vec<_> = (0..pass1.groups.len()) - .filter(|&group| pass1.candidates[group].len() == 5) - .collect(); - assert_eq!(quantile_groups.len(), 2); - for group in quantile_groups { - let sizes = |candidates: &[CandidateForMemoGroup]| { - candidates - .iter() - .filter(|c| c.origin == OriginOfCandidateForMemoGroup::Pass1) - .filter_map(|c| kll(c).first().map(|&(k, _)| (k, c.accuracy_budget_parts))) - .collect::>() - }; - assert_eq!(sizes(&pass1.candidates[group]), [(567, 1), (1184, 2)]); - let candidates = &last.candidates[group]; - assert_eq!( - sizes(candidates), - [(567, 1), (1184, 2), (567, 1), (1184, 2)] - ); - let at_k: Vec = candidates - .iter() - .enumerate() - .filter(|(_, c)| { - c.origin == OriginOfCandidateForMemoGroup::Pass1 - && kll(c).first().is_some_and(|&(k, _)| k == 1184) - }) - .map(|(index, _)| index) - .collect(); - assert_eq!(at_k.len(), 2); - let alone = |index: usize| composed(last, &[(MemoGroupId(group as u32), index)].into()); - assert_eq!(alone(at_k[0]), alone(at_k[1])); - } -} - -/// On the three-query workload, the same relation: the memo's 3 × 3 -/// whole-budget DAGs and the inventory's 3 × 3 assignments share the 2 × 2 -/// that take no KLL (the memo's is sized for the whole result, the -/// inventory's per value), with the default search. -#[test] -fn shared_quantiles_pass1_memo_and_whole_workload_plans() { - let workload = with_queries( - &[ - "quantile_over_time(0.5, data[5m])", - "quantile_over_time(0.9, data[5m])", - "sum by (job) (rate(data[1m]))", - ], - 60_000, - ); - let run = production_run(&workload); - assert_eq!( - assert_pass1_memo_and_whole_workload_plans( - &run, - &pass1_inventory(&workload, &PlanningModels::builtin()) - ), - (9, 9, 9, 4) - ); -} - -/// Pass 2 rejects a memo whose groups read each other, a group reading -/// itself included, so production's memo (coarse and fine quantiles, whose -/// rollups read the finer grouping) has a dependency order and every choice -/// `enumerate` returns composes. A cycle made by hand, a child candidate -/// reading back the group that reads it, composes no DAG: -/// `build_logical_asap_dags_from_chosen_candidates` says so. The memo is -/// built from production's roots on the seed grids, without the search. -#[test] -fn choices_whose_candidates_read_each_other_are_left_out() { - let workload = with_queries( - &[ - "quantile_over_time(0.5, data[1m])", - "quantile_over_time(0.9, data[5m])", - "quantile by (host, job) (0.5, data)", - "quantile by (job) (0.5, data)", - ], - 10_000, - ); - let entries: Vec<_> = workload.query_workload.entries().collect(); - let memo = expanded_memo(production_roots(&workload), &entries); - assert!(memo.topological_order().is_some()); - for (group, candidates) in memo.candidates.iter().enumerate() { - for candidate in candidates { - assert!(!candidate.input_groups.contains(&MemoGroupId(group as u32))); - } - } - let enumeration = enumerate(&memo, 2_000); - assert!(!enumeration.all_chosen_candidates.is_empty()); - for choice in &enumeration.all_chosen_candidates { - build_logical_asap_dags_from_chosen_candidates(&memo, choice).unwrap(); - } - - let (group, index, child) = (0..memo.groups.len()) - .flat_map(|group| (0..memo.candidates[group].len()).map(move |index| (group, index))) - .find_map(|(group, index)| { - memo.candidates[group][index] - .input_groups - .first() - .map(|&child| (MemoGroupId(group as u32), index, child)) - }) - .expect("a candidate reading another group"); - let mut cyclic = memo.clone(); - let mut back = cyclic.candidates[child.0 as usize][0].clone(); - back.root_node = Rc::clone(&cyclic.groups[group.0 as usize].logical_node); - back.input_groups = vec![group]; - cyclic.candidates[child.0 as usize].push(back); - let back_index = cyclic.candidates[child.0 as usize].len() - 1; - assert_eq!(cyclic.topological_order(), None); - assert!(matches!( - build_logical_asap_dags_from_chosen_candidates( - &cyclic, - &[(group, index), (child, back_index)].into() - ), - Err(asap_logical_asap_optimization::ErrorBuildingLogicalASAPDagsFromChosenCandidates::Cycle { .. }) - )); -} diff --git a/crates/integration-tests/tests/operator_design_examples.rs b/crates/integration-tests/tests/operator_design_examples.rs index 88b19291d..d11672f3d 100644 --- a/crates/integration-tests/tests/operator_design_examples.rs +++ b/crates/integration-tests/tests/operator_design_examples.rs @@ -1,5 +1,5 @@ //! #511 examples: source text → unified dag → summary rewrite → flat export. -use crate::physical_common; + use asap_frontend_sql::{lower_sql, SqlCatalog}; use asap_types::ir::operator::AggIntent; use asap_types::ir::properties::AccuracyTarget; @@ -44,7 +44,7 @@ async fn sql_filter_projection_example() { assert!( matches!(root.expect_non_asap(),NonASAPOp::Project { cols,.. } if matches!(cols[0].expr,ScalarExpr::Arithmetic { .. })) ); - let wire = physical_common::compile_physical_asap_dag(&root).unwrap(); + let wire = asap_types::test_support::physical_asap_dag_at_query_time(&root).unwrap(); wire.validate().unwrap(); assert_eq!(wire.nodes.len(), OperatorNode::reachable(&root).len()); } @@ -111,7 +111,7 @@ async fn sql_sum_projection_before_and_after_summary_rewrite() { assert!(dag .iter() .any(|n| matches!(n.asap(), Some(ASAPOp::FinalizeExactAccumulator { .. })))); - let wire = physical_common::compile_physical_asap_dag(&rewritten).unwrap(); + let wire = asap_types::test_support::physical_asap_dag_at_query_time(&rewritten).unwrap(); wire.validate().unwrap(); assert_eq!(wire.nodes.len(), dag.len()); } @@ -128,7 +128,7 @@ async fn sql_scalar_subquery_retains_its_cardinality_contract() { .unwrap(); root.validate_structure().unwrap(); assert!(root.children().len() > 1); - let wire = physical_common::compile_physical_asap_dag(&root).unwrap(); + let wire = asap_types::test_support::physical_asap_dag_at_query_time(&root).unwrap(); assert!(wire .edges .iter() @@ -161,7 +161,7 @@ async fn sql_sum_example_executes_with_sql_null_semantics() { let NonASAPOp::Scan { source, .. } = logical_scan.expect_non_asap() else { panic!() }; - let wire = physical_common::compile_physical_asap_dag(&root).unwrap(); + let wire = asap_types::test_support::physical_asap_dag_at_query_time(&root).unwrap(); let scan = wire .nodes .iter() @@ -236,146 +236,3 @@ async fn sql_window_and_filtered_aggregate_types() { assert_eq!(root.schema.fields[0], Field::plain("s", DataType::Int64, true), "{query}"); } } - -/// A real query batch replaces each SUM by its exact accumulator state, -/// retains two result roots, and executes both selected plans. Pass 2 -/// shares the identical state, and building the shared sub-DAG once per -/// evaluation for both queries (`MatAtQuery`, on the state or on its -/// finalized sum above it) is the cheapest plan, so both roots read one -/// state. No replacement dag is constructed by the test. -#[tokio::test] -async fn batch_planning_replaces_summary_operators() { - use asap_executor::values::Value; - use asap_planner::PlanningModels; - use asap_planner::{e2e_plan, FrontendInput, UserInput}; - use asap_types::workload::*; - let queries = [ - "SELECT SUM(bytes) + 1 AS result FROM requests", - "SELECT SUM(bytes) * 2 AS result FROM requests", - ]; - let workload = PlanningWorkload { - query_workload: QueryWorkload { - language: QueryLanguage::SQL(SqlDialect::DataFusionSQL), - query_batch: Some( - queries - .iter() - .map(|query| BatchEntry { - query: Query((*query).into()), - requirements: QueryRequirements { - accuracy: AccuracyTarget::exact(), - ..Default::default() - }, - predictability: Predictability::Unknown, - invocations: 2, - execute_at: None, - range_evaluation: None, - }) - .collect(), - ), - repeating_queries: None, - }, - // `requests` at rest, holding a million rows. - data_workload: Some(DataWorkload { - arrival: DataArrival::AtRest, - sources: [( - "requests".to_string(), - asap_types::workload::SourceFacts { - rows_held: Some(1_000_000), - ..Default::default() - }, - )] - .into(), - }), - }; - let catalog = SqlCatalog::new().with_table( - "requests", - Schema::new(vec![Field::plain("bytes", DataType::Float64, false)]), - ); - let output = e2e_plan(UserInput::new( - &workload, - FrontendInput::Sql { catalog: &catalog }, - PlanningModels::builtin(), - )) - .await - .unwrap(); - assert_eq!(output.entry_indices(), [0, 1]); - assert_eq!(output.roots().len(), 2); - let states: Vec<_> = output - .operators() - .into_iter() - .filter(|n| matches!(n.asap(), Some(ASAPOp::SummaryAgg { .. }))) - .collect(); - let [state] = &states[..] else { - panic!("one SUM state for both queries: {states:?}") - }; - assert!(output.operators().into_iter().any(|n| { - matches!( - n.materialization, - asap_types::physical::Materialization::MatAtQuery(_) - ) && OperatorNode::reachable(&n) - .iter() - .any(|m| Rc::ptr_eq(m, state)) - })); - for (plan, expected) in output.plans.iter().zip([31.0, 60.0]) { - let root = &plan.root; - root.validate_structure().unwrap(); - assert!(OperatorNode::reachable(root) - .iter() - .any(|n| Rc::ptr_eq(n, state))); - // The table `requests` bound to its two rows. - let rows = physical_common::execute_raw_rows( - root, - vec![vec![Value::Float64(10.0)], vec![Value::Float64(20.0)]], - ); - assert_eq!(rows.len(), 1); - assert!( - matches!(rows[0][0], Value::Float64(v) if v == expected), - "{:?}", - rows - ); - } - // The batch exports as one physical DAG: a root per query, the one SUM - // state, and the shared node the selected plan builds once. - let workload_dag = output.physical_asap_dag().unwrap(); - assert_eq!(workload_dag.roots.len(), 2); - assert_ne!(workload_dag.roots[0], workload_dag.roots[1]); - let states: Vec<_> = workload_dag - .nodes - .iter() - .filter(|n| { - matches!( - n.payload, - asap_types::ir::physical_export::PhysicalASAPOperatorPayload::ASAP( - asap_types::ir::ASAPOp::SummaryAgg { .. } - ) - ) - }) - .collect(); - let [state] = &states[..] else { - panic!("one SUM state: {states:?}") - }; - // The node built once is the state or a node above it. - let reaches_state = |from| { - let mut stack = vec![from]; - let mut seen = std::collections::HashSet::new(); - while let Some(id) = stack.pop() { - if id == state.id { - return true; - } - if seen.insert(id) { - stack.extend( - workload_dag - .edges - .iter() - .filter(|e| e.consumer == id) - .map(|e| e.producer), - ); - } - } - false - }; - assert!(workload_dag.nodes.iter().any(|n| matches!( - n.materialization, - asap_types::physical::Materialization::MatAtQuery(_) - ) && reaches_state(n.id))); -} diff --git a/crates/integration-tests/tests/per_computation_candidates_sql.rs b/crates/integration-tests/tests/per_computation_candidates_sql.rs index 6902cd681..564381f0a 100644 --- a/crates/integration-tests/tests/per_computation_candidates_sql.rs +++ b/crates/integration-tests/tests/per_computation_candidates_sql.rs @@ -26,9 +26,7 @@ async fn targets(sql: &str) -> Vec { .await .unwrap_or_else(|e| panic!("lower failed for {sql:?}: {e}")); generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(root), - ]), + &asap_logical_asap_optimization::memo_of_queries(&[QueryRoot::Operator(root)]).unwrap(), ) .unwrap() .targets @@ -79,7 +77,7 @@ fn reads_quantile_sketch(node: &std::rc::Rc) -> bo Some(asap_types::ir::ASAPOp::SummaryAgg { family: asap_types::ir::schema::FieldDataType::Sketch(kind, _), .. - }) if matches!(kind.algorithm(), SketchAlgorithm::Kll | SketchAlgorithm::DDSketch) + }) if matches!(kind, SketchAlgorithm::Kll | SketchAlgorithm::DDSketch) ) }) } @@ -154,7 +152,7 @@ fn reads_ddsketch(node: &std::rc::Rc) -> bool { Some(asap_types::ir::ASAPOp::SummaryAgg { family: asap_types::ir::schema::FieldDataType::Sketch(kind, _), .. - }) if matches!(kind.algorithm(), SketchAlgorithm::DDSketch) + }) if matches!(kind, SketchAlgorithm::DDSketch) ) }) } @@ -214,9 +212,7 @@ async fn heap_updates_over(fields: Vec, sql: &str) -> Vec<(String, String let accuracy = AccuracyTarget::max_error(0.01).failure_probability(0.01); let root = lower_sql(sql, &catalog, accuracy).await.unwrap(); let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(root), - ]), + &asap_logical_asap_optimization::memo_of_queries(&[QueryRoot::Operator(root)]).unwrap(), ) .unwrap(); let name = |expr: &SummaryInputExpr| match expr { diff --git a/crates/integration-tests/tests/per_series_exact_execution.rs b/crates/integration-tests/tests/per_series_exact_execution.rs index e20fed25b..e71cc9e35 100644 --- a/crates/integration-tests/tests/per_series_exact_execution.rs +++ b/crates/integration-tests/tests/per_series_exact_execution.rs @@ -14,13 +14,10 @@ use asap_executor::sources::InMemoryDataStore; use asap_executor::values::{Batch, Value}; use asap_integration_tests::fixtures::lower_promql; use asap_types::ir::operator::{Reduction, Source}; -use asap_types::ir::physical_export::{compile_physical_asap_dag, PhysicalASAPOperatorPayload}; +use asap_types::ir::physical_export::PhysicalASAPOperatorPayload; use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::schema::{Schema, PROMQL_SERIES_IDENTITY}; -use asap_types::ir::{ - apply_materialization_timings, MaterializationAssignment, NodeToTimedNodeCache, NonASAPOp, - OperatorNode, -}; +use asap_types::ir::{NonASAPOp, OperatorNode}; use futures::{executor::block_on, StreamExt}; /// One raw data sample: every label of its series, its time and value. @@ -96,13 +93,8 @@ fn series_labels(schema: &Schema, row: &[Value]) -> BTreeMap { /// Run `query`'s exact per-entity plan at query time over `samples`, bound /// at its `Scan` of `x`: each answered series' `instance` and value. fn execute(query: &str, samples: &[RawDataSample]) -> Vec<(String, f64)> { - let timed = apply_materialization_timings( - &per_entity_plan(query), - &MaterializationAssignment::default(), - &mut NodeToTimedNodeCache::new(), - ) - .unwrap(); - let dag = compile_physical_asap_dag(&timed).unwrap(); + let plan = per_entity_plan(query); + let dag = asap_types::test_support::physical_asap_dag_at_query_time(&plan).unwrap(); let node = |matches: fn(&PhysicalASAPOperatorPayload) -> bool| { let found: Vec<_> = dag .nodes diff --git a/crates/integration-tests/tests/physical_common/mod.rs b/crates/integration-tests/tests/physical_common/mod.rs index e9f16cfa9..998ee0ea0 100644 --- a/crates/integration-tests/tests/physical_common/mod.rs +++ b/crates/integration-tests/tests/physical_common/mod.rs @@ -33,40 +33,6 @@ pub fn execute( }) } -#[allow(dead_code)] -pub fn compile_physical_asap_dag( - root: &std::rc::Rc, -) -> Result> { - compile_with(root, &asap_types::ir::MaterializationAssignment::default()) -} - -/// As [`compile_physical_asap_dag`], with every sub-DAG `MatAtIngestion` -/// where the DAG allows it. -#[allow(dead_code)] // Not every test binary sharing this module stores states at ingestion. -pub fn compile_at_ingestion_physical_asap_dag( - root: &std::rc::Rc, -) -> Result> { - compile_with( - root, - &asap_types::ir::MaterializationAssignment::with_default( - asap_types::physical::Materialization::MatAtIngestion( - asap_types::physical::Medium::Memory, - ), - ), - ) -} - -fn compile_with( - root: &std::rc::Rc, - assignment: &asap_types::ir::MaterializationAssignment, -) -> Result> { - let root = - asap_types::ir::apply_materialization_timings(root, assignment, &mut Default::default())?; - Ok(asap_types::ir::physical_export::compile_physical_asap_dag( - &root, - )?) -} - /// Execute a relational DAG at query time against an in-memory connector /// for its one scan, scan predicates included; checks that no memory stays /// reserved afterwards. @@ -89,7 +55,7 @@ pub fn execute_raw_rows_at( use asap_executor::physical_planner::compile; use asap_types::ir::physical_export::PhysicalASAPOperatorPayload; use asap_types::ir::NonASAPOp; - let wire = compile_physical_asap_dag(root).unwrap(); + let wire = asap_types::test_support::physical_asap_dag_at_query_time(root).unwrap(); let (source, schema) = wire .nodes .iter() diff --git a/crates/integration-tests/tests/planner_layering_common/mod.rs b/crates/integration-tests/tests/planner_layering_common/mod.rs index 47357ac32..283b0f173 100644 --- a/crates/integration-tests/tests/planner_layering_common/mod.rs +++ b/crates/integration-tests/tests/planner_layering_common/mod.rs @@ -18,12 +18,12 @@ use std::collections::{BTreeMap, BTreeSet, HashSet}; use asap_cost::CostInputs; use asap_devtools::planning_stages::{lower_promql_roots, run_planning_stages_over}; use asap_logical_asap_optimization::DeploymentCapabilities; -use asap_plan_selection::reference_pricer::WholePlanSelection as Selection; use asap_types::ir::flat::{flatten, FlatDag, NodeId}; use asap_types::ir::physical_export::PhysicalASAPDAG; +use asap_types::ir::properties::ExecutionTiming; use asap_types::ir::schema::{FieldDataType, SketchAlgorithm, SketchParams, SketchStatistic}; use asap_types::ir::{ASAPOp, NonASAPOp, Operator, QueryRoot}; -use asap_types::physical::Materialization; +use asap_types::physical::Lifetime; use asap_types::physical::PhysicalWorkloadPlan; use asap_types::workload::{PlanningWorkload, QueryRecurrence, RepeatedDemand}; @@ -31,28 +31,6 @@ pub use asap_devtools::planner_layering_examples::*; pub type Payload = Operator; -/// Lower and plan a PromQL workload. -pub fn run_promql(workload: &PlanningWorkload) -> Run { - run_stages(workload, lower_promql(workload)) -} - -/// Lower and plan a PromQL workload, selecting under `settings` (a cost -/// model other than the default). -pub fn run_promql_under(workload: &PlanningWorkload, settings: &CostInputs) -> Run { - run_stages_under(workload, lower_promql(workload), settings) -} - -/// Lower and plan a PromQL workload for a deployment with `deployment`'s -/// capabilities, under the default cost model. -pub fn run_promql_for(workload: &PlanningWorkload, deployment: &DeploymentCapabilities) -> Run { - run_stages_with( - workload, - lower_promql(workload), - &CostInputs::default(), - deployment, - ) -} - /// Whether `form` answers a query window by merging several summaries. pub fn needs_merge(form: WindowForm, query_window_ms: u64) -> bool { match form { @@ -62,25 +40,6 @@ pub fn needs_merge(form: WindowForm, query_window_ms: u64) -> bool { } } -/// `roots` with every identical sub-DAG made one node (the identical- -/// expression rule, as the memo's group identity applies it). -#[allow(dead_code)] -pub fn share_identical(roots: &[QueryRoot]) -> Vec { - asap_logical_asap_optimization::test_support::share_common_sub_dags( - roots - .iter() - .enumerate() - .map(|(id, root)| match root { - QueryRoot::Operator(node) => (id, std::rc::Rc::clone(node)), - QueryRoot::Scalar(_) => panic!("operator root"), - }) - .collect(), - ) - .into_iter() - .map(|(_, node)| QueryRoot::Operator(node)) - .collect() -} - /// `roots` with no node shared between queries: each query reads its own /// copy of every node it reaches (sharing within one query is kept). Pass 1 /// composes a candidate from memo groups, so queries that read an identical @@ -114,22 +73,6 @@ pub fn unshare(roots: &[QueryRoot]) -> Vec { .collect() } -/// The number of distinct nodes (by identity) `roots` reach: smaller after -/// [`share_identical`] exactly when some sub-DAG was shared. -#[allow(dead_code)] -pub fn distinct_nodes(roots: &[QueryRoot]) -> usize { - roots - .iter() - .filter_map(|root| match root { - QueryRoot::Operator(node) => Some(node), - QueryRoot::Scalar(_) => None, - }) - .flat_map(asap_types::ir::OperatorNode::reachable) - .map(|node| std::rc::Rc::as_ptr(&node)) - .collect::>() - .len() -} - // ── Query-language frontends ────────────────────────────────────────── /// PromQL workload roots, each series' identity as a column. @@ -193,42 +136,6 @@ impl Physical { } } -#[derive(Debug, Clone)] -pub struct Run { - pub stage0_query_language_frontends: Logical, - pub logical: Vec, - pub physical: Vec, - /// Logical candidates physical ASAP-aware optimization cannot implement: - /// (logical id, reason). - pub pruned: Vec<(String, String)>, - pub selection: Selection, -} - -impl Run { - pub fn invalid(&self) -> BTreeMap<&str, &str> { - self.selection - .rejected - .iter() - .filter(|r| r.cause.failed_a_check()) - .map(|r| (r.id.as_str(), r.reason.as_str())) - .collect() - } - - pub fn cost(&self, id: &str) -> Option { - self.selection.costs.get(id).and_then(|c| c.cost_per_second) - } - - pub fn physical(&self, id: &str) -> &Physical { - self.physical.iter().find(|p| p.id == id).expect("id") - } - - pub fn physical_of<'a>(&'a self, logical: &'a Logical) -> impl Iterator { - self.physical - .iter() - .filter(move |p| p.from_logical == logical.id) - } -} - fn logical(id: String, roots: Vec) -> Logical { let dag = flatten(&roots).0; let query_roots = dag @@ -247,212 +154,6 @@ fn logical(id: String, roots: Vec) -> Logical { } } -/// The bounded production stages over `roots`, retaining the selected plan -/// and search diagnostics. Tests that require a particular algorithm or window -/// use `run_explicit_candidates` instead of assuming it appears in the archive. -pub fn run_stages(workload: &PlanningWorkload, roots: Vec) -> Run { - run_stages_under(workload, roots, &CostInputs::default()) -} - -/// [`run_stages`], selecting under `settings`. -pub fn run_stages_under( - workload: &PlanningWorkload, - roots: Vec, - settings: &CostInputs, -) -> Run { - run_stages_with( - workload, - roots, - settings, - &DeploymentCapabilities::default(), - ) -} - -/// [`run_stages`], selecting under `settings` for a deployment with -/// `deployment`'s capabilities. -pub fn run_stages_with( - workload: &PlanningWorkload, - roots: Vec, - settings: &CostInputs, - deployment: &DeploymentCapabilities, -) -> Run { - let models = asap_planner::PlanningModels::builtin() - .with_cost_inputs(settings) - .with_deployment_capabilities(deployment); - run_stages_over(workload, roots, models) -} - -fn run_stages_over( - workload: &PlanningWorkload, - roots: Vec, - models: asap_planner::PlanningModels<'_>, -) -> Run { - let entries: Vec<_> = workload.query_workload.entries().collect(); - let data = workload.data_workload.clone().unwrap_or_default(); - // The facade's run, with the reference pricer over the Stage 2 - // memo choices: the whole-plan costs and rejections these tests read. - let run = run_planning_stages_over(roots, &entries, &data, None, models) - .expect("planning stages run"); - let reference = run.reference.expect("the memo is enumerated"); - let logical_candidates = reference - .choices - .into_iter() - .map(|c| { - let roots = c.roots.unwrap_or_else(|e| panic!("{} composes: {e}", c.id)); - logical(c.id, roots) - }) - .collect(); - let physical = reference - .physical - .into_iter() - .map(|candidate| Physical { - id: candidate.id.clone(), - from_logical: candidate.from_logical.clone(), - query_roots: candidate.dag.roots.clone(), - candidate, - }) - .collect(); - Run { - stage0_query_language_frontends: logical( - "stage0-query-language-frontends".into(), - run.stage0_query_language_frontends, - ), - logical: logical_candidates, - physical, - pruned: reference.stage2_failed, - selection: reference.selection.expect("plan selection selects"), - } -} - -/// Evaluate deliberately supplied logical configurations. This fixture helper -/// does not search or enumerate cross-query combinations; it exercises the -/// existing physical rules and evaluator on the computations named by a test. -/// Production-search tests continue to use `run_stages`. -pub fn run_explicit_candidates( - workload: &PlanningWorkload, - original_roots: Vec, - candidates: Vec>, - models: asap_planner::PlanningModels<'_>, -) -> Run { - let entries: Vec<_> = workload.query_workload.entries().collect(); - let data = workload.data_workload.clone().unwrap_or_default(); - let targets: Vec<_> = entries - .iter() - .map(|entry| entry.requirements.accuracy.clone()) - .collect(); - // Explicit candidates are composed outside the memo: their summaries are - // sized as Pass 1 sizes a candidate, which Stage 3 only checks. - let logical: Vec<_> = candidates - .into_iter() - .enumerate() - .map(|(index, roots)| { - let roots = asap_accuracy::propagation::recompose_guarantees(&roots) - .expect("explicit fixture contracts compose"); - logical( - format!("P{}", index + 1), - asap_accuracy::sizing::with_summaries_sized_for_targets(&roots, &targets, &data) - .expect("explicit fixture summaries rebuild"), - ) - }) - .collect(); - let mut physical = Vec::new(); - for candidate in &logical { - for (index, mut plan) in asap_physical_asap_optimization::optimize_physical_asap( - &candidate.id, - &candidate.roots, - &entries, - &data, - models.deployment_capabilities.query_time_results_retained, - ) - .expect("explicit computation has executable physical configurations") - .into_iter() - .enumerate() - { - plan.id = if index == 0 { - candidate.id.clone() - } else { - format!("{}-m{}", candidate.id, index) - }; - physical.push(Physical { - id: plan.id.clone(), - from_logical: plan.from_logical.clone(), - query_roots: plan.dag.roots.clone(), - candidate: plan, - }); - } - } - // Stage 2's deployment rules (raw data retention) prune the plans the - // deployment cannot run before Stage 3 prices the rest. - let mut plans: Vec<_> = physical.iter().map(|plan| plan.candidate.clone()).collect(); - asap_devtools::memo_pricing::retain_deployable(&mut plans, models); - physical.retain(|p| plans.iter().any(|plan| plan.id == p.id)); - let selection = asap_devtools::memo_pricing::reference_select(&plans, &entries, &data, models) - .expect("explicit fixture contains a valid physical configuration"); - Run { - stage0_query_language_frontends: logical_stage0(original_roots), - logical, - physical, - pruned: Vec::new(), - selection, - } -} - -pub fn run_pattern_b_fixture( - workload: &PlanningWorkload, - settings: &CostInputs, - deployment: &DeploymentCapabilities, -) -> Run { - run_explicit_candidates( - workload, - lower_promql(workload), - pattern_b_computations(workload), - asap_planner::PlanningModels::builtin() - .with_cost_inputs(settings) - .with_deployment_capabilities(deployment), - ) -} - -/// Three fixed Pattern-B computations used by cost/retention fixtures: -/// exact, whole-window KLL, and a KLL chain of sixty-second tumbling windows. -pub fn pattern_b_computations(workload: &PlanningWorkload) -> Vec> { - use asap_logical_asap_optimization::test_support::{ - compose_logical_candidate, generate_per_computation_candidates, - }; - use asap_logical_asap_optimization::test_support::{ - tumbling_window_candidates, window_composition_pass2_candidates, - }; - use asap_types::stages::HowToComputeAggregateMeasure; - let roots = lower_promql(workload); - let local = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&roots), - ) - .unwrap(); - assert_eq!( - local.targets.len(), - 1, - "Pattern B has one quantile computation" - ); - let kll = local.targets[0] - .ways_to_compute - .iter() - .position(|candidate| { - matches!( - candidate, - HowToComputeAggregateMeasure::Summary(SketchAlgorithm::Kll) - ) - }) - .unwrap(); - let whole: Vec<_> = compose_logical_candidate(&local, &[kll]).unwrap(); - let tumbling_windows = tumbling_window_candidates(&whole, &[Some(60_000)]); - assert_eq!(tumbling_windows.len(), 1); - let chain = window_composition_pass2_candidates(&whole, &tumbling_windows) - .unwrap() - .remove(0) - .roots - .expect("sixty-second tumbling windows compose"); - vec![roots, whole, chain] -} - fn logical_stage0(roots: Vec) -> Logical { logical("stage0-query-language-frontends".into(), roots) } @@ -528,7 +229,7 @@ pub fn closure(dag: &impl ExportedDag, root: NodeId) -> HashSet { } /// The queries (indexes into `query_roots`) whose result depends on `node`. -pub fn readers(dag: &impl ExportedDag, query_roots: &[NodeId], node: NodeId) -> BTreeSet { +pub fn queries(dag: &impl ExportedDag, query_roots: &[NodeId], node: NodeId) -> BTreeSet { query_roots .iter() .enumerate() @@ -576,27 +277,6 @@ pub fn is_merge(payload: &Payload) -> bool { matches!(payload, Operator::ASAP(ASAPOp::SummaryMerge { .. })) } -/// The sketch build nodes (`SummaryAgg` or tumbling-window `SummaryTumblingWindowChain` over a -/// sketch family) of `dag`, with their algorithm and parameters. -pub fn sketch_builds(dag: &impl ExportedDag) -> Vec<(NodeId, SketchAlgorithm, SketchParams)> { - dag.node_ids() - .into_iter() - .filter_map(|id| match dag.payload(id) { - Operator::ASAP( - ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), - .. - } - | ASAPOp::SummaryTumblingWindowChain { - family: FieldDataType::Sketch(kind, _), - .. - }, - ) => Some((id, *kind.algorithm(), kind.params().clone())), - _ => None, - }) - .collect() -} - /// The estimation nodes that read `build`, directly or through merges. pub fn estimates_of(dag: &impl ExportedDag, build: NodeId) -> Vec<(NodeId, SketchStatistic)> { let mut out = Vec::new(); @@ -619,168 +299,26 @@ pub fn estimates_of(dag: &impl ExportedDag, build: NodeId) -> Vec<(NodeId, Sketc out } -/// One query's per-computation choice: the sorted sketch algorithms built in -/// its closure, or `"exact"` when it builds none. -pub fn query_option(dag: &impl ExportedDag, query_roots: &[NodeId], query: usize) -> String { - let reach = closure(dag, query_roots[query]); - let mut names: Vec<_> = sketch_builds(dag) - .into_iter() - .filter(|(id, ..)| reach.contains(id)) - .map(|(_, algorithm, _)| format!("{algorithm:?}")) - .collect(); - names.sort(); - if names.is_empty() { - "exact".into() - } else { - names.join("+") - } -} - /// Nodes reachable from more than one query root. pub fn cross_query_nodes(dag: &impl ExportedDag, query_roots: &[NodeId]) -> BTreeSet { dag.node_ids() .into_iter() - .filter(|&id| readers(dag, query_roots, id).len() > 1) + .filter(|&id| queries(dag, query_roots, id).len() > 1) .collect() } -pub fn assert_valid_and_uniquely_named(run: &Run) { - let ids: BTreeSet<_> = run.logical.iter().map(|c| c.id.as_str()).collect(); - assert_eq!(ids.len(), run.logical.len(), "unique logical ids"); - for c in &run.logical { - for root in &c.roots { - root.validate_structure() - .unwrap_or_else(|e| panic!("{}: {e}", c.id)); - } - } -} - -/// No candidate is discarded before plan selection: physical ASAP-aware -/// optimization gives each logical candidate `L` exactly one physical -/// candidate with nothing materialized, `P`, and one `P-m` for each -/// other materialization assignment. -pub fn assert_one_unmaterialized_physical_candidate_per_logical(run: &Run) { - assert!(run.pruned.is_empty(), "pruned: {:?}", run.pruned); - let unmaterialized: Vec<_> = run - .physical - .iter() - .filter(|p| !p.id.contains("-m")) - .collect(); - let sources: BTreeSet<_> = unmaterialized - .iter() - .map(|p| p.from_logical.as_str()) - .collect(); - let logical: BTreeSet<_> = run.logical.iter().map(|c| c.id.as_str()).collect(); - assert_eq!(sources, logical); - assert_eq!(unmaterialized.len(), run.logical.len()); - for p in &run.physical { - let base = p.id.split("-m").next().unwrap(); - assert_eq!(&base[1..], &p.from_logical[1..], "{}", p.id); - let unmaterialized = !p.id.contains("-m"); - assert_eq!(unmaterialized, !p.candidate.materializes_any(), "{}", p.id); - if unmaterialized { - for n in &p.dag().nodes { - assert_eq!( - materialization(p, n.id), - MaterializationOption::NotMaterialized - ); - } - } - } - let ids: BTreeSet<_> = run.physical.iter().map(|p| p.id.as_str()).collect(); - assert_eq!(ids.len(), run.physical.len(), "unique physical ids"); -} - -/// The physical candidates with nothing materialized, one per logical -/// candidate. -pub fn unmaterialized(run: &Run) -> impl Iterator { - run.physical - .iter() - .filter(|p| !p.candidate.materializes_any()) -} - -/// One selected id; every other candidate rejected once, with a reason. -pub fn assert_selects_one_and_explains_the_rest(run: &Run) { - let all: BTreeSet<_> = run.physical.iter().map(|p| p.id.clone()).collect(); - let mut accounted: BTreeSet<_> = run - .selection - .rejected - .iter() - .map(|r| r.id.clone()) - .collect(); - assert_eq!( - accounted.len(), - run.selection.rejected.len(), - "rejected once each" - ); - assert!(run.selection.rejected.iter().all(|r| !r.reason.is_empty())); - assert!( - accounted.insert(run.selection.selected.clone()), - "selected is not rejected" - ); - assert_eq!(accounted, all); -} - -/// The selected plan costs no more than any valid candidate. -pub fn assert_selects_cheapest_valid(run: &Run) { - let invalid = run.invalid(); - let best = run - .cost(&run.selection.selected) - .expect("selected is priced"); - for p in run - .physical - .iter() - .filter(|p| !invalid.contains_key(p.id.as_str())) - { - // Totals within rounding of each other tie (plan selection's rule). - let cost = run.cost(&p.id).unwrap(); - assert!(best <= cost + cost.abs() * 1e-9, "{} is cheaper", p.id); - } -} - -/// Each valid candidate's cost has one entry per DAG node, `total` is their -/// sum, and each node's multiplier is [`computations`]. Invalid candidates -/// are not priced. -pub fn assert_each_node_charged_per_consuming_query(run: &Run) { - let invalid = run.invalid(); - for p in &run.physical { - if invalid.contains_key(p.id.as_str()) { - assert!(run.cost(&p.id).is_none(), "{} is priced", p.id); - continue; - } - let cost = &run.selection.costs[&p.id]; - let nodes: BTreeSet<_> = p.dag().nodes.iter().map(|n| n.id).collect(); - let charged: BTreeSet<_> = cost.per_node.keys().copied().collect(); - assert_eq!(charged, nodes, "{}", p.id); - let sum: f64 = cost - .per_node - .values() - .map(|c| c.cost_per_second.unwrap()) - .sum(); - assert!( - (cost.cost_per_second.unwrap() - sum).abs() <= 1e-9 * sum.abs().max(1.0), - "{}", - p.id - ); - let times = computations(p.dag()); - for (id, c) in &cost.per_node { - assert_eq!(c.multiplier, times[id], "{}: multiplier of {id:?}", p.id); - } - } -} - /// How many times each node is computed per evaluation of its consumers (#509; the /// rule of plan selection's private `computations`): 1 for a materialized -/// node; for a `NotMaterialized` one, the number of consumers that reach +/// node; for a not-materialized one, the number of consumers that reach /// it. A consumer is a query (operator or scalar root) or a materialized /// node, whose walk starts at its inputs; a walk stops at materialized -/// nodes and counts each node once. The `MatAtIngestion` consumers of one +/// nodes and counts each node once. The consumers materialized at ingestion time of one /// refresh window share one ingestion run and count once (#1151). pub fn computations(dag: &PhysicalASAPDAG) -> BTreeMap { - let materialized: BTreeSet<_> = dag + let materialized_nodes: BTreeSet<_> = dag .nodes .iter() - .filter(|n| n.materialization != Materialization::NotMaterialized) + .filter(|n| n.lifetime != asap_types::physical::Lifetime::None) .map(|n| n.id) .collect(); // Each walk's start and, for a consumer refreshed at ingestion, its @@ -794,22 +332,22 @@ pub fn computations(dag: &PhysicalASAPDAG) -> BTreeMap { .iter() .map(|(_, expr)| (expr.operator_refs().into_iter().copied().collect(), None)), ) - .chain(materialized.iter().map(|&id| { - let at_ingestion = matches!( - dag.nodes - .iter() - .find(|n| n.id == id) - .unwrap() - .materialization, - Materialization::MatAtIngestion(_) - ); + .chain(materialized_nodes.iter().map(|&id| { + let at_ingestion = dag + .nodes + .iter() + .find(|n| n.id == id) + .unwrap() + .output_state + .timing + == ExecutionTiming::IngestionTime; let window = at_ingestion.then(|| dag.refresh_window(id)).flatten(); (dag.producers(id), window) })); let mut times: BTreeMap<_, _> = dag .nodes .iter() - .map(|n| (n.id, u64::from(materialized.contains(&n.id)))) + .map(|n| (n.id, u64::from(materialized_nodes.contains(&n.id)))) .collect(); // The (node, refresh window) pairs whose ingestion run is counted. let mut runs = HashSet::new(); @@ -817,7 +355,7 @@ pub fn computations(dag: &PhysicalASAPDAG) -> BTreeMap { let mut seen = HashSet::new(); let mut stack = start; while let Some(id) = stack.pop() { - if materialized.contains(&id) || !seen.insert(id) { + if materialized_nodes.contains(&id) || !seen.insert(id) { continue; } if window.is_none_or(|window| runs.insert((id, window))) { @@ -838,32 +376,30 @@ pub fn evaluation_interval_ms(workload: &PlanningWorkload, query: usize) -> Opti } } -// ── Materialization ────────────────────────────────────────────────────── +// ── Materialized outputs ───────────────────────────────────────────────────────── -/// A node's materialization option (#509 "Materialization"), without its -/// storage medium. +/// Whether a node's output is materialized and at which timing, without how long +/// (`OneEvaluation` or `AcrossEvaluations`) or its storage medium. #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] -pub enum MaterializationOption { - /// Runs as data arrives; output stored before any query asks. - MatAtIngestion, +pub enum Materializing { + /// Runs as data arrives; output materialized until a query reads it. + MaterializedAtIngestion, /// Runs when a query first needs it; output reused by the other - /// queries of the evaluation, or, as `MatAtQueryKeptAcrossEvaluations`, - /// kept for later evaluations. - MatAtQuery, - /// Runs at query time for each execution; output discarded. + /// queries of the evaluation, or, as query-time `AcrossEvaluations`, + /// retained for later evaluations. + MaterializedAtQueryTime, + /// Output discarded after its query (lifetime `None`). NotMaterialized, } -/// The materialization option physical ASAP-aware optimization assigned to -/// `node` (#667). -pub fn materialization(p: &Physical, node: NodeId) -> MaterializationOption { +/// Whether physical ASAP-aware optimization materializes `node`'s output, and at +/// which timing (#667). +pub fn materializing(p: &Physical, node: NodeId) -> Materializing { let n = p.dag().nodes.iter().find(|n| n.id == node).expect("node"); - match n.materialization { - Materialization::MatAtIngestion(_) => MaterializationOption::MatAtIngestion, - Materialization::MatAtQuery(_) | Materialization::MatAtQueryKeptAcrossEvaluations(_) => { - MaterializationOption::MatAtQuery - } - Materialization::NotMaterialized => MaterializationOption::NotMaterialized, + match (n.output_state.timing, n.lifetime) { + (_, Lifetime::None) => Materializing::NotMaterialized, + (ExecutionTiming::IngestionTime, _) => Materializing::MaterializedAtIngestion, + (ExecutionTiming::QueryTime, _) => Materializing::MaterializedAtQueryTime, } } @@ -872,25 +408,6 @@ pub fn runs_at_ingestion(p: &Physical, node: NodeId) -> bool { n.output_state.timing == asap_types::ir::properties::ExecutionTiming::IngestionTime } -/// Every node upstream of an ingestion-time node also runs at ingestion time. -pub fn assert_ingestion_upstream_is_ingestion(run: &Run) { - for p in &run.physical { - for n in &p.dag().nodes { - if !runs_at_ingestion(p, n.id) { - continue; - } - for up in closure(p.dag(), n.id) { - assert!( - runs_at_ingestion(p, up), - "{}: {up:?} feeds ingestion-time {:?} at query time", - p.id, - n.id - ); - } - } - } -} - // ── Pending adapters ───────────────────────────────────────────────────── /// The window summary a summary build node maintains (#509 Pass 2, @@ -923,7 +440,7 @@ pub fn window_form(dag: &impl ExportedDag, build: NodeId) -> WindowForm { } } -/// How far back from an evaluation a materialized node's kept output +/// How far back from an evaluation a materialized node's materialized output /// reaches in event time, derived the way plan selection prices what it /// keeps: a tumbling-window chain keeps its `N` /// tumbling windows of width `w`, plus the one being built when maintained at diff --git a/crates/integration-tests/tests/planner_layering_example1.rs b/crates/integration-tests/tests/planner_layering_example1.rs deleted file mode 100644 index ca983c655..000000000 --- a/crates/integration-tests/tests/planner_layering_example1.rs +++ /dev/null @@ -1,951 +0,0 @@ -//! Rule-fixture and production acceptance for Example 1: aggregation across dimensions. -//! -//! Numerical, accuracy, and operator assertions use explicitly supplied -//! configurations of the existing rules and physical evaluator. They do not -//! assume a bounded MILP run archives every supported workload plan. Separate -//! production checks exercise memo search and validated output selection. -//! Unsupported algorithms retain their existing ignored tests. - -use crate::planner_layering_common; - -use std::collections::{BTreeMap, BTreeSet, HashSet}; - -use asap_devtools::memo_pricing::reference_select as select_plan; -use asap_planner::PlanningModels; -use asap_types::ir::flat::NodeId; -use asap_types::ir::schema::{FieldDataType, GroupingStrategy, HydraKind, SketchAlgorithm}; -use asap_types::ir::{ASAPOp, NonASAPOp, Operator, QueryRoot, ScalarExpr, SortKey}; -use asap_types::physical::{Materialization, Medium}; -use planner_layering_common::*; - -// ── Example 1 workload ─────────────────────────────────────────────────── - -fn pipeline() -> Run { - rule_fixture(&example1()) -} - -// Explicit rule fixtures: vary Q2's three documented algorithms while Q1 -// stays exact, and exercise source sharing and ten-second tumbling windows. This is not -// a production workload search or an exhaustive memo archive. -fn rule_fixture(workload: &asap_types::workload::PlanningWorkload) -> Run { - use asap_logical_asap_optimization::test_support::{ - compose_logical_candidate, generate_per_computation_candidates, - }; - use asap_logical_asap_optimization::test_support::{ - tumbling_window_candidates, window_composition_pass2_candidates, - }; - use asap_types::stages::HowToComputeAggregateMeasure; - let roots = lower_promql(workload); - let local = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&roots), - ) - .unwrap(); - let mut computations = Vec::new(); - for family in [ - None, - Some(SketchAlgorithm::CmsWithHeap), - Some(SketchAlgorithm::CountSketchWithHeap), - ] { - let mut choice = vec![0; local.targets.len()]; - if let Some(family) = family { - let (target, option) = local - .targets - .iter() - .enumerate() - .find_map(|(target, choices)| { - (0..choices.choice_count()) - .find(|&option| { - matches!(choices.computation_for(option), - Some((HowToComputeAggregateMeasure::Summary(algorithm), _)) if *algorithm == family) - }) - .map(|option| (target, option)) - }) - .expect("documented heap family offered locally"); - choice[target] = option; - } - let whole = unshare(&compose_logical_candidate(&local, &choice).unwrap()); - let mut windows = vec![whole.clone()]; - let tumbling_windows = tumbling_window_candidates(&whole, &[Some(10_000), Some(10_000)]); - windows.extend( - window_composition_pass2_candidates(&whole, &tumbling_windows) - .unwrap() - .into_iter() - .map(|candidate| candidate.roots.unwrap()), - ); - // Each window form alone and with its identical input shared. The - // memo shares an identical range selector whole, never its scan - // alone; that variant reads the same rows twice and is dominated. - for window in windows { - let shared = share_identical(&window); - let sharing = distinct_nodes(&shared) < distinct_nodes(&window); - computations.push(window); - if sharing { - computations.push(shared); - } - } - } - run_explicit_candidates(workload, roots, computations, PlanningModels::builtin()) -} - -#[test] -fn production_memo_selects_an_executable_accurate_example1_plan() { - let workload = example1(); - let run = run_stages_under(&workload, lower_promql(&workload), &Default::default()); - assert_selects_cheapest_valid(&run); - compile_in_runtime(run.physical(&run.selection.selected)).unwrap(); -} - -// ── Helpers ────────────────────────────────────────────────────────────── - -/// Q2's per-computation option for its top-k, read off its summary build node. -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] -enum Q2Option { - Exact, - CountMinHeapPerJob, - CountSketchHeapPerJob, - Hydra, -} - -const PASS1_Q2_OPTIONS: [Q2Option; 3] = [ - Q2Option::Exact, - Q2Option::CountMinHeapPerJob, - Q2Option::CountSketchHeapPerJob, -]; - -/// Sketch families built in `nodes`, as Example 1's Q2 options. -fn sketch_options(dag: &impl ExportedDag, nodes: &HashSet) -> BTreeSet { - nodes - .iter() - .filter_map(|&id| match dag.payload(id) { - Operator::ASAP( - ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, grouping), - .. - } - | ASAPOp::SummaryTumblingWindowChain { - family: FieldDataType::Sketch(kind, grouping), - .. - }, - ) => Some(match (grouping, kind.algorithm()) { - ( - GroupingStrategy::SharedMultiSubpopulation { - kind: HydraKind::HydraCms, - .. - }, - _, - ) => Q2Option::Hydra, - (GroupingStrategy::PerSubpopulationInstance, SketchAlgorithm::CmsWithHeap) => { - Q2Option::CountMinHeapPerJob - } - ( - GroupingStrategy::PerSubpopulationInstance, - SketchAlgorithm::CountSketchWithHeap, - ) => Q2Option::CountSketchHeapPerJob, - (other, algorithm) => { - panic!("summary family outside Example 1: {algorithm:?} {other:?}") - } - }), - _ => None, - }) - .collect() -} - -/// Whether every summary of `dag` is built over its whole window: no -/// tumbling windows (Pass 2's window-composition rule). -fn whole_windows(dag: &impl ExportedDag) -> bool { - dag.node_ids().into_iter().all(|id| { - !matches!( - dag.payload(id), - Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) - ) - }) -} - -fn roots(query_roots: &[NodeId]) -> (NodeId, NodeId) { - assert_eq!(query_roots.len(), 2, "every candidate covers Q1 and Q2"); - (query_roots[0], query_roots[1]) -} - -/// Q2's option and whether Q1 and Q2 share any node, for one candidate. -fn classify(dag: &impl ExportedDag, query_roots: &[NodeId]) -> (Q2Option, bool) { - let (q1, q2) = roots(query_roots); - let (c1, c2) = (closure(dag, q1), closure(dag, q2)); - let options = sketch_options(dag, &c2); - assert!(options.len() <= 1, "Q2 uses one option: {options:?}"); - let option = options.into_iter().next().unwrap_or(Q2Option::Exact); - (option, !c1.is_disjoint(&c2)) -} - -/// The candidate with Q2 `option` whose queries share no node and whose -/// summaries are built over whole windows (Pass 2 adds shared-input and -/// tumbling-window variants of each). -fn physical_with(run: &Run, option: Q2Option) -> &Physical { - let mut found = run - .physical - .iter() - .filter(|p| whole_windows(p.dag()) && classify(p.dag(), &p.query_roots) == (option, false)); - let p = found.next().unwrap_or_else(|| panic!("no {option:?}")); - assert!(found.next().is_none(), "one {option:?} candidate"); - p -} - -/// Example 1's panels repeat every 10 s. -fn every_10s() -> asap_types::workload::QueryRecurrence { - use asap_types::workload::{QueryRecurrence, RepeatedDemand, RepetitionInterval}; - QueryRecurrence::Repeated(RepeatedDemand::FixedInterval(RepetitionInterval(10_000))) -} - -/// The workload entry of an exact query evaluated every 10 s. -fn exact_every_10s() -> asap_types::workload::QueryWorkloadEntry { - asap_types::workload::QueryWorkloadEntry { - query: Default::default(), - requirements: Default::default(), - predictability: Default::default(), - recurrence: every_10s(), - range_evaluation: None, - } -} - -// ── Workload ───────────────────────────────────────────────────────────── - -/// The encoded workload is valid and normalizes to Q1 then Q2. -#[test] -fn workload_encodes_example1() { - let workload = example1(); - workload.validate().expect("valid workload"); - let entries: Vec<_> = workload.query_workload.entries().collect(); - assert_eq!(entries.len(), 2); - assert_eq!(entries[0].query.0, EXAMPLE1_Q1); - assert_eq!(entries[1].query.0, EXAMPLE1_Q2); -} - -// ── Query-language frontends ────────────────────────────────────────── - -/// The PromQL frontend lowers each query to the doc's chain. -#[test] -fn stage0_query_language_frontends_lower_each_query_to_doc_chain() { - let roots = lower_promql(&example1()); - let chains: Vec<_> = roots.iter().map(frontend_operations).collect(); - assert_eq!( - chains, - [ - ["aggregate:rate", "aggregate:sum", "scan", "time_range"], - ["aggregate:sum", "aggregate:top_k", "scan", "time_range"], - ] - ); -} - -/// The frontends yield one workload DAG with one root per query and no summaries. -#[test] -fn stage0_query_language_frontends_one_summary_free_workload_dag() { - let stage0_query_language_frontends = pipeline().stage0_query_language_frontends; - roots(&stage0_query_language_frontends.query_roots); - assert!(stage0_query_language_frontends - .dag - .nodes - .iter() - .all(|n| !is_summary(&n.operator))); -} - -/// The frontends keep Q1 and Q2 separate; sharing is a logical ASAP-aware optimization decision. -#[test] -fn stage0_query_language_frontends_queries_do_not_share_nodes() { - let stage0_query_language_frontends = pipeline().stage0_query_language_frontends; - assert!(!stage0_query_language_frontends.shared_input()); -} - -// ── Logical ASAP-aware optimization ────────────────────────────────────── - -/// Pass 1 gives 1 × 3 = 3 candidates without a shared input: Q2's top-k is -/// exact, Count-Min + heap or CountSketch + heap. (Pass 2 adds shared-input -/// and tumbling-window variants of each.) -#[test] -fn stage1_logical_asap_optimization_has_three_pass1_candidates() { - let run = pipeline(); - let found: Vec<_> = run - .logical - .iter() - .filter(|c| whole_windows(&c.dag)) - .map(|c| classify(&c.dag, &c.query_roots)) - .filter(|(_, shared)| !shared) - .collect(); - let expected: Vec<_> = PASS1_Q2_OPTIONS.into_iter().map(|o| (o, false)).collect(); - let mut sorted = found.clone(); - sorted.sort(); - assert_eq!(sorted, expected, "each Q2 option exactly once: {found:?}"); -} - -/// Q1 is exact in every candidate: no sketch build or estimate is reachable -/// from its root (its rate may be merged from exact tumbling windows). -#[test] -fn stage1_logical_asap_optimization_q1_is_always_exact() { - let run = pipeline(); - for c in &run.logical { - let (q1, _) = roots(&c.query_roots); - assert!( - closure(&c.dag, q1).iter().all(|&id| { - let payload = c.dag.payload(id); - !is_summary(payload) || is_merge(payload) - }), - "{}: Q1 reaches a summary", - c.id - ); - } -} - -/// Every Pass 1 target's exact option is its exact accumulator: Q1's rate -/// and sum and Q2's sum_over_time. Q2's heaps are whole-expression: they -/// absorb its sum_over_time, which is then not computed. -#[test] -fn stage1_logical_asap_optimization_exact_targets_use_accumulators() { - let run = pipeline(); - for c in &run.logical { - let expected = match classify(&c.dag, &c.query_roots).0 { - Q2Option::Exact => 3, - _ => 2, - }; - let accumulators = c - .dag - .nodes - .iter() - .filter(|n| { - matches!( - n.operator, - Operator::ASAP(ASAPOp::FinalizeExactAccumulator { .. }) - ) - }) - .count(); - assert_eq!(accumulators, expected, "{}", c.id); - } -} - -/// Q2's summary families are Count-Min + heap and CountSketch + heap, one per job. -#[test] -fn stage1_logical_asap_optimization_q2_summary_families() { - let run = pipeline(); - let families: BTreeSet<_> = run - .logical - .iter() - .flat_map(|c| { - let (_, q2) = roots(&c.query_roots); - sketch_options(&c.dag, &closure(&c.dag, q2)) - }) - .collect(); - assert_eq!( - families, - BTreeSet::from([ - Q2Option::CountMinHeapPerJob, - Q2Option::CountSketchHeapPerJob - ]) - ); -} - -/// Pass 1 also offers Hydra (HydraCms over all (job, series) keys, a heap per job) for Q2's top-k. -#[test] -#[ignore = "needs Pass 1 Hydra for grouped approximate counts (#703)"] -fn stage1_logical_asap_optimization_q2_offers_hydra() { - let run = pipeline(); - assert!(run - .logical - .iter() - .any(|c| classify(&c.dag, &c.query_roots).0 == Q2Option::Hydra)); -} - -/// Pass 2's identical-expression rule adds a shared-input variant of every Q2 option and keeps the independent one. -#[test] -fn stage1_logical_asap_optimization_keeps_independent_and_shared_variants() { - let run = pipeline(); - let found: Vec<_> = run - .logical - .iter() - .map(|c| classify(&c.dag, &c.query_roots)) - .collect(); - for option in PASS1_Q2_OPTIONS { - assert!(found.contains(&(option, false)), "{option:?} independent"); - assert!(found.contains(&(option, true)), "{option:?} shared"); - } -} - -/// Pass 2's window-composition rule builds Q1's rate and Q2's option in six -/// 10-s tumbling windows (the gcd of the 1-min window and the 10-s interval), -/// each alone and both together, next to the whole-window candidates: 27 -/// candidates in all. -#[test] -fn stage1_logical_asap_optimization_window_composition_adds_10s_tumbling_windows() { - let run = pipeline(); - let mut chains = BTreeSet::new(); - for c in &run.logical { - for id in c.dag.node_ids() { - if let Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { - family, - tumbling_window_width_ms, - tumbling_window_count, - .. - }) = c.dag.payload(id) - { - let state = match family { - FieldDataType::ExactAggregate(kind, _) => format!("{kind:?}"), - FieldDataType::Sketch(kind, _) => format!("{:?}", kind.algorithm()), - other => panic!("{}: {other:?}", c.id), - }; - let query = readers(&c.dag, &c.query_roots, id); - chains.insert(( - state, - query, - *tumbling_window_width_ms, - *tumbling_window_count, - )); - } - } - } - let expected: BTreeSet<_> = [ - ("Rate", 0), - ("Sum", 1), - ("CmsWithHeap", 1), - ("CountSketchWithHeap", 1), - ] - .into_iter() - .map(|(state, query)| (state.to_string(), BTreeSet::from([query]), 10_000, 6)) - .collect(); - assert_eq!(chains, expected); - let whole = run.logical.iter().filter(|c| whole_windows(&c.dag)).count(); - assert!(whole > 0); -} - -/// Only the raw input is ever shared between Q1 and Q2; no summary is shared. -#[test] -fn stage1_logical_asap_optimization_shares_input_but_never_a_summary() { - let run = pipeline(); - for c in &run.logical { - for id in cross_query_nodes(&c.dag, &c.query_roots) { - let p = c.dag.payload(id); - assert!( - matches!(relational(p).as_deref(), Some("scan" | "time_range")), - "{}: shared node {id:?} is not the range selector input: {p:?}", - c.id - ); - } - } -} - -/// Logical candidates are valid DAGs with unique ids. -#[test] -fn stage1_logical_asap_optimization_candidates_are_valid_and_uniquely_named() { - assert_valid_and_uniquely_named(&pipeline()); -} - -// ── Physical ASAP-aware optimization ───────────────────────────────────── - -/// No candidate is discarded before plan selection: each logical candidate -/// keeps one physical candidate with nothing materialized, 3 of them -/// without a shared input or tumbling windows. -#[test] -fn stage2_physical_asap_optimization_keeps_every_logical_candidate() { - let run = pipeline(); - assert_one_unmaterialized_physical_candidate_per_logical(&run); - let unshared = unmaterialized(&run) - .filter(|p| whole_windows(p.dag()) && !classify(p.dag(), &p.query_roots).1) - .count(); - assert_eq!(unshared, 3); -} - -/// Each physical candidate keeps its logical Q2 option, and one with -/// nothing materialized keeps its input sharing. Maintaining Q2's tumbling windows at -/// ingestion time splits a scan Q1 reads at query time, one copy per phase. -#[test] -fn stage2_physical_asap_optimization_preserves_logical_choices() { - let run = pipeline(); - for p in &run.physical { - let source = run.logical.iter().find(|c| c.id == p.from_logical).unwrap(); - let (option, shared) = classify(p.dag(), &p.query_roots); - let (logical_option, logical_shared) = classify(&source.dag, &source.query_roots); - assert_eq!(option, logical_option, "{} vs {}", p.id, source.id); - if !p.candidate.materializes_any() { - assert_eq!(shared, logical_shared, "{} vs {}", p.id, source.id); - } - } -} - -/// Exact TopK is implemented as a sort followed by a limit. -#[test] -fn stage2_physical_asap_optimization_exact_topk_is_sort_then_limit() { - let run = pipeline(); - let p = physical_with(&run, Q2Option::Exact); - let sort_then_limit = p.dag().edges.iter().any(|e| { - relational(p.dag().payload(e.producer)).as_deref() == Some("sort") - && relational(p.dag().payload(e.consumer)).as_deref() == Some("limit") - }); - assert!(sort_then_limit, "{}: no sort → limit", p.id); -} - -/// Exact Q2 sorts each job's series by the ranked value, descending, and a -/// `Project` over the limit restores the logical `[job, topk_10]` columns, -/// then keeps each chosen series' identity (its full label set). -#[test] -fn stage2_physical_asap_optimization_exact_topk_returns_job_and_topk_10() { - let run = pipeline(); - let p = physical_with(&run, Q2Option::Exact); - let QueryRoot::Operator(root) = &p.candidate.roots[1] else { - panic!("{}: Q2 is an operator root", p.id) - }; - let names: Vec<_> = root.schema.fields.iter().map(|f| f.name.as_str()).collect(); - assert_eq!( - names, - ["job", "topk_10", "$promql_series_identity"], - "{}", - p.id - ); - let NonASAPOp::Project { child: limit, .. } = root.expect_non_asap() else { - panic!("{}: Q2's root is a Project", p.id) - }; - let NonASAPOp::Limit { - n: Some(10), - child: sort, - .. - } = limit.expect_non_asap() - else { - panic!("{}: a limit 10 under the Project", p.id) - }; - let NonASAPOp::Sort { - keys, - partition_by, - child, - } = sort.expect_non_asap() - else { - panic!("{}: a sort under the limit", p.id) - }; - let [SortKey { - expr: ScalarExpr::Column(ranked), - ascending: false, - .. - }] = keys.as_slice() - else { - panic!("{}: one descending column key: {keys:?}", p.id) - }; - assert_eq!(child.schema.fields[*ranked].name, "value", "{}", p.id); - assert_eq!(partition_by.len(), 1, "{}: partitioned by job", p.id); -} - -/// A summary Q2 over its whole window is a build node feeding a top-10 -/// estimation node, with no merge. -#[test] -fn stage2_physical_asap_optimization_summary_topk_is_build_then_estimate() { - let run = pipeline(); - for p in &run.physical { - if classify(p.dag(), &p.query_roots).0 == Q2Option::Exact || !whole_windows(p.dag()) { - continue; - } - assert!( - !p.dag().nodes.iter().any(|n| is_merge(&n.payload)), - "{}: no window summaries, so no merge", - p.id - ); - let build_to_estimate = p.dag().edges.iter().any(|e| { - matches!( - p.dag().payload(e.producer), - Operator::ASAP(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(..), - .. - }) - ) && matches!( - p.dag().payload(e.consumer), - Operator::ASAP(ASAPOp::SummaryEstimate { .. }) - ) - }); - assert!(build_to_estimate, "{}: no build → estimate", p.id); - } -} - -/// Only tumbling windows can be maintained at ingestion time: the -/// whole-window builds' fixed interval has no phase, and no summary is read -/// by both queries. Q1's rate tumbling windows are counter states, which need each -/// series' samples in time order; continuously ingesting PromQL data is -/// taken to arrive in that order, so they are maintained too. Each tumbling window -/// chain is maintained or not independently: a logical candidate with `c` -/// chains has `2^c` physical candidates. A shared input (the highest node -/// both queries read: the scan, or the time range above it) is also built -/// once per evaluation (`MatAtQuery`), which adds one candidate: the one -/// with every chain at query time, since a chain maintained at ingestion -/// time leaves one query reading the input at query time. -#[test] -fn stage2_physical_asap_optimization_materializes_only_tumbling_windows_at_ingestion() { - let run = pipeline(); - let mut counter_maintained = 0; - let mut shared_once = 0; - for p in &run.physical { - for n in &p.dag().nodes { - match materialization(p, n.id) { - MaterializationOption::NotMaterialized => {} - MaterializationOption::MatAtIngestion => { - let Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { family, .. }) = - p.dag().payload(n.id) - else { - panic!("{}: {:?} is not a tumbling-window chain", p.id, n.id) - }; - if matches!( - family, - FieldDataType::ExactAggregate( - asap_types::ir::schema::ExactKind::Rate - | asap_types::ir::schema::ExactKind::Increase, - _ - ) - ) { - counter_maintained += 1; - } - } - MaterializationOption::MatAtQuery => { - let (q1, q2) = roots(&p.query_roots); - assert!( - closure(p.dag(), q1).contains(&n.id) - && closure(p.dag(), q2).contains(&n.id), - "{}: {:?} is built once for both queries", - p.id, - n.id - ); - assert!( - matches!( - relational(p.dag().payload(n.id)).as_deref(), - Some("scan" | "time_range") - ), - "{}: {:?} is the shared input", - p.id, - n.id - ); - shared_once += 1; - } - } - } - } - assert!( - counter_maintained > 0, - "Q1's rate tumbling windows are maintained" - ); - assert!( - shared_once > 0, - "explicit shared-source fixtures are materialized" - ); - let expected: usize = run - .logical - .iter() - .map(|c| { - let chains = c - .dag - .node_ids() - .into_iter() - .filter(|&id| { - matches!( - c.dag.payload(id), - Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) - ) - }) - .count(); - (1 << chains) + usize::from(c.shared_input()) - }) - .sum(); - assert_eq!(run.physical.len(), expected); -} - -/// Compile `p` in the physical planner (the runtime capability check), with -/// its materialized views as view slots and every `Scan` as raw data. -fn compile_in_runtime(p: &Physical) -> Result<(), String> { - use asap_executor::physical_planner::compile; - let mut dag = p.dag().clone(); - let views = dag - .set_retention(&Default::default(), &Default::default()) - .map_err(|e| format!("{}: {e}", p.id))?; - let roots: Vec = dag.roots.iter().map(|&id| id as u64).collect(); - compile(&dag, &views, &roots) - .map(|_| ()) - .map_err(|e| format!("{}: {e}", p.id)) -} - -// ── Plan selection ─────────────────────────────────────────────────────── - -/// Plan selection selects one candidate and gives every other one a reason. -#[test] -fn stage3_plan_selection_selects_one_and_explains_the_rest() { - assert_selects_one_and_explains_the_rest(&pipeline()); -} - -/// The exact Q2 is selected. Both heaps are invalid because a top-k output -/// has no guarantee and the deployment has no evidence for it. Count-Min's -/// weights are raw data samples of `http_requests_total`, which the workload -/// declares a counter, so they are non-negative. -#[test] -fn stage3_plan_selection_selects_exact_q2_and_rejects_both_heaps() { - let run = pipeline(); - let selected = run.physical(&run.selection.selected); - assert_eq!( - classify(selected.dag(), &selected.query_roots).0, - Q2Option::Exact - ); - let invalid = run.invalid(); - // Only heap candidates are invalid (with or without a shared input). - for id in invalid.keys() { - let p = run.physical(id); - assert_ne!( - classify(p.dag(), &p.query_roots).0, - Q2Option::Exact, - "{invalid:?}" - ); - } - let cms = &physical_with(&run, Q2Option::CountMinHeapPerJob).id; - assert!( - invalid[cms.as_str()].contains("no guarantee"), - "{invalid:?}" - ); - let count_sketch = &physical_with(&run, Q2Option::CountSketchHeapPerJob).id; - assert!( - invalid[count_sketch.as_str()].contains("no guarantee"), - "{invalid:?}" - ); -} - -/// Without the counter declaration, Count-Min's weights are not proven -/// non-negative: plan selection reads no type from the `_total` name. -#[test] -fn stage3_plan_selection_count_min_needs_the_counter_declaration() { - let mut workload = example1(); - workload - .data_workload - .as_mut() - .unwrap() - .sources - .get_mut(EXAMPLE1_METRIC) - .unwrap() - .metric_type = None; - let run = rule_fixture(&workload); - let invalid = run.invalid(); - let cms = &physical_with(&run, Q2Option::CountMinHeapPerJob).id; - assert!( - invalid[cms.as_str()].contains("non-negative"), - "{invalid:?}" - ); -} - -/// A shared NotMaterialized input is charged once per consumer, and both -/// queries read the same 1 m window, so the shared-input candidates cost -/// what the independent one does; their totals differ from it only by -/// floating-point rounding (the same charges summed in another grouping). -/// Built once per evaluation (`MatAtQuery`), the shared input is read once: -/// the shared time range (the doc's "Raw with a shared input") is selected, -/// at 4.38 instead of 7.01 per second. -#[test] -fn stage3_plan_selection_shared_input_ties_up_to_rounding_unless_built_once() { - let run = pipeline(); - let independent = physical_with(&run, Q2Option::Exact); - let base = run.cost(&independent.id).unwrap(); - let exact_shared = |p: &&Physical| { - whole_windows(p.dag()) && classify(p.dag(), &p.query_roots) == (Q2Option::Exact, true) - }; - let built_once = |p: &Physical| { - p.dag() - .nodes - .iter() - .any(|n| materialization(p, n.id) == MaterializationOption::MatAtQuery) - }; - let shared: Vec<_> = run - .physical - .iter() - .filter(exact_shared) - .filter(|p| !built_once(p)) - .map(|p| (p.id.clone(), run.cost(&p.id).unwrap())) - .collect(); - assert!(!shared.is_empty()); - assert!( - shared - .iter() - .any(|(_, total)| total.to_bits() != base.to_bits()), - "the rounding difference the tie tolerance absorbs: {base} vs {shared:?}" - ); - for (id, total) in &shared { - assert!( - (total - base).abs() <= 1e-9 * base, - "{id}: {total} vs {base}" - ); - } - let selected = run.physical(&run.selection.selected); - assert!(exact_shared(&selected) && built_once(selected)); - let time_range = selected - .dag() - .nodes - .iter() - .find(|n| materialization(selected, n.id) == MaterializationOption::MatAtQuery) - .unwrap(); - assert_eq!( - relational(&time_range.payload).as_deref(), - Some("time_range") - ); - let cost = run.cost(&selected.id).unwrap(); - assert!( - (cost - 4.3770).abs() < 1e-3 && (base - 7.0125).abs() < 1e-3, - "{cost} vs {base}" - ); -} - -/// Tumbling windows rebuilt at every evaluation cost more than the whole -/// window: the tumbling windows read the same rows and update the same states, and -/// merging them adds work. Heap tumbling windows are invalid like their whole-window -/// builds (and their merged top-k is only approximate). -#[test] -fn stage3_plan_selection_tumbling_windows_at_query_time_cost_more() { - let run = pipeline(); - let selected = run.cost(&run.selection.selected).unwrap(); - let invalid = run.invalid(); - let mut compared = 0; - for p in run.physical.iter().filter(|p| !whole_windows(p.dag())) { - match run.cost(&p.id) { - Some(cost) => { - assert!(cost > selected * (1.0 + 1e-9), "{}: {cost}", p.id); - compared += 1; - } - None => assert_ne!( - classify(p.dag(), &p.query_roots).0, - Q2Option::Exact, - "{}: {invalid:?}", - p.id - ), - } - } - assert!(compared > 0); -} - -/// The selected plan is the cheapest valid candidate for the whole workload. -#[test] -fn stage3_plan_selection_selects_cheapest_valid() { - assert_selects_cheapest_valid(&pipeline()); -} - -/// Every node of a valid candidate is charged once per consumer that reads -/// it: a shared `NotMaterialized` node is recomputed for each query, and a -/// materialized one is computed once. -#[test] -fn stage3_plan_selection_charges_each_node_per_consuming_query() { - assert_each_node_charged_per_consuming_query(&pipeline()); -} - -/// A hand-built candidate with a materialized node: Q1's plan read by two -/// queries, its `time_range` kept `MatAtQuery`. Plan selection charges the -/// two accumulators (rate, sum) and their finalizations twice, once per query; the walks from -/// both roots stop at the materialized `time_range`, which is computed once -/// and reads the scan once. -fn materialized_case() -> Run { - let mut workload = example1(); - let repeating = workload.query_workload.repeating_queries.as_mut().unwrap(); - repeating.truncate(1); - use asap_logical_asap_optimization::test_support::{ - compose_logical_candidate, generate_per_computation_candidates, - }; - let roots = lower_promql(&workload); - let local = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&roots), - ) - .unwrap(); - let exact = compose_logical_candidate(&local, &vec![0; local.targets.len()]).unwrap(); - let run = run_explicit_candidates(&workload, roots, vec![exact], PlanningModels::builtin()); - let mut p = run.physical[0].clone(); - p.candidate.roots = vec![p.candidate.roots[0].clone(); 2]; - let dag = &mut p.candidate.dag; - dag.roots = vec![dag.roots[0]; 2]; - p.query_roots = dag.roots.clone(); - let time_range = dag - .nodes - .iter_mut() - .find(|n| relational(&n.payload).as_deref() == Some("time_range")) - .unwrap(); - time_range.materialization = Materialization::MatAtQuery(Medium::Memory); - let selection = select_plan( - std::slice::from_ref(&p.candidate), - &[exact_every_10s(), exact_every_10s()], - workload.data_workload.as_ref().unwrap(), - PlanningModels::builtin(), - ) - .unwrap(); - Run { - physical: vec![p], - selection, - ..run - } -} - -/// The charging helper agrees with plan selection when a node is -/// materialized. -#[test] -fn stage3_plan_selection_charging_helper_follows_plan_selection_with_a_materialized_node() { - let run = materialized_case(); - let p = &run.physical[0]; - let mut by_kind: Vec<_> = run.selection.costs[&p.id] - .per_node - .iter() - .map(|(id, c)| (operation(p.dag().payload(*id)), c.multiplier)) - .collect(); - by_kind.sort(); - let expected = [ - ("finalize_exact_accumulator", 2), - ("finalize_exact_accumulator", 2), - ("scan", 1), - ("summary_agg", 2), - ("summary_agg", 2), - ("time_range", 1), - ]; - assert_eq!( - by_kind, - expected.map(|(kind, times)| (kind.to_string(), times)) - ); - assert_each_node_charged_per_consuming_query(&run); -} - -/// Charging the scan under a materialized node once per query is caught. -#[test] -#[should_panic(expected = "multiplier of")] -fn stage3_plan_selection_charging_helper_rejects_recounting_below_a_materialized_node() { - let mut run = materialized_case(); - let p = run.physical[0].clone(); - let scan = p - .dag() - .nodes - .iter() - .find(|n| relational(&n.payload).as_deref() == Some("scan")) - .unwrap() - .id; - run.selection - .costs - .get_mut(&p.id) - .unwrap() - .per_node - .get_mut(&scan) - .unwrap() - .multiplier = 2; - assert_each_node_charged_per_consuming_query(&run); -} - -/// The plan selection picks compiles in the physical planner (added by the -/// implementer, not part of the spec). -#[test] -fn stage3_plan_selection_selected_plan_compiles_in_the_physical_planner() { - let run = pipeline(); - compile_in_runtime(run.physical(&run.selection.selected)).unwrap(); -} - -/// Sharing the input never costs more than reading it separately. -#[test] -fn stage3_plan_selection_shared_input_is_not_costlier() { - let run = pipeline(); - let by_combo: BTreeMap<_, _> = run - .physical - .iter() - .filter(|p| whole_windows(p.dag())) - .filter_map(|p| Some((classify(p.dag(), &p.query_roots), run.cost(&p.id)?))) - .collect(); - let mut compared = 0; - for option in PASS1_Q2_OPTIONS { - if let (Some(shared), Some(separate)) = ( - by_combo.get(&(option, true)), - by_combo.get(&(option, false)), - ) { - assert!(shared <= separate, "{option:?}"); - compared += 1; - } - } - assert!(compared > 0); -} diff --git a/crates/integration-tests/tests/planner_layering_example2.rs b/crates/integration-tests/tests/planner_layering_example2.rs deleted file mode 100644 index 9c09da9fd..000000000 --- a/crates/integration-tests/tests/planner_layering_example2.rs +++ /dev/null @@ -1,1164 +0,0 @@ -//! Rule-fixture and production acceptance for Example 2: shared summaries across statistics. -//! -//! Numerical, accuracy, and operator assertions use explicitly supplied -//! configurations of the existing rules and physical evaluator. They do not -//! assume a bounded MILP run archives every supported workload plan. Separate -//! production checks exercise memo search and validated output selection. -//! Unsupported algorithms retain their existing ignored tests. - -use crate::physical_common; -use crate::planner_layering_common; - -use std::collections::{BTreeMap, BTreeSet}; - -use asap_frontend_sql::{lower_sql_dialect, SqlCatalog}; -use asap_types::ir::properties::ProbabilityExpr; -use asap_types::ir::properties::{AccuracyScope, AccuracyTarget}; -use asap_types::ir::schema::DataType; -use asap_types::ir::schema::{Field, Schema, SketchAlgorithm, SketchParams, SketchStatistic}; -use asap_types::ir::QueryRoot; -use asap_types::workload::{ - DataArrival, DataWorkload, DurationMs, PlanningWorkload, Predictability, Query, QueryLanguage, - QueryRequirements, QueryWorkload, RepeatedDemand, RepeatingEntry, RepetitionInterval, - SourceFacts, SqlDialect, -}; -use planner_layering_common::*; - -// ── Example 2 workload ─────────────────────────────────────────────────── - -const SQL_Q1: &str = - "SELECT COUNT(DISTINCT src_ip) FROM flows WHERE ts >= now() - INTERVAL '1 minute'"; -const SQL_Q2: &str = "SELECT -SUM(p * LN(p)) FROM (\ - SELECT COUNT(*) * 1.0 / SUM(COUNT(*)) OVER () AS p FROM flows \ - WHERE ts >= now() - INTERVAL '1 minute' GROUP BY src_ip)"; -/// Q3 casts the counts to DOUBLE: the design's Int64 `SUM(c * c)` can -/// overflow, so the SQL frontend recognizes only the Float64 product. -const SQL_Q3: &str = "SELECT SQRT(SUM(CAST(c AS DOUBLE) * CAST(c AS DOUBLE))) FROM (\ - SELECT src_ip, COUNT(*) AS c FROM flows \ - WHERE ts >= now() - INTERVAL '1 minute' GROUP BY src_ip)"; - -/// The metric the PromQL stand-in reads. -const STANDIN_METRIC: &str = "flows_src_ip"; -/// PromQL stand-in for Q1–Q3 (see the module docs). -const PROMQL_Q1: &str = "distinct_over_time(flows_src_ip[1m])"; -const PROMQL_Q2: &str = "entropy_over_time(flows_src_ip[1m])"; -const PROMQL_Q3: &str = "l2_over_time(flows_src_ip[1m])"; - -/// (ε, δ) of Q1, Q2, Q3. Q3 is the strictest. -const ACCURACY: [(f64, f64); 3] = [(0.02, 0.01), (0.05, 0.01), (0.01, 0.01)]; - -fn panel(query: &str, (epsilon, delta): (f64, f64), scope: AccuracyScope) -> RepeatingEntry { - RepeatingEntry { - query: Query(query.into()), - demand: RepeatedDemand::FixedInterval(RepetitionInterval(10_000)), - requirements: QueryRequirements { - accuracy: AccuracyTarget { - accuracy_scope: scope, - ..AccuracyTarget::max_error(epsilon).failure_probability(delta) - }, - max_latency_ms: None, - }, - predictability: Predictability::Predictable { known_at: None }, - range_evaluation: None, - } -} - -/// The shared data workload with 10,000,000 distinct source IPs, in the -/// SQL `flows` table and the PromQL stand-in's `flows_src_ip` metric -/// (sampled every 15 s; SQL needs no sample interval). -fn data_workload() -> DataWorkload { - let ten_million = SourceFacts { - series_count: Some(10_000_000), - ..SourceFacts::default() - }; - let sampled = SourceFacts { - sample_interval: Some(DurationMs(15_000)), - ..ten_million.clone() - }; - DataWorkload { - arrival: DataArrival::Ingesting { - rows_per_second: 1_000_000.0 / 15.0, - }, - sources: [ - ("flows".to_string(), ten_million), - (STANDIN_METRIC.to_string(), sampled), - ] - .into(), - } -} - -/// The SQL queries' targets cover their whole result, as the doc's do. The -/// PromQL stand-in states them per value: it reads one series, but the -/// series count of `flows_src_ip` (the 10M source IPs) is what plan -/// selection takes as the number of values a PromQL vector returns, so a -/// whole-result target would be checked over 10M values (#669b). -fn workload(language: QueryLanguage, queries: [&str; 3]) -> PlanningWorkload { - let sql = matches!(language, QueryLanguage::SQL(_)); - let scope = if sql { - AccuracyScope::WholeResult - } else { - AccuracyScope::PerValue - }; - PlanningWorkload { - query_workload: QueryWorkload { - language, - query_batch: None, - repeating_queries: Some( - queries - .iter() - .zip(ACCURACY) - .map(|(q, accuracy)| panel(q, accuracy, scope)) - .collect(), - ), - }, - data_workload: Some(data_workload()), - } -} - -/// Example 2 as the doc writes it. -fn sql_workload() -> PlanningWorkload { - workload( - QueryLanguage::SQL(SqlDialect::DataFusionSQL), - [SQL_Q1, SQL_Q2, SQL_Q3], - ) -} - -fn standin_workload() -> PlanningWorkload { - workload(QueryLanguage::PromQL, [PROMQL_Q1, PROMQL_Q2, PROMQL_Q3]) -} - -/// `flows(ts, src_ip)`: the doc names only these two columns. -fn catalog() -> SqlCatalog { - SqlCatalog::new().with_table( - "flows", - Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("src_ip", DataType::Utf8, false), - ], - 0, - vec![], - ), - ) -} - -async fn lower_sql(workload: &PlanningWorkload) -> Vec { - let mut roots = Vec::new(); - for entry in workload.query_workload.entries() { - let root = lower_sql_dialect( - &entry.query.0, - &catalog(), - SqlDialect::DataFusionSQL, - entry.requirements.accuracy.clone(), - ) - .await - .unwrap_or_else(|e| panic!("{}: {e}", entry.query.0)); - roots.push(QueryRoot::Operator(root)); - } - roots -} - -// Targeted rule fixtures: exact, each local summary in isolation, and every -// query on its own UnivMon, needed by compatibility and cost assertions. -// Shared UnivMons come from the memo (`shared_pipeline`). -// Production optimization is tested separately; this is not a Cartesian search. -fn rule_fixture(workload: &PlanningWorkload, roots: Vec) -> Run { - let computations = rule_fixture_computations(workload, &roots); - run_explicit_candidates( - workload, - roots, - computations, - asap_planner::PlanningModels::builtin(), - ) -} - -/// [`rule_fixture`]'s plans, before Stage 2 and plan selection. -fn rule_fixture_computations( - workload: &PlanningWorkload, - roots: &[QueryRoot], -) -> Vec> { - use asap_logical_asap_optimization::test_support::{ - compose_logical_candidate, generate_per_computation_candidates, - }; - use asap_logical_asap_optimization::test_support::{ - tumbling_window_candidates, window_composition_pass2_candidates, - }; - use asap_types::stages::HowToComputeAggregateMeasure; - let local = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(roots), - ) - .unwrap(); - let exact = vec![0; local.targets.len()]; - let mut choices = BTreeSet::from([exact.clone()]); - let mut univmon = exact.clone(); - let mut shared_targets = Vec::new(); - for (target, options) in local.targets.iter().enumerate() { - for option in 1..options.choice_count() { - let mut choice = exact.clone(); - choice[target] = option; - choices.insert(choice); - } - if let Some(option) = (0..options.choice_count()).find(|&choice| { - matches!( - options.computation_for(choice), - Some(( - HowToComputeAggregateMeasure::Summary(SketchAlgorithm::UnivMon), - _ - )) - ) - }) { - univmon[target] = option; - shared_targets.push(target); - } - } - choices.insert(univmon.clone()); - for &third in &shared_targets { - for option in 0..local.targets[third].choice_count() { - let mut choice = univmon.clone(); - choice[third] = option; - choices.insert(choice); - } - } - let grid: Vec<_> = workload - .query_workload - .entries() - .map(|entry| entry.evaluation_grid_ms()) - .collect(); - let mut computations = Vec::new(); - for choice in choices { - if local.absorbed_choice_varies(&choice) { - continue; - } - let whole: Vec<_> = compose_logical_candidate(&local, &choice).unwrap(); - computations.push(whole.clone()); - let shared = share_identical(&whole); - if distinct_nodes(&shared) < distinct_nodes(&whole) { - computations.push(shared); - } - let tumbling_windows = tumbling_window_candidates(&whole, &grid); - // Isolate one tumbling-window configuration at a time; no Cartesian products of - // independent window implementations are needed for the error tests. - for tumbling_window in tumbling_windows { - computations.extend( - window_composition_pass2_candidates(&whole, &[tumbling_window]) - .unwrap() - .into_iter() - .map(|candidate| candidate.roots.unwrap()), - ); - } - } - computations -} - -#[test] -fn production_memo_selects_a_validated_example2_plan() { - let workload = standin_workload(); - let run = run_stages(&workload, lower_promql(&workload)); - assert_selects_cheapest_valid(&run); - assert_eq!(run.selection.accepted[&run.selection.selected].len(), 3); -} - -fn pipeline() -> Run { - let workload = standin_workload(); - rule_fixture(&workload, lower_promql(&workload)) -} - -/// The stand-in with the deployment's bound on distinct items per -/// evaluation of its one source, when given. -fn standin_with_distinct_bound(bound: Option) -> PlanningWorkload { - let mut workload = standin_workload(); - if let Some(bound) = bound { - workload - .data_workload - .as_mut() - .expect("data workload") - .sources - .get_mut(STANDIN_METRIC) - .expect("the stand-in's source") - .max_distinct_per_evaluation = Some(bound); - } - workload -} - -fn pipeline_with_distinct_bound(bound: Option) -> Run { - let workload = standin_with_distinct_bound(bound); - rule_fixture(&workload, lower_promql(&workload)) -} - -/// Plans composed of the memo's candidates, priced by Stage 2 and plan -/// selection: every query on its group's first candidate; each query alone -/// on its own UnivMon; and, per UnivMon state group the memo shares between -/// queries, those queries reading it. -fn memo_pipeline() -> Run { - let workload = standin_workload(); - let roots = lower_promql(&workload); - let computations = memo_computations(&workload, &roots); - run_explicit_candidates( - &workload, - roots, - computations, - asap_planner::PlanningModels::builtin(), - ) -} - -/// [`rule_fixture`]'s plans and [`memo_pipeline`]'s together, so the -/// memo's shared UnivMons compete with every query's own options. -fn shared_pipeline(workload: &PlanningWorkload, roots: Vec) -> Run { - let mut computations = rule_fixture_computations(workload, &roots); - computations.extend(memo_computations(workload, &roots)); - run_explicit_candidates( - workload, - roots, - computations, - asap_planner::PlanningModels::builtin(), - ) -} - -/// [`shared_pipeline`] over the stand-in. -fn standin_shared_pipeline() -> Run { - let workload = standin_workload(); - shared_pipeline(&workload, lower_promql(&workload)) -} - -/// [`memo_pipeline`]'s plans of `roots`, before Stage 2 and plan selection. -fn memo_computations(workload: &PlanningWorkload, roots: &[QueryRoot]) -> Vec> { - use asap_integration_tests::planning::{compose_memo_plan, expanded_memo, own_choices}; - use asap_types::ir::schema::FieldDataType; - use asap_types::ir::{ASAPOp, OperatorNode}; - use asap_types::stages::{CandidateForMemoGroup, OriginOfCandidateForMemoGroup}; - let entries: Vec<_> = workload.query_workload.entries().collect(); - let memo = expanded_memo(roots.to_vec(), &entries); - let queries = roots.len(); - let is_univmon = |node: &std::rc::Rc| { - matches!(node.asap(), Some(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), .. - }) if *kind.algorithm() == SketchAlgorithm::UnivMon) - }; - // Query `q`'s topmost own group's first candidate `keep` accepts. - let own = |q: usize, keep: &dyn Fn(&CandidateForMemoGroup) -> bool| { - own_choices(&memo, q, keep).into_iter().next() - }; - let mut choices = vec![BTreeMap::new()]; - for q in 0..queries { - if let Some(choice) = own(q, &|c| { - c.origin == OriginOfCandidateForMemoGroup::Pass1 - && OperatorNode::reachable(&c.root_node).iter().any(is_univmon) - }) { - choices.push(BTreeMap::from([choice])); - } - } - for (state, group) in memo.groups.iter().enumerate() { - if !is_univmon(&group.logical_node) || group.reached_by_queries().len() < 2 { - continue; - } - let reading: Option> = group - .reached_by_queries() - .iter() - .map(|&q| { - own(q, &|c| { - c.input_groups.iter().any(|child| child.0 as usize == state) - }) - }) - .collect(); - choices.extend(reading); - } - choices - .iter() - .map(|choice| { - compose_memo_plan(&memo, queries, choice) - .into_iter() - .map(QueryRoot::Operator) - .collect() - }) - .collect() -} - -// ── Helpers ────────────────────────────────────────────────────────────── - -/// UnivMon build nodes of `c` with the queries that read each. -fn univmons(c: &Logical) -> Vec<(BTreeSet, SketchParams, BTreeSet)> { - sketch_builds(&c.dag) - .into_iter() - .filter(|(_, algorithm, _)| *algorithm == SketchAlgorithm::UnivMon) - .map(|(id, _, params)| { - let statistics = estimates_of(&c.dag, id) - .into_iter() - .map(|(_, s)| format!("{s:?}")) - .collect(); - (readers(&c.dag, &c.query_roots, id), params, statistics) - }) - .collect() -} - -/// The UnivMon read by every query in `queries` and by no other. -fn shared_univmon(c: &Logical, queries: &[usize]) -> Option { - let want: BTreeSet<_> = queries.iter().copied().collect(); - univmons(c) - .into_iter() - .find(|(readers, ..)| *readers == want) - .map(|(_, params, _)| params) -} - -/// Whether some summary build is read by more than one query. -fn shares_a_summary(c: &Logical) -> bool { - sketch_builds(&c.dag) - .iter() - .any(|(id, ..)| readers(&c.dag, &c.query_roots, *id).len() > 1) -} - -fn options(run: &Run, query: usize) -> BTreeSet { - run.logical - .iter() - .filter(|c| !shares_a_summary(c)) - .map(|c| query_option(&c.dag, &c.query_roots, query)) - .collect() -} - -/// UnivMon state size: heap entries plus counters over every layer. -fn footprint(params: &SketchParams) -> u64 { - let SketchParams::UnivMon { - heap_size, - sketch_rows, - sketch_cols, - layers, - .. - } = params - else { - panic!("UnivMon params: {params:?}"); - }; - u64::from(*heap_size) + u64::from(*sketch_rows) * u64::from(*sketch_cols) * u64::from(*layers) -} - -/// Q`query`'s own UnivMon parameters, from an independent candidate. -fn own_univmon(run: &Run, query: usize) -> SketchParams { - run.logical - .iter() - .filter(|c| !shares_a_summary(c)) - .find_map(|c| shared_univmon(c, &[query])) - .expect("Pass 1 offers UnivMon") -} - -// ── Workload and query-language frontends ───────────────────────────────────────────────── - -/// The encoded SQL workload is valid and normalizes to Q1, Q2, Q3. -#[test] -fn workload_encodes_example2() { - let workload = sql_workload(); - workload.validate().expect("valid workload"); - let queries: Vec<_> = workload - .query_workload - .entries() - .map(|e| e.query.0) - .collect(); - assert_eq!(queries, [SQL_Q1, SQL_Q2, SQL_Q3]); - standin_workload().validate().expect("valid stand-in"); -} - -/// The SQL frontend lowers Q1 to a distinct count over `flows`. -#[tokio::test] -async fn stage0_query_language_frontends_sql_q1_lowers_to_a_distinct_count() { - let roots = lower_sql(&sql_workload()).await; - let ops = frontend_operations(&roots[0]); - assert!(ops.contains(&"scan".to_string()), "{ops:?}"); - assert!( - ops.contains(&"aggregate:cardinality".to_string()), - "{ops:?}" - ); -} - -/// The SQL frontend recognizes Q2 as `Entropy(src_ip)` and Q3 as `L2(src_ip)`. -#[tokio::test] -async fn stage0_query_language_frontends_sql_q2_q3_lower_to_entropy_and_l2() { - let roots = lower_sql(&sql_workload()).await; - assert!(frontend_operations(&roots[1]).contains(&"aggregate:frequency_entropy".to_string())); - assert!(frontend_operations(&roots[2]).contains(&"aggregate:frequency_l2".to_string())); -} - -/// Once the SQL forms are recognized, the SQL workload has the stand-in's Pass 1 options. -#[tokio::test] -async fn stage1_logical_asap_optimization_sql_has_the_standin_options() { - let workload = sql_workload(); - let sql = rule_fixture(&workload, lower_sql(&workload).await); - let standin = pipeline(); - for q in 0..3 { - assert_eq!(options(&sql, q), options(&standin, q), "Q{}", q + 1); - } -} - -/// The doc's SQL workload itself gets the memo's shared candidate: one -/// UnivMon read by Q1, Q2 and Q3. -#[tokio::test] -async fn stage1_logical_asap_optimization_sql_adds_one_univmon_for_all_three() { - let workload = sql_workload(); - let run = shared_pipeline(&workload, lower_sql(&workload).await); - assert!( - run.logical.iter().any(|c| { - let ums = univmons(c); - ums.len() == 1 && ums[0].0 == BTreeSet::from([0, 1, 2]) - }), - "no SQL candidate reads one UnivMon for Q1, Q2 and Q3" - ); -} - -/// The stand-in lowers to the three statistics, each over a 1-min range of one input. -#[test] -fn stage0_query_language_frontends_standin_lowers_to_three_statistics_of_one_input() { - let roots = lower_promql(&standin_workload()); - let ops: Vec<_> = roots.iter().map(frontend_operations).collect(); - for (ops, statistic) in ops.iter().zip([ - "aggregate:cardinality", - "aggregate:frequency_entropy", - "aggregate:frequency_l2", - ]) { - let mut want = vec!["scan".to_string(), statistic.into(), "time_range".into()]; - want.sort(); - assert_eq!(ops, &want); - } -} - -/// The frontends give one summary-free workload DAG whose queries share no node. -#[test] -fn stage0_query_language_frontends_one_summary_free_dag_without_sharing() { - let run = pipeline(); - let stage0_query_language_frontends = &run.stage0_query_language_frontends; - assert_eq!(stage0_query_language_frontends.query_roots.len(), 3); - assert!(stage0_query_language_frontends - .dag - .nodes - .iter() - .all(|n| !is_summary(&n.operator))); - assert!(cross_query_nodes( - &stage0_query_language_frontends.dag, - &stage0_query_language_frontends.query_roots - ) - .is_empty()); -} - -// ── Logical ASAP-aware optimization ────────────────────────────────────── - -/// Pass 1 offers an exact and a UnivMon option for each of the three statistics. -#[test] -fn stage1_logical_asap_optimization_pass1_offers_exact_and_univmon_for_each_statistic() { - let run = pipeline(); - for q in 0..3 { - let found = options(&run, q); - assert!(found.contains("exact"), "Q{}: {found:?}", q + 1); - assert!(found.contains("UnivMon"), "Q{}: {found:?}", q + 1); - } -} - -/// Pass 1 offers a specialized distinct-count summary for Q1. -#[test] -fn stage1_logical_asap_optimization_pass1_offers_a_distinct_count_summary() { - let found = options(&pipeline(), 0); - assert!( - found.iter().any(|o| ["Hll", "Kmv"].contains(&o.as_str())), - "{found:?}" - ); -} - -/// Pass 1 offers a specialized entropy summary for Q2 and a norm summary for Q3. -#[test] -#[ignore = "needs Pass 1 specialized entropy and L2 summary families (#444)"] -fn stage1_logical_asap_optimization_pass1_offers_specialized_entropy_and_l2_summaries() { - let run = pipeline(); - for q in [1, 2] { - let found = options(&run, q); - assert!( - found.iter().any(|o| o != "exact" && o != "UnivMon"), - "Q{}: {found:?}", - q + 1 - ); - } -} - -/// The summary-capability rule adds one UnivMon build feeding a distinct-count, an entropy and an L2 estimate. -#[test] -fn stage1_logical_asap_optimization_summary_capability_adds_one_univmon_for_all_three() { - let run = memo_pipeline(); - let shared = run.logical.iter().find(|c| { - let ums = univmons(c); - ums.len() == 1 && ums[0].0 == BTreeSet::from([0, 1, 2]) - }); - let c = shared.expect("a candidate with one UnivMon read by Q1, Q2 and Q3"); - let statistics = &univmons(c)[0].2; - for s in [ - SketchStatistic::Cardinality, - SketchStatistic::FrequencyEntropy, - SketchStatistic::FrequencyL2, - ] { - assert!(statistics.contains(&format!("{s:?}")), "{statistics:?}"); - } -} - -/// A UnivMon shared by several statistics is sized for every consumer: -/// each consumer is sized alone, and the shared build takes each -/// parameter's largest value among them. -#[test] -fn stage1_logical_asap_optimization_shared_univmon_takes_each_parameters_largest_own_size() { - let run = memo_pipeline(); - let own = |q| match own_univmon(&run, q) { - SketchParams::UnivMon { - heap_size, - sketch_rows, - sketch_cols, - layers, - .. - } => [heap_size, sketch_rows, sketch_cols, u32::from(layers)], - other => panic!("{other:?}"), - }; - let mut seen = 0; - for queries in [vec![0, 1, 2], vec![0, 1], vec![0, 2], vec![1, 2]] { - let mut want = [0u32; 4]; - for &q in &queries { - for (w, p) in want.iter_mut().zip(own(q)) { - *w = (*w).max(p); - } - } - let want = SketchParams::UnivMon { - heap_size: want[0], - sketch_rows: want[1], - sketch_cols: want[2], - layers: want[3] as u8, - // Ignored by equality. - distinct_failure_probability: None, - }; - for c in &run.logical { - if let Some(params) = shared_univmon(c, &queries) { - seen += 1; - assert_eq!(params, want, "{} {queries:?}", c.id); - } - } - } - assert!(seen > 0, "no shared UnivMon"); -} - -/// Pass 1 sizes UnivMon for its query's target: ε = 0.01 (Q3) needs a larger state than ε = 0.05 (Q2). -#[test] -fn stage1_logical_asap_optimization_univmon_is_sized_per_accuracy_target() { - let run = pipeline(); - assert!(footprint(&own_univmon(&run, 2)) > footprint(&own_univmon(&run, 1))); -} - -/// Only a UnivMon, never a specialized summary, is shared across the statistics. -#[test] -fn stage1_logical_asap_optimization_only_univmon_is_shared_across_statistics() { - let run = standin_shared_pipeline(); - for c in &run.logical { - for (id, algorithm, _) in sketch_builds(&c.dag) { - if readers(&c.dag, &c.query_roots, id).len() > 1 { - assert_eq!(algorithm, SketchAlgorithm::UnivMon, "{}", c.id); - } - } - } -} - -/// Logical candidates are valid DAGs with unique ids. -#[test] -fn stage1_logical_asap_optimization_candidates_are_valid_and_uniquely_named() { - assert_valid_and_uniquely_named(&pipeline()); -} - -// ── Physical ASAP-aware optimization ───────────────────────────────────── - -/// No candidate is discarded before plan selection: each logical candidate keeps one physical candidate with nothing materialized, plus one per other materialization assignment. -#[test] -fn stage2_physical_asap_optimization_keeps_every_logical_candidate() { - assert_one_unmaterialized_physical_candidate_per_logical(&pipeline()); -} - -/// Physical ASAP-aware optimization keeps the shared UnivMon as one build node read by all three queries. -#[test] -fn stage2_physical_asap_optimization_keeps_the_shared_univmon_as_one_build() { - let run = memo_pipeline(); - let mut seen = 0; - for c in run - .logical - .iter() - .filter(|c| shared_univmon(c, &[0, 1, 2]).is_some()) - { - for p in run.physical_of(c) { - seen += 1; - let builds: Vec<_> = sketch_builds(p.dag()) - .into_iter() - .filter(|(_, a, _)| *a == SketchAlgorithm::UnivMon) - .collect(); - assert_eq!(builds.len(), 1, "{}", p.id); - assert_eq!(readers(p.dag(), &p.query_roots, builds[0].0).len(), 3); - } - } - assert!(seen > 0); -} - -// ── Plan selection ─────────────────────────────────────────────────────── - -/// Plan selection selects one candidate and gives every other one a reason. -#[test] -fn stage3_plan_selection_selects_one_and_explains_the_rest() { - assert_selects_one_and_explains_the_rest(&pipeline()); -} - -/// The selected plan is the cheapest valid candidate for the whole workload. -#[test] -fn stage3_plan_selection_selects_cheapest_valid() { - assert_selects_cheapest_valid(&pipeline()); -} - -/// Every node of a valid candidate is charged once per query that reads it: nothing is materialized, so a shared node is recomputed for each. -#[test] -fn stage3_plan_selection_charges_each_node_per_consuming_query() { - assert_each_node_charged_per_consuming_query(&pipeline()); -} - -/// `asap_sketchlib` has no Theta sketch, so no candidate builds one. -#[test] -fn stage3_plan_selection_has_no_theta_candidate() { - let run = pipeline(); - for (id, cost) in &run.selection.costs { - assert!( - !cost - .per_node - .values() - .any(|n| n.detail.starts_with("build Theta")), - "{id} builds Theta" - ); - } -} - -/// A UnivMon estimate has a guarantee, so plan selection judges it against its target instead of rejecting it for having none. -#[test] -#[ignore = "needs a UnivMon entropy accuracy model (#788); L2 and the distinct count have one"] -fn stage3_plan_selection_judges_univmon_with_an_accuracy_model() { - let run = pipeline(); - let invalid = run.invalid(); - for c in &run.logical { - for p in run.physical_of(c) { - let Some(reason) = invalid.get(p.id.as_str()) else { - continue; - }; - for q in 0..3 { - if query_option(&c.dag, &c.query_roots, q) == "UnivMon" { - let unguaranteed = format!("q{}: the output has no guarantee", q + 1); - assert!(!reason.starts_with(&unguaranteed), "{}: {reason}", p.id); - } - } - } - } -} - -/// The physical candidates that answer Q1 with `algorithm` and Q2 and Q3 -/// exactly: their validity depends only on Q1's estimate. -fn q1_only_candidates<'a>(run: &'a Run, algorithm: &str) -> Vec<&'a str> { - let ids: Vec<_> = run - .logical - .iter() - .filter(|c| { - !shares_a_summary(c) - && (0..3) - .map(|q| query_option(&c.dag, &c.query_roots, q)) - .eq([algorithm, "exact", "exact"]) - }) - .flat_map(|c| run.physical_of(c).map(|p| p.id.as_str())) - .collect(); - assert!( - !ids.is_empty(), - "a candidate answers Q1 with {algorithm} alone" - ); - ids -} - -fn hll_only_candidates(run: &Run) -> Vec<&str> { - q1_only_candidates(run, "Hll") -} - -/// With the deployment's bound on distinct items per evaluation, plan -/// selection binds classic HLL's failure probability and accepts Q1's HLL -/// estimate; without a bound it stays unknown and the target unmet. Pass 1 -/// sizes HLL from ε = 0.02 and δ = 0.01 (#694): `m ≥ 1/(2ε²δ)` gives -/// precision 17, which certifies δ = 0.01 at ε = 0.02 for bounds up to -/// 1,842 distinct items. -#[test] -fn stage3_plan_selection_accepts_hll_given_a_distinct_bound() { - for bound in [9, 1_842] { - let bounded = pipeline_with_distinct_bound(Some(bound)); - for id in hll_only_candidates(&bounded) { - assert!( - bounded.cost(id).is_some(), - "bound {bound}: {id}: {:?}", - bounded.invalid().get(id) - ); - } - } - let unbounded = pipeline_with_distinct_bound(None); - let invalid = unbounded.invalid(); - for id in hll_only_candidates(&unbounded) { - let reason = invalid.get(id).unwrap_or_else(|| panic!("{id} is invalid")); - assert!(reason.starts_with("q1:"), "{id}: {reason}"); - assert!( - reason.contains("failure probability None"), - "{id}: {reason}" - ); - } -} - -/// A bound too large for the precision sized for ε alone (17 certifies at -/// most 1,842 distinct items) is met by a larger precision: Pass 1's sizing -/// for the failure probability raises HLL's precision until its failure -/// probability over the bound at ε = 0.02 meets the target, up to precision -/// 18 (#669b), which certifies up to 3,791; Stage 3 only checks it. Past -/// that, up to Example 2's 10M source IPs, classic HLL's linear-counting -/// contract certifies nothing. -#[test] -fn pass1_sizing_enlarges_hll_for_a_larger_distinct_bound() { - let run = pipeline_with_distinct_bound(Some(3_000)); - for id in hll_only_candidates(&run) { - assert!(run.cost(id).is_some(), "{id}: {:?}", run.invalid().get(id)); - } - for bound in [10_000, 10_000_000] { - let run = pipeline_with_distinct_bound(Some(bound)); - let invalid = run.invalid(); - for id in hll_only_candidates(&run) { - let reason = invalid - .get(id) - .unwrap_or_else(|| panic!("bound {bound}: {id} is invalid")); - assert!(reason.starts_with("q1:"), "{id}: {reason}"); - assert!( - !reason.contains("failure probability None"), - "{id}: {reason}" - ); - } - } -} - -/// KMV certifies Q1 over the 10M source IPs without a distinct bound -/// (#694): its bound holds for any number of distinct items, and Pass 1 -/// sizes `k` for the requested δ (#700), so a stricter δ = 10⁻⁴ is met -/// too, by a larger `k`. -#[test] -fn stage3_plan_selection_accepts_kmv_for_any_failure_probability_without_a_bound() { - let mut ks = Vec::new(); - for delta in [0.01, 1e-4] { - let mut workload = standin_workload(); - let q1 = &mut workload - .query_workload - .repeating_queries - .as_mut() - .expect("repeating queries")[0]; - q1.requirements.accuracy = AccuracyTarget { - accuracy_scope: AccuracyScope::PerValue, - ..AccuracyTarget::max_error(0.02).failure_probability(delta) - }; - let run = rule_fixture(&workload, lower_promql(&workload)); - for id in q1_only_candidates(&run, "Kmv") { - assert!( - run.cost(id).is_some(), - "δ = {delta}: {id}: {:?}", - run.invalid().get(id) - ); - let k: BTreeSet<_> = sketch_builds(run.physical(id).dag()) - .into_iter() - .filter_map(|(_, _, params)| match params { - SketchParams::Kmv { k, .. } => Some(k), - _ => None, - }) - .collect(); - ks.push((delta, k)); - } - } - assert!( - ks.iter().any(|(d, k)| *d == 0.01 && k.contains(&26_656)), - "{ks:?}" - ); - assert!( - ks.iter() - .any(|(d, k)| *d == 1e-4 && k.iter().all(|k| *k > 26_656)), - "{ks:?}" - ); -} - -/// The physical candidates among `ids` that build their sketch as a -/// tumbling-window `SummaryTumblingWindowChain`. -fn tumbling<'a>(run: &Run, ids: Vec<&'a str>) -> Vec<&'a str> { - ids.into_iter() - .filter(|id| { - let dag = run.physical(id).dag(); - sketch_builds(dag).iter().any(|(build, _, _)| { - matches!(window_form(dag, *build), WindowForm::Tumbling { .. }) - }) - }) - .collect() -} - -/// The enlarged precision reaches a sketch carried by a tumbling-window -/// `SummaryTumblingWindowChain` too (#713): Q1's HLL over a bound of 3,000 is valid only at -/// precision 18, above the 17 sized for ε alone, so a valid tumbling-window -/// HLL candidate carries precision 18 in its chain. -#[test] -fn pass1_sizing_enlarges_tumbling_window_hll() { - let run = pipeline_with_distinct_bound(Some(3_000)); - let hll = tumbling(&run, hll_only_candidates(&run)); - assert!( - !hll.is_empty(), - "a tumbling-window HLL candidate answers Q1" - ); - for id in hll { - assert!(run.cost(id).is_some(), "{id}: {:?}", run.invalid().get(id)); - } -} - -/// Stage 1 does not read the distinct bound: its candidates are the same -/// with and without one, and Q1's HLL estimate leaves its failure -/// probability symbolic in the source's bound. -#[test] -fn stage1_logical_asap_optimization_leaves_the_hll_failure_probability_symbolic() { - let bounded = pipeline_with_distinct_bound(Some(9)); - let unbounded = pipeline_with_distinct_bound(None); - let stage1 = |run: &Run| { - run.logical - .iter() - .map(|c| format!("{} {:?}", c.id, c.roots)) - .collect::>() - }; - assert_eq!(stage1(&bounded), stage1(&unbounded)); - let mut seen = 0; - for c in unbounded - .logical - .iter() - .filter(|c| !shares_a_summary(c) && query_option(&c.dag, &c.query_roots, 0) == "Hll") - { - let QueryRoot::Operator(q1) = &c.roots[0] else { - panic!("operator root") - }; - let guarantee = q1.guarantee.as_ref().expect("Q1 has a guarantee"); - assert_eq!( - guarantee.spec.failure_probability, - ProbabilityExpr::Unknown { - statistic: - "classic_hll_failure_probability[p=17,eps=0.02](max_distinct[flows_src_ip])" - .into() - }, - "{}", - c.id - ); - seen += 1; - } - assert!(seen > 0, "a candidate answers Q1 with HLL"); -} - -/// The physical candidates that answer Q`query` with UnivMon and the other -/// two exactly: their validity depends only on that UnivMon estimate. -fn univmon_only_candidates(run: &Run, query: usize) -> Vec<&str> { - let ids: Vec<_> = run - .logical - .iter() - .filter(|c| { - !shares_a_summary(c) - && (0..3).all(|q| { - query_option(&c.dag, &c.query_roots, q) - == if q == query { "UnivMon" } else { "exact" } - }) - }) - .flat_map(|c| run.physical_of(c).map(|p| p.id.as_str())) - .collect(); - assert!( - !ids.is_empty(), - "a candidate answers Q{} with UnivMon alone", - query + 1 - ); - ids -} - -/// Q1's UnivMon is sized for (0.02, 0.01): a 337,500-key heap on 19 -/// layers. Its distinct count is certified once the deployment bounds the -/// source's distinct items, both at 9 (the top layer is always complete) -/// and at 10M (7 layers suffice, where classic HLL certifies nothing); -/// without a bound its failure probability stays unknown. -#[test] -fn stage3_plan_selection_accepts_univmon_distinct_given_a_distinct_bound() { - for bound in [9, 10_000_000] { - let run = pipeline_with_distinct_bound(Some(bound)); - for id in univmon_only_candidates(&run, 0) { - assert!( - run.cost(id).is_some(), - "{bound} {id}: {:?}", - run.invalid().get(id) - ); - } - } - let unbounded = pipeline_with_distinct_bound(None); - let invalid = unbounded.invalid(); - for id in univmon_only_candidates(&unbounded, 0) { - let reason = invalid.get(id).unwrap_or_else(|| panic!("{id} is invalid")); - assert!(reason.starts_with("q1:"), "{id}: {reason}"); - assert!( - reason.contains("failure probability None"), - "{id}: {reason}" - ); - } -} - -/// A distinct bound past what 19 layers cover for Q1's heap -/// (337,500 · 2^17) certifies nothing. -#[test] -fn stage3_plan_selection_rejects_univmon_distinct_past_its_layers() { - let run = pipeline_with_distinct_bound(Some(337_500 << 17 | 1)); - let invalid = run.invalid(); - for id in univmon_only_candidates(&run, 0) { - let reason = invalid.get(id).unwrap_or_else(|| panic!("{id} is invalid")); - assert!(reason.starts_with("q1:"), "{id}: {reason}"); - } -} - -/// Q3's L2 at (0.01, 0.01) is certified from layer 0 alone: a 5 × 56,817 -/// Count Sketch on one layer, with no data-workload statistic. -#[test] -fn stage3_plan_selection_accepts_univmon_l2() { - let run = pipeline(); - assert_eq!( - own_univmon(&run, 2), - SketchParams::UnivMon { - heap_size: 1, - sketch_rows: 5, - sketch_cols: 56_817, - layers: 1, - distinct_failure_probability: None, - } - ); - for id in univmon_only_candidates(&run, 2) { - assert!(run.cost(id).is_some(), "{id}: {:?}", run.invalid().get(id)); - } -} - -/// SQL Q3's L2 through UnivMon keeps its guarantee: the exact population -/// count joined beside it (a cross join, no key) and the `CASE` deciding the -/// empty-input NULL on that exact count keep the estimate's guarantee, so -/// plan selection accepts it. -#[tokio::test] -async fn stage3_plan_selection_accepts_sql_univmon_l2_beside_its_exact_population_count() { - let workload = sql_workload(); - let run = rule_fixture(&workload, lower_sql(&workload).await); - for id in univmon_only_candidates(&run, 2) { - assert!(run.cost(id).is_some(), "{id}: {:?}", run.invalid().get(id)); - } -} - -/// One materialized UnivMon for all three costs no more than three separate UnivMons. -#[test] -#[ignore = "needs a UnivMon entropy accuracy model (#788): without one, the shared UnivMon candidate is left unpriced"] -fn stage3_plan_selection_shared_univmon_costs_no_more_than_three() { - let run = standin_shared_pipeline(); - let cost = |c: &Logical| run.cost(&run.physical_of(c).next().unwrap().id); - let separate: Vec<_> = run - .logical - .iter() - .filter(|c| { - !shares_a_summary(c) - && (0..3).all(|q| query_option(&c.dag, &c.query_roots, q) == "UnivMon") - }) - .map(|c| cost(c).unwrap_or_else(|| panic!("{} is not priced", c.id))) - .collect(); - assert!(!separate.is_empty(), "independent all-UnivMon candidate"); - let mut compared = 0; - for shared in run - .logical - .iter() - .filter(|c| shared_univmon(c, &[0, 1, 2]).is_some()) - { - let shared_cost = cost(shared).unwrap_or_else(|| panic!("{} is not priced", shared.id)); - for separate_cost in &separate { - assert!( - shared_cost <= *separate_cost, - "{shared_cost} vs {separate_cost}" - ); - } - compared += 1; - } - assert!(compared > 0); -} - -/// The selected plan updates at most one UnivMon per flow record: separate UnivMons are dominated by the shared one. -#[test] -fn stage3_plan_selection_selected_plan_has_at_most_one_univmon() { - let run = standin_shared_pipeline(); - let selected = run.physical(&run.selection.selected); - let count = sketch_builds(selected.dag()) - .iter() - .filter(|(_, a, _)| *a == SketchAlgorithm::UnivMon) - .count(); - assert!(count <= 1, "{}: {count} UnivMons", selected.id); -} - -/// Stage 3 over the candidates that answer Q1 with KMV and Q2 and Q3 -/// exactly selects one, and the executor runs its Q1: the distinct count of -/// the window's sample values. Below KMV's `k` distinct values its estimate -/// is exact. Only Q1 is compiled: the exact entropy and L2 of Q2 and Q3 do -/// not compile yet ("per-entity aggregate requires an explicit entity -/// binding"). -#[test] -fn stage3_plan_selection_selected_kmv_plan_executes_end_to_end() { - use asap_devtools::memo_pricing::reference_select as select_plan; - use asap_executor::physical_planner::{compile, promql_rows}; - use asap_executor::runtime::Scope; - use asap_executor::values::Value; - use asap_planner::PlanningModels; - - let workload = standin_workload(); - let roots = lower_promql(&workload); - let run = rule_fixture(&workload, roots.clone()); - let kmv: Vec<_> = run - .logical - .iter() - .filter(|c| { - !shares_a_summary(c) - && (0..3) - .map(|q| query_option(&c.dag, &c.query_roots, q)) - .eq(["Kmv", "exact", "exact"]) - }) - .flat_map(|c| run.physical_of(c).map(|p| p.candidate.clone())) - .collect(); - assert!(!kmv.is_empty(), "a candidate answers Q1 with KMV"); - let entries: Vec<_> = workload.query_workload.entries().collect(); - let selection = select_plan( - &kmv, - &entries, - workload.data_workload.as_ref().unwrap(), - PlanningModels::builtin(), - ) - .expect("Stage 3 selects a KMV plan"); - let p = run.physical(&selection.selected); - assert_eq!(query_option(p.dag(), &p.query_roots, 0), "Kmv"); - - // 1,000 flow records over 50 s with 300 distinct source IPs. - let distinct = 300u32; - let samples: Vec<(i64, f64)> = (0..1_000u32) - .map(|i| (i64::from(i) * 50, f64::from(i * 7 % distinct))) - .collect(); - let labels = BTreeMap::from([("__name__".to_string(), "flows_src_ip".to_string())]); - // The whole plan runs at query time over the samples bound at its `Scan`. - let dag = p.dag(); - let store = physical_common::raw_data_store(dag, |_, schema| { - samples - .iter() - .map(|(ts, v)| promql_rows::series_row(schema, &labels, *ts, *v).unwrap()) - .collect() - }); - let q1 = p.query_roots[0] as u64; - let plan = compile(dag, &[], &[q1]).unwrap_or_else(|e| panic!("{}: {e}", p.id)); - let result = physical_common::execute( - &plan, - store, - Scope::Query { - evaluation_time_ms: 55_000, - }, - ); - let values: Vec = result[0] - .iter() - .flat_map(|batch| batch.to_rows()) - .flatten() - .filter_map(|value| match value { - Value::Float64(v) => Some(v), - _ => None, - }) - .collect(); - assert_eq!(values, [f64::from(distinct)], "{}", p.id); -} diff --git a/crates/integration-tests/tests/planner_layering_example3.rs b/crates/integration-tests/tests/planner_layering_example3.rs deleted file mode 100644 index c0c8ab6ed..000000000 --- a/crates/integration-tests/tests/planner_layering_example3.rs +++ /dev/null @@ -1,635 +0,0 @@ -//! Rule-fixture and production acceptance for Example 3: window implementations. -//! -//! Numerical, accuracy, and operator assertions use explicitly supplied -//! configurations of the existing rules and physical evaluator. They do not -//! assume a bounded MILP run archives every supported workload plan. Separate -//! production checks exercise memo search and validated output selection. -//! Unsupported algorithms retain their existing ignored tests. - -use crate::planner_layering_common; - -use std::collections::BTreeSet; - -use asap_types::ir::flat::NodeId; -use asap_types::ir::schema::{SketchAlgorithm, SketchParams, SketchStatistic}; -use asap_types::workload::DataArrival; - -use planner_layering_common::*; - -// Fixed algorithm/window fixtures keep numerical evaluator checks independent -// of a time-bounded production search's archived incumbents. -fn run_a() -> Run { - use asap_logical_asap_optimization::test_support::{ - compose_logical_candidate, generate_per_computation_candidates, - }; - use asap_types::stages::HowToComputeAggregateMeasure; - let workload = pattern_a( - PatternARecurrence::OnceAdHoc, - DataArrival::Mixed { - rows_per_second: SHARED_ROWS_PER_SECOND, - }, - ); - let roots = lower_promql(&workload); - let local = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&roots), - ) - .unwrap(); - let mut computations = Vec::new(); - for kll in [false, true] { - let choices: Vec<_> = local - .targets - .iter() - .map(|target| { - if kll { - (0..target.choice_count()) - .find(|&choice| { - matches!( - target.computation_for(choice), - Some(( - HowToComputeAggregateMeasure::Summary(SketchAlgorithm::Kll), - _ - )) - ) - }) - .unwrap() - } else { - 0 - } - }) - .collect(); - let whole = unshare(&compose_logical_candidate(&local, &choices).unwrap()); - computations.push(whole.clone()); - computations.push(share_identical(&whole)); - } - run_explicit_candidates( - &workload, - roots, - computations, - asap_planner::PlanningModels::builtin(), - ) -} - -fn run_b() -> Run { - use asap_logical_asap_optimization::test_support::{ - compose_logical_candidate, generate_per_computation_candidates, - }; - use asap_logical_asap_optimization::test_support::{ - tumbling_window_candidates, window_composition_pass2_candidates, - }; - use asap_types::stages::HowToComputeAggregateMeasure; - let workload = pattern_b(); - let roots = lower_promql(&workload); - let local = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&roots), - ) - .unwrap(); - let mut computations = vec![roots.clone()]; - for option in 0..local.targets[0].choice_count() { - if !matches!( - local.targets[0].computation_for(option), - Some((HowToComputeAggregateMeasure::Summary(_), _)) - ) { - continue; - } - let whole: Vec<_> = compose_logical_candidate(&local, &[option]).unwrap(); - let tumbling_windows = tumbling_window_candidates(&whole, &[Some(PATTERN_B_INTERVAL_MS)]); - let chain = window_composition_pass2_candidates(&whole, &tumbling_windows) - .unwrap() - .remove(0) - .roots - .unwrap(); - computations.extend([whole, chain]); - } - run_explicit_candidates( - &workload, - roots, - computations, - asap_planner::PlanningModels::builtin(), - ) -} - -#[test] -fn production_memo_covers_pattern_b_with_validated_accuracy() { - let workload = pattern_b(); - let run = run_stages_under(&workload, lower_promql(&workload), &Default::default()); - assert_selects_cheapest_valid(&run); - assert_eq!(run.selection.accepted[&run.selection.selected].len(), 1); -} - -/// The per-computation options of `query` across all candidates. -fn options(run: &Run, query: usize) -> BTreeSet { - run.logical - .iter() - .map(|c| query_option(&c.dag, &c.query_roots, query)) - .collect() -} - -fn op(name: &str) -> String { - name.to_string() -} - -/// The summary builds of `c`: (build, algorithm, window form, readers). -fn windowed_builds(c: &Logical) -> Vec<(NodeId, SketchAlgorithm, WindowForm, BTreeSet)> { - sketch_builds(&c.dag) - .into_iter() - .map(|(id, algorithm, _)| { - let form = window_form(&c.dag, id); - (id, algorithm, form, readers(&c.dag, &c.query_roots, id)) - }) - .collect() -} - -// ── Workloads ──────────────────────────────────────────────────────────── - -/// Pattern A is one ad hoc batch of five queries run once at T; Pattern B one repeating query. -#[test] -fn workload_encodes_example3() { - let a = pattern_a( - PatternARecurrence::OnceAdHoc, - DataArrival::Mixed { - rows_per_second: SHARED_ROWS_PER_SECOND, - }, - ); - a.validate().expect("valid Pattern A"); - let queries: Vec<_> = a.query_workload.entries().map(|e| e.query.0).collect(); - assert_eq!(queries, PATTERN_A.map(|pattern| pattern.promql)); - let b = pattern_b(); - b.validate().expect("valid Pattern B"); - assert_eq!(evaluation_interval_ms(&b, 0), Some(PATTERN_B_INTERVAL_MS)); -} - -// ── Pattern A: query-language frontends ─────────────────────────────── - -/// Each Pattern A query lowers to scan → (time shift) → range → quantile. -#[test] -fn stage0_query_language_frontends_a_lowers_each_query_to_its_interval() { - let roots = lower_promql(&pattern_a( - PatternARecurrence::OnceAdHoc, - DataArrival::Mixed { - rows_per_second: SHARED_ROWS_PER_SECOND, - }, - )); - let ops: Vec<_> = roots.iter().map(frontend_operations).collect(); - let plain = vec![op("aggregate:quantile"), op("scan"), op("time_range")]; - let shifted = vec![ - op("aggregate:quantile"), - op("scan"), - op("time_range"), - op("time_shift"), - ]; - assert_eq!( - ops, - [ - plain.clone(), - plain, - shifted.clone(), - shifted.clone(), - shifted - ] - ); -} - -/// The frontends give one summary-free DAG whose five queries share no node. -#[test] -fn stage0_query_language_frontends_a_one_summary_free_dag_without_sharing() { - let run = run_a(); - let stage0_query_language_frontends = &run.stage0_query_language_frontends; - assert_eq!(stage0_query_language_frontends.query_roots.len(), 5); - assert!(stage0_query_language_frontends - .dag - .nodes - .iter() - .all(|n| !is_summary(&n.operator))); - assert!(cross_query_nodes( - &stage0_query_language_frontends.dag, - &stage0_query_language_frontends.query_roots - ) - .is_empty()); -} - -// ── Pattern A: logical ASAP-aware optimization ─────────────────────────── - -/// Pass 1 offers each query an exact and a KLL candidate over its own interval. -#[test] -fn stage1_logical_asap_optimization_a_pass1_offers_exact_and_kll_per_query() { - let run = run_a(); - for q in 0..5 { - let found = options(&run, q); - assert!( - found.contains("exact") && found.contains("Kll"), - "q{}: {found:?}", - q + 1 - ); - } -} - -/// The five independent KLLs: one per query, each read by its query only, all sized for ε = 0.005. -#[test] -fn stage1_logical_asap_optimization_a_keeps_five_independent_klls() { - let run = run_a(); - let all_kll = run - .logical - .iter() - .find(|c| { - !c.shared_input() && (0..5).all(|q| query_option(&c.dag, &c.query_roots, q) == "Kll") - }) - .expect("independent all-KLL candidate"); - let builds = sketch_builds(&all_kll.dag); - assert_eq!(builds.len(), 5); - let readers: BTreeSet<_> = builds - .iter() - .map(|(id, ..)| readers(&all_kll.dag, &all_kll.query_roots, *id)) - .collect(); - assert_eq!(readers, (0..5).map(|q| BTreeSet::from([q])).collect()); - let sizes: BTreeSet<_> = builds - .iter() - .map(|(_, _, params)| match params { - SketchParams::Kll { k, .. } => *k, - other => panic!("{other:?}"), - }) - .collect(); - assert_eq!(sizes.len(), 1, "one ε, one KLL size: {sizes:?}"); -} - -/// The identical-expression rule shares only raw input (scan, range, shift), never a quantile or summary, and keeps the unshared variant. -#[test] -fn stage1_logical_asap_optimization_a_identical_expression_rule_shares_only_raw_input() { - let run = run_a(); - assert!(run.logical.iter().any(|c| c.shared_input())); - assert!(run.logical.iter().any(|c| !c.shared_input())); - for c in &run.logical { - for id in cross_query_nodes(&c.dag, &c.query_roots) { - let kind = relational(c.dag.payload(id)); - assert!( - matches!(kind.as_deref(), Some("scan" | "time_range" | "time_shift")), - "{}: shared {kind:?}", - c.id - ); - } - } -} - -/// One Exponential Histogram of KLLs over [T − 5y, T] serves all five queries, each through its own merge and p99 estimate. -#[test] -#[ignore = "needs Pass 2 window composition across queries: one window summary for all five (#628, #634)"] -fn stage1_logical_asap_optimization_a_window_composition_adds_one_eh_for_all_five() { - let run = run_a(); - let found = run.logical.iter().find(|c| { - windowed_builds(c).iter().any(|(_, a, form, readers)| { - *a == SketchAlgorithm::Kll - && matches!(form, WindowForm::ExponentialHistogram { horizon_ms } if *horizon_ms >= 5 * YEAR_MS) - && readers.len() == 5 - }) - }); - let c = found.expect("a candidate with one EH of KLLs read by all five queries"); - let (eh, ..) = windowed_builds(c) - .into_iter() - .find(|(_, _, form, _)| matches!(form, WindowForm::ExponentialHistogram { .. })) - .unwrap(); - let estimates = estimates_of(&c.dag, eh); - assert_eq!(estimates.len(), 5, "one estimate per query"); - for (estimate, statistic) in estimates { - assert_eq!(statistic, SketchStatistic::Quantile { q: 0.99 }); - let merged = c - .dag - .producers(estimate) - .into_iter() - .any(|p| is_merge(c.dag.payload(p))); - assert!( - merged, - "{}: estimate {estimate:?} does not read a merge", - c.id - ); - } -} - -/// The shared EH candidate is added next to the independent KLL candidates, not instead of them. -#[test] -#[ignore = "needs Pass 2 window composition across queries: one window summary for all five (#628, #634)"] -fn stage1_logical_asap_optimization_a_keeps_independent_and_shared_window_summaries() { - let run = run_a(); - let shared = run.logical.iter().any(|c| { - windowed_builds(c).iter().any(|(_, _, f, r)| { - matches!(f, WindowForm::ExponentialHistogram { .. }) && r.len() == 5 - }) - }); - let independent = run.logical.iter().any(|c| { - let builds = windowed_builds(c); - builds.len() == 5 - && builds.iter().all(|(_, a, f, r)| { - *a == SketchAlgorithm::Kll && *f == WindowForm::None && r.len() == 1 - }) - }); - assert!(shared && independent); -} - -/// Pass 2 adds a candidate for each way of grouping two queries onto one shared window summary. -#[test] -#[ignore = "needs Pass 2 window composition across queries: partial groupings (#628, #634)"] -fn stage1_logical_asap_optimization_a_window_composition_groups_every_pair() { - let run = run_a(); - let groups: BTreeSet<_> = run - .logical - .iter() - .flat_map(windowed_builds) - .filter(|(_, _, f, r)| *f != WindowForm::None && r.len() > 1) - .map(|(.., r)| r) - .collect(); - for a in 0..5 { - for b in a + 1..5 { - assert!( - groups.contains(&BTreeSet::from([a, b])), - "pair q{} q{}", - a + 1, - b + 1 - ); - } - } -} - -// ── Pattern A: plan selection ──────────────────────────────────────────── - -/// Plan selection selects one Pattern A candidate, explains the rest, and picks the cheapest valid one. -#[test] -fn stage3_plan_selection_a_selects_cheapest_valid() { - let run = run_a(); - assert_selects_one_and_explains_the_rest(&run); - assert_selects_cheapest_valid(&run); -} - -/// Every node of a valid candidate is charged once per query that reads it: nothing is materialized, so a shared node is recomputed for each. -#[test] -fn stage3_plan_selection_a_charges_each_node_per_consuming_query() { - assert_each_node_charged_per_consuming_query(&run_a()); -} - -/// Sharing the scan never costs more than scanning separately, for the same per-computation choices. -#[test] -#[ignore = "needs plan selection to charge a shared scan that is not materialized per query by the \ - window each query reads, not by the longest window it serves (#632)"] -fn stage3_plan_selection_a_shared_scan_is_not_costlier() { - let run = run_a(); - let key = |c: &Logical| { - (0..5) - .map(|q| query_option(&c.dag, &c.query_roots, q)) - .collect::>() - }; - let cost = |c: &Logical| run.cost(&run.physical_of(c).next().unwrap().id); - let mut compared = 0; - for shared in run.logical.iter().filter(|c| c.shared_input()) { - let separate = run - .logical - .iter() - .find(|c| !c.shared_input() && key(c) == key(shared)) - .expect("independent counterpart"); - if let (Some(s), Some(i)) = (cost(shared), cost(separate)) { - assert!(s <= i, "{} {s} vs {} {i}", shared.id, separate.id); - compared += 1; - } - } - assert!(compared > 0); -} - -// ── Pattern B: query-language frontends and Pass 1 ──────────────────── - -/// Pattern B lowers to scan → range 5m → quantile, with no summary. -#[test] -fn stage0_query_language_frontends_b_lowers_to_one_range_quantile() { - let run = run_b(); - let roots = lower_promql(&pattern_b()); - assert_eq!( - frontend_operations(&roots[0]), - [op("aggregate:quantile"), op("scan"), op("time_range")] - ); - assert!(run - .stage0_query_language_frontends - .dag - .nodes - .iter() - .all(|n| !is_summary(&n.operator))); -} - -/// Pass 1 offers an exact and a KLL candidate for the 5-min window. -#[test] -fn stage1_logical_asap_optimization_b_pass1_offers_exact_and_kll() { - let found = options(&run_b(), 0); - assert!( - found.contains("exact") && found.contains("Kll"), - "{found:?}" - ); -} - -// ── Pattern B: window composition ──────────────────────────────────────── - -/// Each Pass 1 option gets three window forms: none, a 5-min sliding window with a 1-min slide, and 1-min tumbling windows. -#[test] -#[ignore = "needs Pass 2 window composition: sliding windows (#522); tumbling windows pass in \ - stage1_logical_asap_optimization_b_window_composition_adds_tumbling_per_mergeable_option"] -fn stage1_logical_asap_optimization_b_window_composition_adds_sliding_and_tumbling_per_option() { - let run = run_b(); - let summaries: BTreeSet<_> = options(&run, 0) - .into_iter() - .filter(|o| o != "exact") - .collect(); - let found: BTreeSet<_> = run - .logical - .iter() - .flat_map(windowed_builds) - .map(|(_, a, f, _)| (format!("{a:?}"), f)) - .collect(); - let forms = [ - WindowForm::None, - WindowForm::Sliding { - length_ms: PATTERN_B_WINDOW_MS, - slide_ms: PATTERN_B_INTERVAL_MS, - }, - WindowForm::Tumbling { - length_ms: PATTERN_B_INTERVAL_MS, - }, - ]; - for option in &summaries { - for form in forms { - assert!(found.contains(&(option.clone(), form)), "{option} {form:?}"); - } - } -} - -/// Each summary option of Pass 1 (KLL and DDSketch, both mergeable) also -/// comes as 1-min tumbling windows, gcd(5 min, 1 min), merged five at a time -/// before its p99 estimate; the whole-window builds are kept and read no -/// merge. Exact has no summary to split: 2 options × 2 forms + exact = 5. -#[test] -fn stage1_logical_asap_optimization_b_window_composition_adds_tumbling_per_mergeable_option() { - let run = run_b(); - assert_eq!(run.logical.len(), 5); - let mut found = BTreeSet::new(); - for c in &run.logical { - for (build, algorithm, form, _) in windowed_builds(c) { - let merged = estimates_of(&c.dag, build).iter().all(|(e, _)| { - c.dag - .producers(*e) - .into_iter() - .any(|p| is_merge(c.dag.payload(p))) - }); - assert_eq!(merged, form != WindowForm::None, "{}: {form:?}", c.id); - if let asap_types::ir::Operator::ASAP( - asap_types::ir::ASAPOp::SummaryTumblingWindowChain { - tumbling_window_count, - .. - }, - ) = c.dag.payload(build) - { - assert_eq!(*tumbling_window_count, 5, "{}", c.id); - } - found.insert((format!("{algorithm:?}"), form)); - } - } - let tumbling = WindowForm::Tumbling { - length_ms: PATTERN_B_INTERVAL_MS, - }; - let expected: BTreeSet<_> = ["DDSketch", "Kll"] - .into_iter() - .flat_map(|a| [(a.to_string(), WindowForm::None), (a.to_string(), tumbling)]) - .collect(); - assert_eq!(found, expected); -} - -/// Window parameters are legal: L divides W, s divides L and the evaluation interval, a tumbling length divides both. -#[test] -fn stage1_logical_asap_optimization_b_window_parameters_are_legal() { - let tumbling = run_b() - .logical - .iter() - .flat_map(windowed_builds) - .filter(|(_, _, form, _)| matches!(form, WindowForm::Tumbling { .. })) - .count(); - assert!(tumbling > 0, "not vacuous"); - let (w, every) = (PATTERN_B_WINDOW_MS, PATTERN_B_INTERVAL_MS); - for c in &run_b().logical { - for (_, _, form, _) in windowed_builds(c) { - match form { - WindowForm::Sliding { - length_ms, - slide_ms, - } => { - assert_eq!(w % length_ms, 0, "{}: L | W", c.id); - assert_eq!(length_ms % slide_ms, 0, "{}: s | L", c.id); - assert_eq!(every % slide_ms, 0, "{}: s | interval", c.id); - } - WindowForm::Tumbling { length_ms } => { - assert_eq!(w % length_ms, 0, "{}: length | W", c.id); - assert_eq!(every % length_ms, 0, "{}: length | interval", c.id); - } - WindowForm::ExponentialHistogram { .. } => { - panic!("{}: Pattern B has no EH", c.id) - } - WindowForm::None => {} - } - } - } -} - -/// A sliding window with L = W reads one completed window (no merge); tumbling windows merge W / length of them. -#[test] -#[ignore = "needs Pass 2 window composition: sliding windows (#522); tumbling merges pass in \ - stage1_logical_asap_optimization_b_window_composition_adds_tumbling_per_mergeable_option"] -fn stage1_logical_asap_optimization_b_merge_only_where_the_window_form_needs_it() { - let run = run_b(); - let mut seen = BTreeSet::new(); - for c in &run.logical { - for (build, _, form, _) in windowed_builds(c) { - let merged = estimates_of(&c.dag, build).iter().any(|(e, _)| { - c.dag - .producers(*e) - .into_iter() - .any(|p| is_merge(c.dag.payload(p))) - }); - assert_eq!( - merged, - needs_merge(form, PATTERN_B_WINDOW_MS), - "{}: {form:?}", - c.id - ); - if form != WindowForm::None { - seen.insert(merged); - } - } - } - assert_eq!( - seen, - BTreeSet::from([false, true]), - "both a sliding and a tumbling candidate" - ); -} - -/// A window form that merges is used only with a summary whose states merge (KLL, DDSketch). -#[test] -fn stage1_logical_asap_optimization_window_merges_use_mergeable_summaries() { - for (run, window) in [(run_a(), 5 * YEAR_MS), (run_b(), PATTERN_B_WINDOW_MS)] { - for c in &run.logical { - for (_, algorithm, form, _) in windowed_builds(c) { - if needs_merge(form, window) { - assert!( - matches!(algorithm, SketchAlgorithm::Kll | SketchAlgorithm::DDSketch), - "{}: {algorithm:?} {form:?}", - c.id - ); - } - } - } - } -} - -// ── Pattern B: plan selection ──────────────────────────────────────────── - -/// Plan selection selects one Pattern B candidate, explains the rest, and picks the cheapest valid one. -#[test] -fn stage3_plan_selection_b_selects_cheapest_valid() { - let run = run_b(); - assert_selects_one_and_explains_the_rest(&run); - assert_selects_cheapest_valid(&run); - assert_each_node_charged_per_consuming_query(&run); -} - -/// With nothing materialized, the 1-min KLL tumbling windows are all rebuilt at every -/// evaluation: they read the same 5 minutes of samples and update the same -/// states as one whole-window KLL, and merging five tumbling windows per series adds -/// 1M × 5 operations a minute. Each evaluation also holds its states while -/// it runs: the whole window's 1M KLLs, or the five tumbling windows' 5M smaller ones, -/// for the evaluation's latency. So whole-window KLL is selected at 2.363 per -/// second and the tumbling windows cost 3.611; exact costs 2.458. DDSketch, whole or in -/// tumbling windows, misses the rank target. -#[test] -fn stage3_plan_selection_b_tumbling_windows_rebuilt_at_query_time_add_the_merge() { - let run = run_b(); - let cost = |c: &Logical| run.cost(&run.physical_of(c).next().unwrap().id); - let with = |algorithm: &str, tumbling_windows: bool| { - run.logical - .iter() - .find(|c| { - query_option(&c.dag, &c.query_roots, 0) == algorithm - && windowed_builds(c) - .iter() - .any(|(_, _, f, _)| (*f != WindowForm::None) == tumbling_windows) - }) - .unwrap_or_else(|| panic!("{algorithm} tumbling_windows={tumbling_windows}")) - }; - let whole = cost(with("Kll", false)).unwrap(); - let tumbling_windows = cost(with("Kll", true)).unwrap(); - assert!((whole - 2.362_500_025_2).abs() < 1e-9, "{whole}"); - assert!( - (tumbling_windows - 3.611_233_350_8).abs() < 1e-9, - "{tumbling_windows}" - ); - assert_eq!( - run.selection.selected, - run.physical_of(with("Kll", false)).next().unwrap().id - ); - for tumbling_windows in [false, true] { - assert_eq!( - cost(with("DDSketch", tumbling_windows)), - None, - "tumbling_windows={tumbling_windows}" - ); - } -} diff --git a/crates/integration-tests/tests/planner_layering_example4.rs b/crates/integration-tests/tests/planner_layering_example4.rs deleted file mode 100644 index a47c5fbfa..000000000 --- a/crates/integration-tests/tests/planner_layering_example4.rs +++ /dev/null @@ -1,509 +0,0 @@ -//! Rule-fixture and production acceptance for Example 4: materialization and objective models. -//! -//! Numerical, accuracy, and operator assertions use explicitly supplied -//! configurations of the existing rules and physical evaluator. They do not -//! assume a bounded MILP run archives every supported workload plan. Separate -//! production checks exercise memo search and validated output selection. -//! Unsupported algorithms retain their existing ignored tests. - -use crate::planner_layering_common; - -use std::collections::{BTreeMap, BTreeSet}; - -use asap_cost::{ClusterLimits, ClusterObjective, CostInputs, CostModel, ResourceUnitCost}; -use asap_logical_asap_optimization::DeploymentCapabilities; -use asap_types::ir::flat::NodeId; -use asap_types::ir::schema::SketchAlgorithm; -use asap_types::workload::{DataArrival, PlanningWorkload}; -use planner_layering_common::*; - -/// Pattern A as given: one ad hoc batch at T over mixed data. -fn once() -> PlanningWorkload { - pattern_a( - PatternARecurrence::OnceAdHoc, - DataArrival::Mixed { - rows_per_second: SHARED_ROWS_PER_SECOND, - }, - ) -} - -fn monthly() -> PlanningWorkload { - pattern_a( - PatternARecurrence::MonthlyPredictable, - DataArrival::Mixed { - rows_per_second: SHARED_ROWS_PER_SECOND, - }, - ) -} - -fn at_rest() -> PlanningWorkload { - pattern_a(PatternARecurrence::OnceAdHoc, DataArrival::AtRest) -} - -/// The window-summary build of `c` with form `want` (`Some(form)` matches -/// exactly; `None` matches any EH) read by `readers` queries. -fn window_build(c: &Logical, want: Option, readers_count: usize) -> Option { - sketch_builds(&c.dag) - .into_iter() - .find_map(|(id, algorithm, _)| { - let form = window_form(&c.dag, id); - let matches = match want { - Some(want) => form == want, - None => matches!(form, WindowForm::ExponentialHistogram { .. }), - }; - (algorithm == SketchAlgorithm::Kll - && matches - && readers(&c.dag, &c.query_roots, id).len() == readers_count) - .then_some(id) - }) -} - -/// The physical candidates of the logical candidate with a KLL window -/// summary of `form` read by `readers_count` queries, keyed by the -/// materialization of that build. Several logical candidates may match (for -/// example with and without the shared input); the first is used. -fn options_of( - run: &Run, - form: Option, - readers_count: usize, -) -> BTreeMap { - let logical = run - .logical - .iter() - .find(|c| window_build(c, form, readers_count).is_some()) - .unwrap_or_else(|| { - let form = form.map_or("EH".to_string(), |f| format!("{f:?}")); - panic!("no logical candidate with a {form} KLL window summary") - }); - let build = window_build(logical, form, readers_count).unwrap(); - // Physical ASAP-aware optimization renumbers nodes; find the build again - // in each physical DAG. - run.physical_of(logical) - .map(|p| { - let id = sketch_builds(p.dag()) - .into_iter() - .find(|(id, a, _)| { - *a == SketchAlgorithm::Kll - && window_form(p.dag(), *id) == window_form(&logical.dag, build) - }) - .map(|(id, ..)| id) - .expect("the window summary survives physical ASAP-aware optimization"); - (materialization(p, id), (p.id.clone(), id)) - }) - .collect() -} - -fn eh_options(run: &Run) -> BTreeMap { - options_of(run, None, 5) -} - -fn tumbling() -> Option { - Some(WindowForm::Tumbling { - length_ms: PATTERN_B_INTERVAL_MS, - }) -} - -fn sliding() -> Option { - Some(WindowForm::Sliding { - length_ms: PATTERN_B_WINDOW_MS, - slide_ms: PATTERN_B_INTERVAL_MS, - }) -} - -fn cost(run: &Run, id: &str) -> f64 { - run.cost(id).unwrap_or_else(|| panic!("{id} is not priced")) -} - -use MaterializationOption::{MatAtIngestion, MatAtQuery, NotMaterialized}; - -// ── Constraints over every candidate ───────────────────────────────────── - -/// Every node upstream of an ingestion-time node also runs at ingestion time, in every workload variant. -/// The monthly variant runs in -/// `stage2_physical_asap_optimization_monthly_ingestion_time_upstream_is_ingestion_time`. -#[test] -fn stage2_physical_asap_optimization_ingestion_time_upstream_is_ingestion_time() { - for workload in [once(), at_rest(), pattern_b()] { - assert_ingestion_upstream_is_ingestion(&run_promql(&workload)); - } -} - -/// The monthly variant of -/// `stage2_physical_asap_optimization_ingestion_time_upstream_is_ingestion_time`: -/// it plans 67,228 candidates, too slow for a debug build in CI. -#[test] -#[ignore = "slow: run in release by hand"] -fn stage2_physical_asap_optimization_monthly_ingestion_time_upstream_is_ingestion_time() { - assert_ingestion_upstream_is_ingestion(&run_promql(&monthly())); -} - -/// Data at rest has no ingestion, so no candidate runs anything at ingestion time. -#[test] -fn stage2_physical_asap_optimization_at_rest_runs_nothing_at_ingestion_time() { - let run = run_promql(&at_rest()); - for p in &run.physical { - for n in &p.dag().nodes { - assert!(!runs_at_ingestion(p, n.id), "{}: {:?}", p.id, n.id); - } - } -} - -/// Each Pattern A query's range window and offset, as its PromQL text states them. -fn pattern_a_windows_and_offsets() -> (Vec, Vec) { - let windows = PATTERN_A.iter().map(|p| p.window_ms).collect(); - let offsets = PATTERN_A.iter().map(|p| p.offset_before_t_ms).collect(); - (windows, offsets) -} - -/// Every materialized output of every candidate of `workload` is kept for at -/// least its longest reader's `windows[q] + offsets[q]`. -fn assert_materialized_outputs_cover_their_consumers( - workload: &PlanningWorkload, - windows: &[u64], - offsets: &[u64], -) { - let run = run_promql(workload); - for p in &run.physical { - for n in &p.dag().nodes { - // A shared scan or range built once per evaluation holds its - // rows within that evaluation; plan selection reads it over the - // longest window above it. - let shared_rows = materialization(p, n.id) == MatAtQuery - && matches!( - relational(&n.payload).as_deref(), - Some("scan" | "time_range" | "time_shift") - ); - if materialization(p, n.id) == NotMaterialized || shared_rows { - continue; - } - let need = readers(p.dag(), &p.query_roots, n.id) - .into_iter() - .map(|q| windows[q] + offsets[q]) - .max() - .unwrap_or(0); - let kept = retention_ms(p, n.id).unwrap_or_else(|| { - panic!("{}: {:?} is materialized without a retention", p.id, n.id) - }); - assert!(kept >= need, "{}: {:?} kept {kept} < {need}", p.id, n.id); - } - } -} - -/// A materialized output is kept for as long as its consumers read: it covers every reader's lookback and offset. -/// The monthly variant runs in -/// `stage2_physical_asap_optimization_monthly_materialized_output_covers_its_consumers`. -#[test] -fn stage2_physical_asap_optimization_materialized_output_covers_its_consumers() { - let (windows, offsets) = pattern_a_windows_and_offsets(); - for workload in [once(), at_rest()] { - assert_materialized_outputs_cover_their_consumers(&workload, &windows, &offsets); - } - assert_materialized_outputs_cover_their_consumers(&pattern_b(), &[PATTERN_B_WINDOW_MS], &[0]); -} - -/// The monthly variant of -/// `stage2_physical_asap_optimization_materialized_output_covers_its_consumers`: -/// it plans 67,228 candidates, too slow for a debug build in CI. -#[test] -#[ignore = "slow: run in release by hand"] -fn stage2_physical_asap_optimization_monthly_materialized_output_covers_its_consumers() { - let (windows, offsets) = pattern_a_windows_and_offsets(); - assert_materialized_outputs_cover_their_consumers(&monthly(), &windows, &offsets); -} - -// ── Pattern A ──────────────────────────────────────────────────────────── - -/// The shared EH candidate yields A1 (query time, kept), A2 (ingestion time) and A3 (not materialized). -#[test] -#[ignore = "needs Pass 2 window segments shared across queries (#628, #634); Stage 2 materialization (MatAtQuery within one batch) is in place"] -fn stage2_physical_asap_optimization_a_shared_eh_has_three_materialization_options() { - let found: BTreeSet<_> = eh_options(&run_promql(&once())).into_keys().collect(); - assert_eq!( - found, - BTreeSet::from([MatAtIngestion, MatAtQuery, NotMaterialized]) - ); -} - -/// With data at rest, A2 is not generated; A1 and A3 remain. -#[test] -#[ignore = "needs Pass 2 window segments shared across queries (#628, #634); Stage 2 materialization (MatAtQuery within one batch) is in place"] -fn stage2_physical_asap_optimization_a_at_rest_drops_the_ingestion_time_option() { - let found: BTreeSet<_> = eh_options(&run_promql(&at_rest())).into_keys().collect(); - assert_eq!(found, BTreeSet::from([MatAtQuery, NotMaterialized])); -} - -/// A materialized shared EH is one build node read by all five queries, charged once. -#[test] -#[ignore = "needs Pass 2 window segments shared across queries (#628, #634); Stage 2 materialization (MatAtQuery within one batch) is in place"] -fn stage2_physical_asap_optimization_a_materialized_eh_is_built_once_for_all_consumers() { - let run = run_promql(&once()); - for (m, (id, build)) in eh_options(&run) { - if m == NotMaterialized { - continue; - } - let p = run.physical(&id); - let ehs = sketch_builds(p.dag()) - .into_iter() - .filter(|(b, ..)| { - matches!( - window_form(p.dag(), *b), - WindowForm::ExponentialHistogram { .. } - ) - }) - .count(); - assert_eq!(ehs, 1, "{id}"); - assert_eq!(readers(p.dag(), &p.query_roots, build).len(), 5, "{id}"); - assert!( - run.selection.costs[&id].per_node.contains_key(&build), - "{id}" - ); - } -} - -/// A3 rebuilds the EH for each of the five queries (plan selection charges a -/// `NotMaterialized` node once per consuming query), so it costs more than A1, -/// which builds it once. -#[test] -#[ignore = "needs Pass 2 window segments shared across queries (#628, #634); Stage 2 materialization (MatAtQuery within one batch) is in place"] -fn stage3_plan_selection_a_rebuilding_per_query_costs_more_than_building_once() { - let run = run_promql(&once()); - let options = eh_options(&run); - let (a1, a3) = (&options[&MatAtQuery].0, &options[&NotMaterialized].0); - assert!( - cost(&run, a3) > cost(&run, a1), - "A3 {} vs A1 {}", - cost(&run, a3), - cost(&run, a1) - ); -} - -/// As given (run once, ad hoc), A1 is the cheapest of the three: A2 maintains years of history for one batch. -#[test] -#[ignore = "needs Pass 2 window segments shared across queries (#628, #634); Stage 2 materialization (MatAtQuery within one batch) is in place"] -fn stage3_plan_selection_a_once_adhoc_prefers_the_query_time_eh() { - let run = run_promql(&once()); - let options = eh_options(&run); - let a1 = cost(&run, &options[&MatAtQuery].0); - for (m, (id, _)) in &options { - assert!(a1 <= cost(&run, id), "A1 {a1} vs {m:?} {}", cost(&run, id)); - } -} - -/// Repeated monthly and predictable, A2's maintenance is shared by many batches, so A2 gains on A1. -#[test] -#[ignore = "needs Pass 2 window segments shared across queries (#628, #634); Stage 2 materialization (MatAtQuery within one batch) is in place"] -fn stage3_plan_selection_a_monthly_amortizes_ingestion_time_maintenance() { - let ratio = |workload: PlanningWorkload| { - let run = run_promql(&workload); - let options = eh_options(&run); - cost(&run, &options[&MatAtIngestion].0) / cost(&run, &options[&MatAtQuery].0) - }; - let (once, monthly) = (ratio(once()), ratio(monthly())); - assert!(monthly < once, "A2/A1: monthly {monthly} vs once {once}"); -} - -// ── Pattern B ──────────────────────────────────────────────────────────── - -/// A deployment that keeps query-time results across evaluations, which B3 -/// needs; the default deployment keeps none. -fn keeping_query_time_results() -> DeploymentCapabilities { - DeploymentCapabilities { - query_time_results_retained: true, - ..DeploymentCapabilities::default() - } -} - -/// The 1-min tumbling KLL candidate yields B1 (ingestion time), B2 (not materialized) and B3 (query time, kept). -#[test] -fn stage2_physical_asap_optimization_b_tumbling_kll_has_three_materialization_options() { - let run = run_pattern_b_fixture( - &pattern_b(), - &CostInputs::default(), - &keeping_query_time_results(), - ); - let found: BTreeSet<_> = options_of(&run, tumbling(), 1).into_keys().collect(); - assert_eq!( - found, - BTreeSet::from([MatAtIngestion, MatAtQuery, NotMaterialized]) - ); -} - -/// B1 builds the tumbling KLLs at ingestion time and merges and estimates at query time. -#[test] -fn stage2_physical_asap_optimization_b_b1_builds_at_ingestion_and_merges_at_query_time() { - let run = run_pattern_b_fixture( - &pattern_b(), - &CostInputs::default(), - &DeploymentCapabilities::default(), - ); - let (id, build) = options_of(&run, tumbling(), 1)[&MatAtIngestion].clone(); - let p = run.physical(&id); - assert!(runs_at_ingestion(p, build)); - let estimates = estimates_of(p.dag(), build); - assert!(!estimates.is_empty()); - for (estimate, _) in estimates { - assert!( - !runs_at_ingestion(p, estimate), - "{id}: estimate at ingestion" - ); - let merge = p - .dag() - .producers(estimate) - .into_iter() - .find(|&m| is_merge(p.dag().payload(m))); - let merge = merge.unwrap_or_else(|| panic!("{id}: no merge before the estimate")); - assert!(!runs_at_ingestion(p, merge), "{id}: merge at ingestion"); - } -} - -/// Implementer test: for the default deployment, which keeps no query-time -/// result across evaluations, the tumbling KLL candidate yields B1 -/// (maintained at ingestion time) and B2 (rebuilt at every evaluation), not -/// B3; both are priced, and B2's evaluation stays within the 200 ms bound. -#[test] -fn stage2_physical_asap_optimization_b_tumbling_kll_has_b1_and_b2() { - let run = run_pattern_b_fixture( - &pattern_b(), - &CostInputs::default(), - &DeploymentCapabilities::default(), - ); - let options = options_of(&run, tumbling(), 1); - assert_eq!( - options.keys().copied().collect::>(), - BTreeSet::from([MatAtIngestion, NotMaterialized]) - ); - for (id, _) in options.values() { - assert!(run.cost(id).is_some(), "{id} is priced"); - let latency = run.selection.costs[id].usage.latency_per_query[0].total_ms(); - assert!(latency <= 200.0, "{id}: {latency} ms"); - } -} - -/// B3 runs nothing at ingestion time; it keeps the tumbling KLLs it builds at query time. -/// Implementer addition: kept for the readers' 5-min look-back, it is -/// priced as building one 1-min tumbling window per evaluation, so it reads a fifth of -/// B2's raw data and keeps a fifth of it on disk. -#[test] -fn stage2_physical_asap_optimization_b_b3_keeps_query_time_windows() { - let run = run_pattern_b_fixture( - &pattern_b(), - &CostInputs::default(), - &keeping_query_time_results(), - ); - let options = options_of(&run, tumbling(), 1); - let (id, build) = options[&MatAtQuery].clone(); - let p = run.physical(&id); - assert!( - p.dag().nodes.iter().all(|n| !runs_at_ingestion(p, n.id)), - "{id}" - ); - assert_eq!(retention_ms(p, build), Some(PATTERN_B_WINDOW_MS)); - let disk = |id: &str| run.selection.costs[id].usage.disk_bytes; - assert_eq!(disk(&id) * 5, disk(&options[&NotMaterialized].0)); -} - -/// The sliding-window KLL is kept from ingestion time or from query time; not materializing it is the no-window plan. -#[test] -#[ignore = "needs sliding windows (#522)"] -fn stage2_physical_asap_optimization_b_sliding_kll_has_two_materialization_options() { - let found: BTreeSet<_> = options_of( - &run_pattern_b_fixture( - &pattern_b(), - &CostInputs::default(), - &DeploymentCapabilities::default(), - ), - sliding(), - 1, - ) - .into_keys() - .collect(); - assert_eq!(found, BTreeSet::from([MatAtIngestion, MatAtQuery])); -} - -// ── Cost models (#683, #660) ───────────────────────────────────────────── - -/// Pattern B under `cost_model`, with the default resource speed and horizon. -fn pattern_b_under(cost_model: CostModel) -> Run { - run_pattern_b_fixture( - &pattern_b(), - &CostInputs { - cost_model, - ..Default::default() - }, - &DeploymentCapabilities::default(), - ) -} - -/// The candidates the cost models choose between: the exact plan, the KLL -/// rebuilt over the whole window at query time and B1 (the tumbling windows maintained at ingestion time). -struct PatternBCandidates { - exact: String, - whole_window_kll: String, - b1: String, -} - -fn pattern_b_candidates(run: &Run) -> PatternBCandidates { - let tumbling_windows = options_of(run, tumbling(), 1); - PatternBCandidates { - exact: run - .physical - .iter() - .find(|p| sketch_builds(p.dag()).is_empty()) - .expect("an exact candidate") - .id - .clone(), - whole_window_kll: options_of(run, Some(WindowForm::None), 1)[&NotMaterialized] - .0 - .clone(), - b1: tumbling_windows[&MatAtIngestion].0.clone(), - } -} - -/// A cluster with `memory_gb` GB of memory and 1 TB of disk. -fn cluster(memory_gb: u64, objective: ClusterObjective) -> CostModel { - CostModel::ClusterCapacity(ClusterLimits { - memory_bytes: memory_gb * 1_000_000_000, - disk_bytes: 1_000_000_000_000, - cpu_cores: None, - objective, - }) -} - -/// Each cost model chooses what its deployment pays for. Cost model 1 -/// (usage over time) picks the KLL rebuilt every minute: it uses memory only -/// while it runs, and B1 pays for 5.2 GB of tumbling windows every second. Cost model 2 -/// (provisioned capacity) picks the exact plan: every candidate needs one -/// core for its evaluation, and the exact plan's peak memory (192 MB of -/// per-series values) is the smallest, while the KLL's 1M states, each -/// grown for its 20 samples, take 1 GB at once. Cost model 3 (own cluster, -/// lowest latency) picks B1 when its 6.1 GB fit, since merging maintained -/// tumbling windows takes 7 ms against 126 ms for rebuilding; in 4 GB, B1 is invalid -/// and the KLL is fastest. -#[test] -fn stage3_plan_selection_b_cost_models_choose_differently() { - let usage_over_time = - pattern_b_under(CostModel::UsageOverTime(ResourceUnitCost::illustrative())); - let ids = pattern_b_candidates(&usage_over_time); - assert_eq!(usage_over_time.selection.selected, ids.whole_window_kll); - - let provisioned = pattern_b_under(CostModel::ProvisionedCapacity( - ResourceUnitCost::illustrative(), - )); - assert_eq!(provisioned.selection.selected, ids.exact); - let peak = |run: &Run, id: &str| run.selection.costs[id].usage.peak_memory_bytes; - assert!(peak(&provisioned, &ids.exact) < peak(&provisioned, &ids.whole_window_kll)); - - let large = pattern_b_under(cluster(256, ClusterObjective::LowestLatency)); - assert_eq!(large.selection.selected, ids.b1); - assert!(large - .selection - .costs - .values() - .all(|c| c.cost_per_second.is_none())); - - let small = pattern_b_under(cluster(4, ClusterObjective::LowestLatency)); - assert_eq!(small.selection.selected, ids.whole_window_kll); - let invalid = small.invalid(); - let b1 = invalid[ids.b1.as_str()]; - assert!(b1.starts_with("peak memory"), "{b1}"); -} diff --git a/crates/integration-tests/tests/precompute_raw_samples.rs b/crates/integration-tests/tests/precompute_raw_samples.rs deleted file mode 100644 index 9db3c4947..000000000 --- a/crates/integration-tests/tests/precompute_raw_samples.rs +++ /dev/null @@ -1,641 +0,0 @@ -//! Planner-selected summaries over raw data samples compile, bound to raw data -//! at their `Scan`, and produce the same estimates as feeding their kernel -//! sample by sample. -use crate::physical_common; -use asap_types::ir::physical_export::{PhysicalASAPDAG, PhysicalASAPOperatorPayload}; -use asap_types::ir::ASAPOp; -use asap_types::ir::OperatorNode; -use physical_common::compile_at_ingestion_physical_asap_dag; -use std::{collections::BTreeMap, collections::BTreeSet, rc::Rc, sync::Arc}; - -use asap_executor::{ - factory::create_planner_accumulator, - physical_planner::{compile, promql_rows, CompiledPhysicalDAG}, - runtime::{Limits, RunContext, Scope}, - summary_kernels::{exact::ExactAccumulator, weighted_frequency::WeightedFrequency}, - values::Value, - AggregateCore, -}; -use asap_integration_tests::fixtures::lower_promql; -use asap_integration_tests::planning::explicit_local_candidates; -use asap_types::ir::operator::Reduction; -use asap_types::ir::properties::AccuracyTarget; -use asap_types::ir::scalar::expr_ir::ColumnRef; -use asap_types::ir::schema::{ - FieldDataType, SketchAlgorithm, SketchStatistic, SummaryInputExpr, SummaryUpdate, -}; -use futures::{executor::block_on, StreamExt}; - -type Series = BTreeMap; - -/// A series label set; `None` omits the label. An empty value is present -/// in the input but is not part of the series identity. -fn series(service: Option<&str>, instance: &str) -> Series { - [ - ("__name__", Some("m")), - ("service", service), - ("instance", Some(instance)), - ] - .into_iter() - .filter_map(|(k, v)| Some((k.to_owned(), v?.to_owned()))) - .collect() -} - -fn canonical(labels: &Series) -> Series { - labels - .iter() - .filter(|(_, v)| !v.is_empty()) - .map(|(k, v)| (k.clone(), v.clone())) - .collect() -} - -/// Exact and explicit single-rule candidates for `query`, with -/// `m` declared a counter: plan selection's answer to a heap's open -/// requirement that its weights are counter samples, which the executor -/// needs before it builds a Count-Min heap. -fn candidates(query: &str, accuracy: AccuracyTarget) -> Vec> { - use asap_types::ir::QueryRoot; - use asap_types::workload::bounds::with_confirmed_weights; - use asap_types::workload::{DataWorkload, PrometheusMetricType, SourceFacts}; - let root = lower_promql(query, accuracy.clone()).expect("lowering failed"); - let root = promql_rows::with_series_identity(&root).unwrap(); - let counter = SourceFacts { - metric_type: Some(PrometheusMetricType::Counter), - ..SourceFacts::default() - }; - let data = DataWorkload { - sources: [("m".to_string(), counter)].into(), - ..Default::default() - }; - explicit_local_candidates(root) - .into_iter() - .map(|candidate| { - match &with_confirmed_weights(&[QueryRoot::Operator(candidate)], &data)[..] { - [QueryRoot::Operator(confirmed)] => Rc::clone(confirmed), - _ => unreachable!("one operator root"), - } - }) - .collect() -} - -/// Raw-input summary nodes: `(dag, raw source id, summary id)`. -fn raw_summaries(dag: &PhysicalASAPDAG) -> Vec<(u64, u64)> { - dag.nodes - .iter() - .filter(|node| { - matches!( - node.payload, - PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryAgg { .. }) - ) - }) - .filter_map(|node| { - let inputs = dag - .edges - .iter() - .filter(|edge| edge.consumer == node.id) - .collect::>(); - let [edge] = inputs.as_slice() else { - return None; - }; - let source = dag.nodes.iter().find(|n| n.id == edge.producer)?; - matches!( - source.payload, - PhysicalASAPOperatorPayload::NonASAP(asap_types::ir::NonASAPOp::TimeRange { .. }) - ) - .then_some((source.id as u64, node.id as u64)) - }) - .collect() -} - -fn samples() -> Vec<(Series, i64, f64)> { - let mut rows = Vec::new(); - let series_set = [ - (Some("a"), "1"), - (Some("a"), "2"), - (Some("b"), "1"), - (None, "3"), - (Some("b"), ""), - ]; - for (index, (service, instance)) in series_set.iter().enumerate() { - for step in 1..=5i64 { - let value = (index as f64 + 1.0) * step as f64 + (step % 2) as f64; - rows.push((series(*service, instance), step * 1000, value)); - } - } - rows -} - -/// Compile the summary `root` alone, its raw data read at the `Scan`. -fn compile_summary(dag: &PhysicalASAPDAG, root: u64) -> Result { - compile(dag, &[], &[root]).map_err(|error| error.to_string()) -} - -/// Run the summary `root` in one ingestion run over `rows`, bound as raw -/// data at its `Scan`: each population's labels and state. -fn execute( - dag: &PhysicalASAPDAG, - root: u64, - rows: &[(Series, i64, f64)], -) -> Vec<(Series, Arc)> { - let program = compile_summary(dag, root).unwrap_or_else(|error| { - panic!("raw summary {root} does not compile: {error}"); - }); - let store = physical_common::raw_data_store(dag, |_, schema| { - rows.iter() - .map(|(labels, time, value)| { - promql_rows::series_row(schema, &canonical(labels), *time, *value).unwrap() - }) - .collect() - }); - let output = program.output_schema(root).unwrap(); - let physical_dag = program.instantiate(Arc::new(store)).unwrap(); - let context = RunContext::new( - Scope::Ingestion { - window_start_ms: 0, - window_end_ms: 6000, - }, - Limits::default(), - ) - .unwrap(); - block_on(async { - let mut stream = physical_dag - .execute(program.roots(), context) - .unwrap() - .remove(0); - let mut result = Vec::new(); - while let Some(batch) = stream.next().await { - for row in &batch.unwrap().to_rows() { - let mut labels = Series::new(); - let mut state = None; - for (field, value) in output.fields.iter().zip(row) { - match value { - Value::Utf8(identity) - if field.name == promql_rows::SERIES_IDENTITY_COLUMN => - { - labels.extend(promql_rows::decode_series_identity(identity).unwrap()) - } - Value::Utf8(label) => { - labels.insert(field.name.clone(), label.to_string()); - } - Value::Summary { state: s, .. } => state = Some(s.clone()), - Value::Timestamp(6000) => {} - other => panic!("unexpected population column {other:?} in {row:?}"), - } - } - result.push((canonical(&labels), state.expect("a summary state"))); - } - } - result - }) -} - -/// Each series' latest sample, moved to the ingestion run's end (6000 ms). -fn latest_per_series(rows: &[(Series, i64, f64)]) -> Vec<(Series, i64, f64)> { - let mut latest = BTreeMap::::new(); - for (labels, time, value) in rows { - let entry = latest.entry(canonical(labels)).or_insert((*time, *value)); - if *time >= entry.0 { - *entry = (*time, *value); - } - } - latest - .into_iter() - .map(|(labels, (_, value))| (labels, 6000, value)) - .collect() -} - -/// PromQL grouping of a canonical label set: `by` keeps the named labels, -/// `without` drops them and `__name__`. -fn population(reduction: &Reduction, dag_labels: &[String], labels: &Series) -> Series { - let labels = canonical(labels); - match reduction { - Reduction::PerEntity => labels, - Reduction::Reduce(keys) => labels - .into_iter() - .filter(|(k, _)| { - if keys.is_without() { - k != "__name__" && !dag_labels.contains(k) - } else { - dag_labels.contains(k) - } - }) - .collect(), - } -} - -/// Item identity used by a keyed summary: labels by name, the sample value, -/// or the canonical label-set identity. -fn item(expr: &SummaryInputExpr, labels: &Series, value: f64, out: &mut Vec) { - match expr { - SummaryInputExpr::Column(ColumnRef::SampleValue) => out.push(Value::Float64(value)), - SummaryInputExpr::Column(ColumnRef::Named(name)) if name == "value" => { - out.push(Value::Float64(value)) - } - SummaryInputExpr::Column(ColumnRef::Named(name)) - if name == promql_rows::SERIES_IDENTITY_COLUMN => - { - out.push(Value::Utf8( - promql_rows::encode_series_identity(&canonical(labels)) - .unwrap() - .into(), - )) - } - SummaryInputExpr::Column(ColumnRef::Named(name)) => out.push(Value::Utf8( - canonical(labels) - .get(name) - .cloned() - .unwrap_or_default() - .into(), - )), - SummaryInputExpr::Tuple(items) => items.iter().for_each(|i| item(i, labels, value, out)), - other => panic!("unsupported fixture item {other:?}"), - } -} - -fn weight(update: &SummaryUpdate, value: f64) -> f64 { - match update.weight { - SummaryInputExpr::Constant(weight) => weight, - _ => value, - } -} - -/// Estimates that identify a state's content for comparison. -fn evaluations(state: &dyn AggregateCore, family: &FieldDataType) -> Vec { - if let Some(exact) = state.as_any().downcast_ref::() { - let FieldDataType::ExactAggregate(kind, _) = family else { - unreachable!() - }; - return vec![exact.evaluation(*kind, None, None).unwrap().unwrap()]; - } - let FieldDataType::Sketch(kind, _) = family else { - panic!("sketch state for exact family") - }; - match kind.algorithm() { - SketchAlgorithm::Kll | SketchAlgorithm::DDSketch => [0.1, 0.5, 0.9] - .into_iter() - .map(|q| state.estimate(&SketchStatistic::Quantile { q }).unwrap()) - .collect(), - SketchAlgorithm::Hll | SketchAlgorithm::Kmv => { - vec![state.estimate(&SketchStatistic::Cardinality).unwrap()] - } - SketchAlgorithm::UnivMon => [ - SketchStatistic::Cardinality, - SketchStatistic::FrequencyL2, - SketchStatistic::FrequencyEntropy, - ] - .iter() - .map(|statistic| state.estimate(statistic).unwrap()) - .collect(), - other => panic!("unexpected unkeyed sketch {other:?}"), - } -} - -/// Compile one raw-input summary, execute it over `rows`, and compare each -/// population with its kernel fed sample by sample. Returns the family label, -/// or the family when it has no native state. -fn check( - query: &str, - dag: &PhysicalASAPDAG, - source: u64, - root: u64, - rows: &[(Series, i64, f64)], -) -> Result { - let node = dag.nodes.iter().find(|n| n.id as u64 == root).unwrap(); - let PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryAgg { - family, - input, - reduction, - grouping, - .. - }) = &node.payload - else { - unreachable!() - }; - let source_node = dag.nodes.iter().find(|n| n.id as u64 == source).unwrap(); - // An instant selector keeps each series' latest sample within its - // lookback before the run's end: feed one such sample per series. - let latest_rows; - let rows = if matches!( - source_node.payload, - PhysicalASAPOperatorPayload::NonASAP(asap_types::ir::NonASAPOp::TimeRange { - kind: asap_types::ir::TimeRangeKind::Instant, - .. - }) - ) { - latest_rows = latest_per_series(rows); - &latest_rows[..] - } else { - rows - }; - let keys = match reduction { - Reduction::Reduce(keys) => keys - .keys() - .iter() - .map(|i| source_node.output_schema.fields[*i].name.clone()) - .collect(), - Reduction::PerEntity => vec![], - }; - let stored_only = matches!(family, FieldDataType::Sketch(kind, _) - if kind.algorithm() == &asap_types::ir::schema::SketchAlgorithm::Cms); - if stored_only || asap_executor::capability::validate_native_family(family).is_err() { - // Families without a native state, or with native stored state only (plain CMS), are outside raw summary execution; - // their compile must fail. - assert!(compile_summary(dag, root).is_err()); - return Err(format!("{family:?}")); - } - let actual = execute(dag, root, rows); - let label = match family { - FieldDataType::ExactAggregate(kind, _) => format!("{kind:?}"), - FieldDataType::Sketch(kind, _) => format!("{:?}", kind.algorithm()), - other => format!("{other:?}"), - }; - if let FieldDataType::Sketch(kind, _) = family { - if let (Some(keyed), false) = ( - &input.item, - matches!( - kind.algorithm(), - SketchAlgorithm::Hll | SketchAlgorithm::Kmv | SketchAlgorithm::UnivMon - ), - ) { - // Keyed heaps: every item's estimated weight is its exact - // total at this scale (no collisions in the fixture). - let mut expected = BTreeMap::>::new(); - for (labels, _, value) in rows { - let mut items = Vec::new(); - item(keyed, labels, *value, &mut items); - *expected - .entry(population(reduction, &keys, labels)) - .or_default() - .entry(format!("{items:?}")) - .or_default() += weight(input, *value); - } - assert_eq!(actual.len(), expected.len(), "{query}"); - for (labels, state) in &actual { - let heap = state.as_any().downcast_ref::().unwrap(); - let got = heap - .rows(usize::MAX >> 1) - .into_iter() - .map(|mut row| { - let Some(Value::Float64(score)) = row.pop() else { - panic!("heap score") - }; - (format!("{row:?}"), score) - }) - .collect::>(); - assert_eq!(&got, &expected[labels], "{query}"); - } - return Ok(label); - } - } - let mut expected = - BTreeMap::>::new(); - for (labels, time, value) in rows { - let updater = expected - .entry(population(reduction, &keys, labels)) - .or_insert_with(|| create_planner_accumulator(family, input, grouping).unwrap()); - let unit = input.item.is_some(); - updater - .update_single(if unit { *value } else { weight(input, *value) }, *time) - .unwrap(); - } - assert_eq!(actual.len(), expected.len(), "{query}"); - for (labels, state) in actual { - let reference = expected[&labels].snapshot_accumulator(); - assert_eq!( - evaluations(state.as_ref(), family), - evaluations(reference.as_ref(), family), - "{query}: {labels:?}" - ); - } - Ok(label) -} - -// Every raw-input summary Planner composes compiles over raw data sample rows, -// and each population's estimates equal feeding its kernel sample by sample. -#[test] -fn raw_data_sample_summaries_compile_and_match_their_kernels() { - let exact = AccuracyTarget::exact(); - let sketch = AccuracyTarget::max_error(0.02); - // A UnivMon sized for a distinct count at ε = 0.02 holds a heap of - // millions of keys on each of its 19 layers, past the default 64 MiB. - let distinct = AccuracyTarget::max_error(0.2).failure_probability(0.1); - let queries = [ - ("sum_over_time(m[5m])", &exact), - ("count_over_time(m[5m])", &exact), - ("min_over_time(m[5m])", &exact), - ("max_over_time(m[5m])", &exact), - ("rate(m[5m])", &exact), - ("increase(m[5m])", &exact), - ("sum by (service) (sum_over_time(m[5m]))", &exact), - ("sum by (service) (rate(m[5m]))", &exact), - ("topk(2, sum_over_time(m[5m]))", &exact), - ("quantile_over_time(0.9, m[5m])", &sketch), - ("sum by (service) (quantile_over_time(0.9, m[5m]))", &sketch), - ("quantile by (service) (0.9, m)", &sketch), - ("distinct_over_time(m[5m])", &distinct), - ("count(m)", &sketch), - ("topk(2, m)", &sketch), - ( - "topk(2, sum by (service) (count_over_time(m[5m])))", - &sketch, - ), - ("topk(2, sum_over_time(m[5m]))", &sketch), - ("topk by (service) (2, sum_over_time(m[5m]))", &sketch), - ]; - let rows = samples(); - let mut families = BTreeSet::new(); - let mut unsupported = BTreeSet::new(); - let mut checked = BTreeMap::new(); - for (query, accuracy) in queries { - for candidate in candidates(query, accuracy.clone()) { - let dag = compile_at_ingestion_physical_asap_dag(&candidate).unwrap(); - for (source, root) in raw_summaries(&dag) { - match check(query, &dag, source, root, &rows) { - Ok(family) => { - families.insert(family); - *checked.entry(query).or_insert(0) += 1; - } - Err(family) => { - unsupported.insert(family); - } - } - } - } - } - println!("checked {checked:?}; families {families:?}; without native state {unsupported:?}"); - for query in [ - "topk by (service) (2, sum_over_time(m[5m]))", - "quantile by (service) (0.9, m)", - "distinct_over_time(m[5m])", - ] { - assert!( - checked.contains_key(query), - "{query} has no checked raw summary" - ); - } - for family in [ - "Sum", - "Count", - "Min", - "Max", - "Rate", - "Increase", - "Kll", - "DDSketch", - "Hll", - "Kmv", - "UnivMon", - // The whole-expression heaps over raw data samples. - "CmsWithHeap", - "CountSketchWithHeap", - ] { - assert!( - families.contains(family), - "no {family} fixture: {families:?}" - ); - } -} - -/// Replace the raw summary of `sum by (service) (sum_over_time(m{instance!="none"}[5m]))` with -/// another update, keeping its raw input and reduction. -fn grouped_raw_summary(family: FieldDataType, input: SummaryUpdate) -> (PhysicalASAPDAG, u64, u64) { - // The matcher, true of every sample, makes the `Scan` read `instance`. - let candidate = candidates( - r#"sum by (service) (sum_over_time(m{instance!="none"}[5m]))"#, - AccuracyTarget::exact(), - ) - .pop() - .unwrap(); - let mut dag = compile_at_ingestion_physical_asap_dag(&candidate).unwrap(); - let (source, root) = raw_summaries(&dag)[0]; - let node = dag.nodes.iter_mut().find(|n| n.id as u64 == root).unwrap(); - let PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryAgg { - family: old, - input: update, - .. - }) = &mut node.payload - else { - unreachable!() - }; - for field in &mut node.output_schema.fields { - if field.dtype == *old { - field.dtype = family.clone(); - } - } - *old = family; - *update = input; - let schema = node.output_schema.clone(); - for edge in dag.edges.iter_mut().filter(|e| e.producer as u64 == root) { - *edge.intermediate_schema_mut() = schema.clone(); - } - (dag, source, root) -} - -// Keyed heaps over raw data samples resolve items from the series label set and -// estimate each item's exact total; invalid weight contracts do not compile. -#[test] -fn raw_data_sample_heaps_resolve_items_from_labels() { - use asap_types::ir::schema::GroupingStrategy::PerSubpopulationInstance; - use asap_types::ir::schema::{NonNegativeWeightProof, SketchKind, SketchParams, WeightDomain}; - let heap = |algorithm, params| { - FieldDataType::Sketch(SketchKind::new(algorithm, params), PerSubpopulationInstance) - }; - let cms = heap( - SketchAlgorithm::CmsWithHeap, - SketchParams::CmsWithHeap { - width: 64, - depth: 3, - heap_size: 8, - }, - ); - let count_sketch = heap( - SketchAlgorithm::CountSketchWithHeap, - SketchParams::CountSketchWithHeap { - width: 64, - depth: 3, - heap_size: 8, - }, - ); - // The series label set, read from the `Scan`'s series identity column. - let identity = SummaryUpdate { - item: Some(SummaryInputExpr::Column(ColumnRef::Named( - promql_rows::SERIES_IDENTITY_COLUMN.into(), - ))), - weight: SummaryInputExpr::Constant(1.0), - weight_domain: WeightDomain::NonNegative { - proof: NonNegativeWeightProof::UnitCount, - }, - }; - let by_instance = SummaryUpdate { - item: Some(SummaryInputExpr::Tuple(vec![SummaryInputExpr::Column( - ColumnRef::Named("instance".into()), - )])), - weight: SummaryInputExpr::Column(ColumnRef::SampleValue), - weight_domain: WeightDomain::UnknownOrSigned, - }; - let rows = samples(); - for (family, input) in [ - (cms.clone(), identity.clone()), - (count_sketch.clone(), identity), - (count_sketch, by_instance.clone()), - ] { - let unit_count = input.weight == SummaryInputExpr::Constant(1.0); - let (dag, source, root) = grouped_raw_summary(family, input); - assert!(check("heap", &dag, source, root, &rows).is_ok()); - // A unit count adds the constant 1 per row; no column of ones is - // projected for it. - let compiled = serde_json::to_value(compile_summary(&dag, root).unwrap()).unwrap(); - assert_eq!( - compiled - .to_string() - .contains(r#""row_weight":{"Constant":1}"#), - unit_count, - "{compiled}" - ); - } - let signed_cms = grouped_raw_summary(cms, by_instance.clone()); - assert!(compile_summary(&signed_cms.0, signed_cms.2).is_err()); -} - -// End to end: `topk(2, count by (service) (m))` at ε = 0.02 has a Count-Min -// heap candidate whose update is the unit count (`Constant(1.0)`). Placed at -// ingestion, its raw-data-sample precompute adds the constant 1 per sample to each -// service's count, with no column of ones projected, and each heap reads the -// exact per-service sample counts of the fixture. -#[test] -fn count_ranked_heap_precompute_adds_a_constant_row_weight() { - let query = "topk(2, count by (service) (m))"; - let rows = samples(); - let mut checked = 0; - for candidate in candidates(query, AccuracyTarget::max_error(0.02)) { - let dag = compile_at_ingestion_physical_asap_dag(&candidate).unwrap(); - for (source, root) in raw_summaries(&dag) { - let node = dag.nodes.iter().find(|n| n.id as u64 == root).unwrap(); - let PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), - input, - .. - }) = &node.payload - else { - continue; - }; - if kind.algorithm() != &SketchAlgorithm::CmsWithHeap { - continue; - } - assert_eq!(input.weight, SummaryInputExpr::Constant(1.0)); - let compiled = serde_json::to_value(compile_summary(&dag, root).unwrap()) - .unwrap() - .to_string(); - assert!( - compiled.contains(r#""row_weight":{"Constant":1}"#), - "{compiled}" - ); - assert_eq!( - check(query, &dag, source, root, &rows), - Ok("CmsWithHeap".into()) - ); - checked += 1; - } - } - assert!(checked > 0, "{query} has no Count-Min heap candidate"); -} diff --git a/crates/integration-tests/tests/production_rollup_workloads.rs b/crates/integration-tests/tests/production_rollup_workloads.rs index c44f7724a..5cad992ea 100644 --- a/crates/integration-tests/tests/production_rollup_workloads.rs +++ b/crates/integration-tests/tests/production_rollup_workloads.rs @@ -181,96 +181,6 @@ async fn production_sql_distinct_region_service_rollup() { check(&sql(&queries, accuracy).await, 2, 1, "Kmv", None); } -#[test] -// The earlier controlled fixture omitted raw retention. This production -// workload explicitly holds one current sample for each of its 10,000 series. -fn production_promql_compares_rollup_with_exact() { - let accuracy = AccuracyTarget::max_error(0.01) - .failure_probability(0.001) - .accuracy_metric(AccuracyMetric::Rank); - let roots: Vec<_> = [ - "quantile by (host, job) (0.5, data)", - "quantile by (job) (0.5, data)", - ] - .iter() - .map(|query| { - let root = lower_promql(query, accuracy.clone()).unwrap(); - QueryRoot::Operator(promql_rows::with_series_identity(&root).unwrap()) - }) - .collect(); - let data: DataWorkload = serde_json::from_value(serde_json::json!({"arrival":{"ingesting":{"rows_per_second":666.67}},"sources":{"data":{"series_count":10000,"metric_type":"gauge","sample_interval":15000,"rows_held":10000}}})).unwrap(); - let entries = [entry(&accuracy, 10000), entry(&accuracy, 10000)]; - let separate = independent_kll_cost(roots.clone(), &entries, &data); - println!("independent KLL cost={separate}"); - check( - &production(roots, &entries, &data), - 2, - 1, - "Kll", - Some(separate), - ); -} - -// Price the same workload with independently built, unmaterialized KLL states. -fn independent_kll_cost( - roots: Vec, - entries: &[QueryWorkloadEntry], - data: &DataWorkload, -) -> f64 { - use asap_logical_asap_optimization::test_support::{ - compose_logical_candidate, generate_per_computation_candidates, - }; - use asap_types::stages::HowToComputeAggregateMeasure; - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&roots), - ) - .unwrap(); - let choices: Vec<_> = inventory - .targets - .iter() - .map(|target| { - target - .ways_to_compute - .iter() - .position(|c| { - matches!( - c, - HowToComputeAggregateMeasure::Summary( - asap_types::ir::schema::SketchAlgorithm::Kll - ) - ) - }) - .unwrap() - }) - .collect(); - let roots: Vec<_> = compose_logical_candidate(&inventory, &choices).unwrap(); - // Composed outside the memo: sized as Pass 1 sizes a candidate. - let targets: Vec<_> = entries - .iter() - .map(|entry| entry.requirements.accuracy.clone()) - .collect(); - let roots = - asap_accuracy::sizing::with_summaries_sized_for_targets(&roots, &targets, data).unwrap(); - let mut settings = CostInputs::default(); - settings.cost_model = CostModel::UsageOverTime(ResourceUnitCost::aws_fargate_us_east_1( - settings.resource_speed, - )); - let models = PlanningModels::builtin().with_cost_inputs(&settings); - let plan = asap_physical_asap_optimization::test_support::optimize_physical_asap_chosen( - "independent", - &roots, - entries, - data, - models.deployment_capabilities.query_time_results_retained, - &mut |_, _, _| Some(asap_types::physical::Materialization::NotMaterialized), - ) - .unwrap() - .unwrap(); - let result = - asap_devtools::memo_pricing::reference_select(&[plan], entries, data, models).unwrap(); - result.costs[&result.selected].cost_per_second.unwrap() -} - // Exercise multi-input operators through the production route under more than // three heterogeneous query schedules, not just the local pricing kernel. #[test] diff --git a/crates/integration-tests/tests/promql_numeric_regressions.rs b/crates/integration-tests/tests/promql_numeric_regressions.rs index 0e8469f03..5626606a4 100644 --- a/crates/integration-tests/tests/promql_numeric_regressions.rs +++ b/crates/integration-tests/tests/promql_numeric_regressions.rs @@ -120,7 +120,7 @@ fn checked_ratio_must_not_certify_cross_zero_interpolation() { ) .unwrap(); let node = explicit_summary_candidate(root, asap_types::ir::schema::SketchAlgorithm::DDSketch); - assert_eq!(OperatorNode::reachable(&node).iter().filter(|node| matches!(node.asap(), Some(ASAPOp::SummaryAgg { family: FieldDataType::Sketch(kind, _), .. }) if *kind.algorithm() == asap_types::ir::schema::SketchAlgorithm::DDSketch)).count(), 2, + assert_eq!(OperatorNode::reachable(&node).iter().filter(|node| matches!(node.asap(), Some(ASAPOp::SummaryAgg { family: FieldDataType::Sketch(kind, _), .. }) if *kind == asap_types::ir::schema::SketchAlgorithm::DDSketch)).count(), 2, "the signed interpolation counterexample requires two explicit DDSketch builds"); assert!( matches!(node.operator, Operator::NonASAP(NonASAPOp::BinaryOp { .. })) @@ -273,7 +273,7 @@ fn sketch_counts_use_unit_weights_and_signed_sums_keep_value_weights() { .find_map(|node| { let state = heap(node)?; let (family, _, _) = aggregate(&state); - matches!(family, FieldDataType::Sketch(kind, _) if kind.algorithm() == &wanted) + matches!(family, FieldDataType::Sketch(kind, _) if kind == &wanted) .then_some((node, state)) }) .expect("weighted sketch candidate"); @@ -295,7 +295,7 @@ fn sketch_counts_use_unit_weights_and_signed_sums_keep_value_weights() { // so plan selection rejects it. for state in candidates.iter().filter_map(heap) { let (family, update, _) = aggregate(&state); - if matches!(family, FieldDataType::Sketch(kind, _) if kind.algorithm() == &SketchAlgorithm::CmsWithHeap) + if matches!(family, FieldDataType::Sketch(kind, _) if kind == &SketchAlgorithm::CmsWithHeap) { assert!(!matches!( update.weight_domain, diff --git a/crates/integration-tests/tests/promql_to_post_asap.rs b/crates/integration-tests/tests/promql_to_post_asap.rs index a2e6670d3..398ee91de 100644 --- a/crates/integration-tests/tests/promql_to_post_asap.rs +++ b/crates/integration-tests/tests/promql_to_post_asap.rs @@ -6,29 +6,23 @@ //! the summary-bound shape node by node, including the family `(Kind, //! Params)` committed on each edge's schema. -use asap_types::stages::HowToComputeAggregateMeasure; use std::rc::Rc; use asap_integration_tests::fixtures::lower_promql; -use asap_integration_tests::physical_export::{ - at_ingestion, at_ingestion_physical_asap_dag, physical_asap_dag, -}; +use asap_integration_tests::physical_export::physical_asap_dag; use asap_integration_tests::planning::{ logical_candidates, logical_candidates_ignoring_guarantees, }; -use asap_plan_selection::reference_pricer::WholePlanSelection as Selection; use asap_types::ir::operator::operator_properties::Reduction; -use asap_types::ir::physical_export::PhysicalASAPOperatorPayload; -use asap_types::ir::properties::AccuracyMetric; + use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::scalar::expr_ir::ColumnRef; use asap_types::ir::schema::DataType; use asap_types::ir::schema::{ EntityIdentity, ExactKind, ExactParams, FieldDataType, GroupingStrategy, Schema, - SketchAlgorithm, SketchKind, SketchParams, SketchStatistic, SummaryInputExpr, SummaryUpdate, + SketchAlgorithm, SketchStatistic, SummaryInputExpr, SummaryUpdate, }; -use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode, QueryRoot, ScalarExpr}; -use asap_types::physical::PhysicalWorkloadPlan; +use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode, ScalarExpr}; /// The first logical candidate (Stage 1, in enumeration order) of `root` as /// a query at `accuracy` that uses an ASAP operator, or the plain exact one @@ -52,7 +46,7 @@ fn uses_sketch(node: &OperatorNode, algorithm: &SketchAlgorithm) -> bool { Operator::ASAP(ASAPOp::SummaryAgg { family: FieldDataType::Sketch(kind, _), .. - }) if kind.algorithm() == algorithm => true, + }) if kind == algorithm => true, _ => node .children() .iter() @@ -254,90 +248,6 @@ fn promql_binary_arithmetic_preserves_nested_structure_and_vector_matching() { } } -#[test] -fn planner_only_e2e_temporal_topk_preserves_query_update_and_evaluation_contract() { - // Self-contained Planner E2E: each case starts from PromQL text and ends - // at the post-ASAP summary DAG. No controller/backend types, - // fixtures, configuration, or runtime are involved. - let cases = [ - ( - "topk by (service) (5, count_over_time(requests[1m]))", - SummaryInputExpr::Constant(1.0), - SketchAlgorithm::CmsWithHeap, - vec![ColumnRef::Named("service".into())], - ), - ( - "topk(5, sum_over_time(requests[1m]))", - SummaryInputExpr::Column(ColumnRef::SampleValue), - SketchAlgorithm::CountSketchWithHeap, - vec![], - ), - ]; - for (source, expected_update, expected_family, excluded_labels) in cases { - let pre = lower_promql( - source, - AccuracyTarget::max_error(0.01).failure_probability(0.01), - ) - .expect("lower temporal Top-K"); - let candidates = heap_topk_candidates_using( - &pre, - AccuracyTarget::max_error(0.01).failure_probability(0.01), - expected_family, - ); - assert!( - !candidates.is_empty(), - "heap-backed temporal Top-K candidate" - ); - // The search archive may visit tighter local error allocations first. - // Every visited configuration must preserve the same update contract. - for candidate in candidates { - let Some(ASAPOp::SummaryEstimate { - summary_input, - query: SketchStatistic::TopK { k, .. }, - }) = candidate.asap() - else { - panic!("expected Top-K estimate, got {:?}", candidate.operator) - }; - assert_eq!( - *k, 5, - "the requested Top-K cardinality must survive binding" - ); - let Some(ASAPOp::SummaryAgg { - input: state_input, - family, - child, - .. - }) = summary_input.asap() - else { - panic!("expected structured Top-K state input") - }; - let FieldDataType::Sketch(kind, _) = family else { - panic!("expected a heap-backed sketch family, got {family:?}") - }; - assert_eq!(kind.algorithm(), &expected_family); - let heap_size = match kind.params() { - SketchParams::CmsWithHeap { heap_size, .. } - | SketchParams::CountSketchWithHeap { heap_size, .. } => *heap_size, - params => panic!("expected heap-bearing Top-K parameters, got {params:?}"), - }; - assert!( - heap_size >= 100u32.max(*k as u32), - "a tighter local error budget may increase, never reduce, capacity" - ); - assert_eq!( - state_input.item.as_ref(), - Some(&SummaryInputExpr::EntityIdentity( - EntityIdentity::PromqlLabelSet { - excluding: excluded_labels.clone() - } - )) - ); - assert_eq!(state_input.weight, expected_update); - assert!(!child.contains_asap()); - } - } -} - /// Execute the ungrouped temporal TopK subset with exact state. This tests /// the emitted update contract, not sketch approximation or backend execution. fn execute_topk_reference(plan: &OperatorNode) -> Vec<(String, f64)> { @@ -533,16 +443,7 @@ fn promql_quantile_of_rate_binds_kll_over_rate_accumulator() { }; assert_eq!( family, - &FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: 269, - failure_probability: None - } - ), - GroupingStrategy::default() - ) + &FieldDataType::Sketch(SketchAlgorithm::Kll, GroupingStrategy::default()) ); assert_eq!(input, &SummaryUpdate::column(ColumnRef::SampleValue)); assert_eq!( @@ -552,16 +453,7 @@ fn promql_quantile_of_rate_binds_kll_over_rate_accumulator() { ); assert_eq!( dtype(&summary_input.schema, "state"), - &FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: 269, - failure_probability: None - } - ), - GroupingStrategy::default() - ) + &FieldDataType::Sketch(SketchAlgorithm::Kll, GroupingStrategy::default()) ); let Some(ASAPOp::FinalizeExactAccumulator { child }) = child.asap() else { @@ -622,255 +514,6 @@ fn promql_quantile_of_rate_binds_kll_over_rate_accumulator() { ); } -#[test] -fn nested_summary_explicitly_finalizes_exact_child_at_ingestion_time() { - // Real workload selection must expose the state-to-value edge; an outer - // sketch must not interpret exact accumulator bytes as input samples. - let pre = lower_promql( - "quantile(0.9, sum_over_time(m[1m]))", - AccuracyTarget::max_error(0.05), - ) - .unwrap(); - // The KLL candidate over the inner exact Sum accumulator. - let plan = candidates_using(&pre, AccuracyTarget::max_error(0.05), SketchAlgorithm::Kll) - .into_iter() - .find(|plan| { - matches!(plan.asap(), Some(ASAPOp::SummaryEstimate { summary_input, .. }) - if matches!(summary_input.asap(), Some(ASAPOp::SummaryAgg { child, .. }) - if matches!(child.asap(), Some(ASAPOp::FinalizeExactAccumulator { .. })))) - }) - .expect("a KLL over the finalized Sum accumulator"); - // Stored timings are gone: time the plan with its outer summary - // maintained and read the timed copy. - let timed_plan = at_ingestion(&plan); - let Some(ASAPOp::SummaryEstimate { summary_input, .. }) = timed_plan.asap() else { - panic!("expected selected quantile summary"); - }; - let Some(ASAPOp::SummaryAgg { child, .. }) = summary_input.asap() else { - panic!("expected maintained outer summary"); - }; - let Some(ASAPOp::FinalizeExactAccumulator { child: source }) = child.asap() else { - panic!( - "missing explicit accumulator finalization: {:?}", - child.operator - ); - }; - assert_eq!( - child.timing, - Some(asap_types::ir::properties::ExecutionTiming::IngestionTime) - ); - assert!(matches!( - source.asap(), - Some(ASAPOp::SummaryAgg { - family: FieldDataType::ExactAggregate(ExactKind::Sum, _), - .. - }) - )); - assert!(child - .schema - .fields - .iter() - .all(|field| matches!(field.dtype, FieldDataType::Plain(_)))); - assert!(child - .schema - .fields - .iter() - .any(|field| matches!(field.dtype, FieldDataType::Plain(DataType::Float64)))); - // Explicit boundary is a valid post-ASAP DAG. - physical_asap_dag(&plan); -} - -#[test] -fn physical_node_owns_phase_independently_of_binary_payload() { - use asap_types::ir::properties::ExecutionTiming; - // The one-selector case (both operands over the same series, so the - // BinaryOp may run at ingestion) is `arithmetic_over_one_selector_executes` - // in materialization_e2e.rs, ignored until #580. - let query = "sum_over_time(m[1m]) + sum_over_time(n[1m])"; - let expected = ExecutionTiming::QueryTime; - let input = lower_promql(query, AccuracyTarget::max_error(0.05)).unwrap(); - let plan = realize(&input, AccuracyTarget::max_error(0.05)); - let dag = at_ingestion_physical_asap_dag(&plan); - let node = dag - .nodes - .iter() - .find(|node| { - matches!( - node.payload, - PhysicalASAPOperatorPayload::NonASAP(NonASAPOp::BinaryOp { .. }) - ) - }) - .unwrap(); - assert_eq!(node.output_state.timing, expected); - let wire = serde_json::to_value(&node.payload).unwrap(); - assert!(wire.get("timing").is_none()); - let mut obsolete = wire.clone(); - obsolete["timing"] = serde_json::json!(expected.as_str()); - assert!(serde_json::from_value::(obsolete).is_err()); - let restored: PhysicalASAPOperatorPayload = serde_json::from_value(wire).unwrap(); - assert_eq!(restored, node.payload); -} - -/// `query` lowered at `planned` and selected at `accuracy`: the physical -/// candidates whose two summaries are both DDSketch, and plan selection over -/// all of them. -fn ddsketch_ratio_selection( - query: &str, - planned: AccuracyTarget, - accuracy: AccuracyTarget, - non_negative: bool, -) -> (Vec, Selection) { - let root = lower_promql(query, planned).unwrap(); - let mut data = asap_integration_tests::planning::declared_data(&root); - for facts in data.sources.values_mut() { - facts.values_non_negative = non_negative; - } - use asap_logical_asap_optimization::test_support::generate_per_computation_candidates; - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(root), - ]), - ) - .unwrap(); - let exact = vec![0; inventory.targets.len()]; - let dds = inventory - .targets - .iter() - .map(|target| { - target - .ways_to_compute - .iter() - .position(|candidate| { - *candidate == HowToComputeAggregateMeasure::Summary(SketchAlgorithm::DDSketch) - }) - .expect("quantile supports DDS") - }) - .collect(); - let entries = [asap_types::workload::QueryWorkloadEntry { - query: Default::default(), - requirements: asap_types::workload::QueryRequirements { - accuracy, - max_latency_ms: None, - }, - predictability: Default::default(), - recurrence: Default::default(), - range_evaluation: None, - }]; - // Pass 1 choices composed and implemented by Stage 2; one query, so no - // sharing rule applies. - use asap_accuracy::propagation::recompose_guarantees; - use asap_logical_asap_optimization::test_support::compose_logical_candidate; - let models = asap_planner::PlanningModels::builtin(); - let mut physical = Vec::new(); - for choice in [exact, dds] { - let roots: Vec<_> = compose_logical_candidate(&inventory, &choice).unwrap(); - let roots = recompose_guarantees(&roots).unwrap(); - physical.extend( - asap_physical_asap_optimization::optimize_physical_asap( - &format!("L{}", physical.len() + 1), - &roots, - &entries, - &data, - models.deployment_capabilities.query_time_results_retained, - ) - .unwrap(), - ); - } - for (index, plan) in physical.iter_mut().enumerate() { - plan.id = format!("fixture-{index}"); - } - let selection = - asap_devtools::memo_pricing::reference_select(&physical, &entries, &data, models).unwrap(); - let ratios = physical - .into_iter() - .filter(|candidate| { - let sketches: Vec<_> = candidate - .dag - .nodes - .iter() - .filter_map(|node| match &node.payload { - PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), - .. - }) => Some(*kind.algorithm()), - _ => None, - }) - .collect(); - sketches == [SketchAlgorithm::DDSketch, SketchAlgorithm::DDSketch] - }) - .collect(); - (ratios, selection) -} - -/// Declared nonnegative inputs permit formal ratio admission; an unknown -/// sign does not. The same explicitly sized DDS fixture is also evaluated -/// numerically on all-positive and all-negative samples without claiming -/// a general bound for arbitrary signed interpolation. -#[test] -fn ddsketch_ratio_checks_sign_contract_and_one_sided_sketch_evaluations() { - for sign in [-1., 1.] { - let target = AccuracyTarget::max_error(0.05).accuracy_metric(AccuracyMetric::Relative); - let (ratios, selection) = ddsketch_ratio_selection( - "quantile_over_time(0.9, data[5m]) / quantile_over_time(0.5, data[5m])", - AccuracyTarget::max_error(0.01), - target.clone(), - sign > 0.0, - ); - let accepted: Vec<_> = ratios - .iter() - .filter(|candidate| selection.costs.contains_key(&candidate.id)) - .collect(); - assert!( - !ratios.is_empty(), - "explicit DDS fixture exists regardless of admission" - ); - assert_eq!(!accepted.is_empty(), sign > 0.0, "{selection:?}"); - // Both explicit operands use the planner's composed-budget alpha. - let alphas: Vec = ratios[0] - .dag - .nodes - .iter() - .filter_map(|node| match &node.payload { - PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), - .. - }) => match kind.params() { - SketchParams::DDSketch { alpha } => Some(*alpha), - _ => None, - }, - _ => None, - }) - .collect(); - assert_eq!(alphas.len(), 2); - assert_eq!(alphas[0], alphas[1]); - let alpha = alphas[0]; - let bound = target.max_error; - for values in [ - vec![sign; 30], - (1..=100).map(|i| sign * i as f64).collect(), - vec![sign, sign, sign, sign * 30., sign * 100.], - ] { - let mut sorted = values.clone(); - sorted.sort_by(f64::total_cmp); - let exact = |q: f64| { - let r = q * (sorted.len() - 1) as f64; - sorted[r.floor() as usize] * (1. - r.fract()) - + sorted[r.ceil() as usize] * r.fract() - }; - let mut sketch = asap_sketchlib::DdSketch::new(alpha); - for v in values { - sketch.try_add(&v).unwrap(); - } - // PromQL's quantile: interpolated between neighbouring order - // statistics. - let interpolated = |q: f64| sketch.quantile_interpolated(q).unwrap().unwrap(); - let want = exact(0.9) / exact(0.5); - let got = interpolated(0.9) / interpolated(0.5); - assert!((got - want).abs() / want.abs() <= bound + 1e-12); - } - } -} - // Every `without` aggregation candidate exports a valid DAG: its summary state // column carries the family instead of the evaluation's Float64 value. #[test] diff --git a/crates/integration-tests/tests/range_evaluation.rs b/crates/integration-tests/tests/range_evaluation.rs deleted file mode 100644 index 2f97eeb72..000000000 --- a/crates/integration-tests/tests/range_evaluation.rs +++ /dev/null @@ -1,423 +0,0 @@ -//! #757: a query answered as a range request (`RangeEvaluation`), e.g. a -//! dashboard panel that evaluates the last hour at 1-minute steps on every -//! refresh. Stage 1 and Stage 2 keep tumbling-window widths on the step grid, and -//! Stage 3 prices `span_ms / step_ms + 1` evaluations per request, credits -//! the tumbling windows kept across steps and requests, and retains kept tumbling windows for -//! the span plus the window. - -use crate::planner_layering_common; - -use asap_logical_asap_optimization::DeploymentCapabilities; -use asap_plan_selection::RejectionCause; -use asap_types::ir::physical_export::PhysicalASAPOperatorPayload; -use asap_types::ir::ASAPOp; -use asap_types::physical::{Materialization, Medium}; -use asap_types::workload::{ - DataArrival, DataWorkload, DurationMs, PlanningWorkload, Predictability, Query, QueryLanguage, - QueryRequirements, QueryWorkload, RangeEvaluation, RepeatedDemand, RepeatingEntry, - RepetitionInterval, SourceFacts, -}; -use planner_layering_common::*; - -/// The dashboard's range: the last hour at 1-minute steps, 61 evaluations -/// per request. -const LAST_HOUR_BY_MINUTE: RangeEvaluation = RangeEvaluation { - step_ms: 60_000, - span_ms: 3_600_000, -}; - -/// Price explicitly named computations under the same models as production. -/// This fixture controls its tumbling-window width; a timed MILP archive is not a catalog -/// of every legal materialization or a stable source of P1/P2 identifiers. -fn run_explicit_for(workload: &PlanningWorkload, deployment: &DeploymentCapabilities) -> Run { - use asap_logical_asap_optimization::test_support::{ - compose_logical_candidate, generate_per_computation_candidates, - }; - use asap_logical_asap_optimization::test_support::{ - tumbling_window_candidates, window_composition_pass2_candidates, - }; - use asap_types::ir::schema::SketchAlgorithm; - use asap_types::stages::HowToComputeAggregateMeasure; - let roots = lower_promql(workload); - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&roots), - ) - .unwrap(); - assert_eq!(inventory.targets.len(), 1); - let compose = |choice| compose_logical_candidate(&inventory, &[choice]).unwrap(); - let mut candidates = vec![compose(0)]; - if let Some(choice) = inventory.targets[0] - .ways_to_compute - .iter() - .position(|candidate| { - matches!( - candidate, - HowToComputeAggregateMeasure::Summary(SketchAlgorithm::Kll) - ) - }) - { - candidates.push(compose(choice)); - } - let build = candidates.last().unwrap(); - let grids = workload - .query_workload - .entries() - .map(|entry| entry.evaluation_grid_ms()) - .collect::>(); - let tumbling_windows = tumbling_window_candidates(build, &grids); - assert_eq!( - tumbling_windows.len(), - 1, - "one explicit tumbling window computation" - ); - candidates.push( - window_composition_pass2_candidates(build, &tumbling_windows) - .unwrap() - .remove(0) - .roots - .unwrap(), - ); - let settings = asap_cost::CostInputs::default(); - run_explicit_candidates( - workload, - roots, - candidates, - asap_planner::PlanningModels::builtin() - .with_cost_inputs(&settings) - .with_deployment_capabilities(deployment), - ) -} - -fn run_explicit(workload: &PlanningWorkload) -> Run { - run_explicit_for(workload, &DeploymentCapabilities::default()) -} - -/// Pattern B (#509 Example 3: `quantile_over_time(0.99, latency_ms[5m])` -/// every minute over 1M series) with each request answered as `range`. -fn pattern_b_with(range: Option) -> PlanningWorkload { - let mut workload = pattern_b(); - for entry in workload - .query_workload - .repeating_queries - .iter_mut() - .flatten() - { - entry.range_evaluation = range; - } - workload -} - -/// A deployment that keeps query-time results across evaluations (B3). -fn keeping_query_time_results() -> DeploymentCapabilities { - DeploymentCapabilities { - query_time_results_retained: true, - ..DeploymentCapabilities::default() - } -} - -/// `query` every `interval_ms` over 1M series of `m` sampled every 1.5 s, -/// arriving continuously, exact, answered as `range`. -fn exact_panel(query: &str, interval_ms: u32, range: Option) -> PlanningWorkload { - PlanningWorkload { - query_workload: QueryWorkload { - language: QueryLanguage::PromQL, - query_batch: None, - repeating_queries: Some(vec![RepeatingEntry { - query: Query(query.into()), - demand: RepeatedDemand::FixedInterval(RepetitionInterval(interval_ms)), - requirements: QueryRequirements::default(), - predictability: Predictability::Predictable { known_at: None }, - range_evaluation: range, - }]), - }, - data_workload: Some(DataWorkload { - arrival: DataArrival::Ingesting { - rows_per_second: SHARED_ROWS_PER_SECOND, - }, - sources: [( - "m".to_string(), - SourceFacts { - series_count: Some(1_000_000), - sample_interval: Some(DurationMs(1_500)), - ..SourceFacts::default() - }, - )] - .into(), - }), - } -} - -fn cost(run: &Run, id: &str) -> f64 { - run.cost(id).unwrap_or_else(|| panic!("{id} is not priced")) -} - -fn close(actual: f64, expected: f64) -> bool { - (actual - expected).abs() <= 1e-6 * expected.abs().max(1.0) -} - -/// Each physical candidate's tumbling-window chains: (tumbling-window width, materialization). -fn tumbling_window_chains(run: &Run) -> Vec<(String, u64, Materialization)> { - run.physical - .iter() - .flat_map(|p| { - p.dag().nodes.iter().filter_map(|n| match &n.payload { - PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms, - .. - }) => Some((p.id.clone(), *tumbling_window_width_ms, n.materialization)), - _ => None, - }) - }) - .collect() -} - -/// The candidate whose single tumbling-window chain is materialized as `wanted`. -fn with_chain(run: &Run, wanted: Materialization) -> String { - tumbling_window_chains(run) - .into_iter() - .find(|(_, _, m)| *m == wanted) - .unwrap_or_else(|| panic!("no candidate with a {wanted:?} chain")) - .0 -} - -/// (a) The issue's acceptance example: Pattern B as a dashboard over the -/// last hour at 1-minute steps prices 61 evaluations per request. Every -/// query-time computation runs at every step, so the whole-window KLL -/// (2.3625 → 144.1125 cost/s), the tumbling windows rebuilt per step (3.6112 → -/// 220.2852) and the exact plan (2.4576 → 149.9136) cost 61 times as much, -/// the memory each step holds while it runs included. The kept tumbling windows (B3) -/// are built once per request (one new 1-min tumbling window) but merged at every -/// step, and keep 64 older tumbling windows instead of 4 (1 h + 5 min of tumbling windows): -/// 436.59 → 6,987.05. B1 keeps 66 instead of 6: 654.54 → 7,202.31. The -/// whole-window KLL is still selected. -#[test] -fn stage3_plan_selection_prices_61_evaluations_per_dashboard_request() { - let deployment = keeping_query_time_results(); - let none = run_explicit_for(&pattern_b_with(None), &deployment); - let range = run_explicit_for(&pattern_b_with(Some(LAST_HOUR_BY_MINUTE)), &deployment); - assert_eq!(LAST_HOUR_BY_MINUTE.evaluations_per_request(), 61); - for (id, before, after) in [ - ("P1", 2.457_600_019_2, 149.913_601_171_2), - ("P2", 2.362_500_025_2, 144.112_501_537_2), - ("P3", 3.611_233_350_8, 220.285_234_398_8), - ("P3-m1", 654.542_718_733_8, 7_202.305_718_761_8), - ("P3-m2", 436.587_770_004_24, 6_987.053_970_131_44), - ] { - assert!(close(cost(&none, id), before), "{id}: {}", cost(&none, id)); - assert!(close(cost(&range, id), after), "{id}: {}", cost(&range, id)); - } - // Every query-time computation that is not kept runs once per step. - for id in ["P1", "P2", "P3"] { - assert!(close(cost(&range, id), 61.0 * cost(&none, id)), "{id}"); - } - // B3: 4 → 64 tumbling windows kept between requests; B1: 6 → 66 tumbling windows kept. - let keeps = |run: &Run, id: &str, bytes: &str| { - assert!( - run.selection.costs[id] - .per_node - .values() - .any(|n| n.detail.contains(&format!("; keeps {bytes} bytes"))), - "{id} keeps {bytes} bytes" - ) - }; - keeps(&none, "P3-m2", "3488000000"); - keeps(&range, "P3-m2", "55808000000"); - keeps(&none, "P3-m1", "5232000000"); - keeps(&range, "P3-m1", "57552000000"); - assert_eq!(range.selection.selected, "P2"); -} - -/// (b) For this explicit fixture, refreshing every minute at 30-second steps -/// supplies tumbling windows of -/// `gcd(5 min, 1 min, 30 s) = 30 s`, on the step grid, and Stage 2 keeps -/// them; without the range, the fixture uses 1 min tumbling windows. Both configurations -/// must divide the evaluation step; production also offers finer legal widths. -#[test] -fn explicit_tumbling_window_candidates_follow_the_evaluation_step_grid() { - let step_30s = RangeEvaluation { - step_ms: 30_000, - span_ms: 3_600_000, - }; - let range = run_explicit(&exact_panel("max_over_time(m[5m])", 60_000, Some(step_30s))); - let widths: Vec = tumbling_window_chains(&range) - .into_iter() - .map(|(_, w, _)| w) - .collect(); - assert!(!widths.is_empty(), "tumbling windows are offered"); - assert!(widths.iter().all(|&w| w == 30_000), "{widths:?}"); - assert!(range.pruned.is_empty(), "{:?}", range.pruned); - - let none = run_explicit(&exact_panel("max_over_time(m[5m])", 60_000, None)); - let widths: Vec = tumbling_window_chains(&none) - .into_iter() - .map(|(_, w, _)| w) - .collect(); - assert!( - !widths.is_empty() && widths.iter().all(|&w| w == 60_000), - "{widths:?}" - ); -} - -/// (c) A plan that keeps tumbling windows reports how long the runtime must keep -/// them: the window plus the range-evaluation span, 5 min + 1 h = -/// 3,900,000 ms, so the next refresh's earliest step finds its tumbling windows. -/// Without the range it is the window, 300,000 ms. -#[test] -fn stage3_plan_selection_reports_the_retention_of_kept_tumbling_windows() { - let deployment = keeping_query_time_results(); - let query = "max_over_time(m[5m])"; - let range = run_explicit_for( - &exact_panel(query, 60_000, Some(LAST_HOUR_BY_MINUTE)), - &deployment, - ); - let kept = with_chain( - &range, - Materialization::MatAtQueryKeptAcrossEvaluations(Medium::Memory), - ); - assert_eq!(range.selection.selected, kept); - let retention: Vec = range - .selection - .tumbling_window_retention_ms - .values() - .copied() - .collect(); - assert_eq!(retention, [3_900_000]); - - // Pattern B without a range selects the whole-window KLL, which keeps - // nothing; B1 reports its window when it is the plan (no raw data). - let none = run_explicit(&pattern_b_with(None)); - assert!(none.selection.tumbling_window_retention_ms.is_empty()); - let b1 = run_explicit_for( - &pattern_b_with(None), - &DeploymentCapabilities { - raw_data_retained: false, - ..DeploymentCapabilities::default() - }, - ); - let retention: Vec = b1 - .selection - .tumbling_window_retention_ms - .values() - .copied() - .collect(); - assert_eq!(retention, [PATTERN_B_WINDOW_MS]); -} - -/// (d) Without a range evaluation nothing changes: Pattern B's pinned -/// costs (#509 Example 3 and 4: whole-window KLL 2.3625, tumbling windows 3.6112, B3 -/// 436.59, B1 5.232 GB kept) are as before, and a range of one step per -/// request (`span_ms: 0`) prices exactly as no range. -#[test] -fn no_range_evaluation_leaves_every_price_unchanged() { - let deployment = keeping_query_time_results(); - let none = run_explicit_for(&pattern_b_with(None), &deployment); - assert!(close(cost(&none, "P2"), 2.362_500_025_2)); - assert!(close(cost(&none, "P3"), 3.611_233_350_8)); - assert!(close(cost(&none, "P3-m2"), 436.587_770_004_24)); - assert_eq!( - none.selection.costs["P3-m1"].usage.average_memory_bytes, - 5_232_101_733.337_067 - ); - let one_step = run_explicit_for( - &pattern_b_with(Some(RangeEvaluation { - step_ms: 60_000, - span_ms: 0, - })), - &deployment, - ); - assert_eq!(one_step.selection.costs, none.selection.costs); - assert_eq!(one_step.selection.selected, none.selection.selected); -} - -/// The example where declaring the range makes Stage 3 prefer sharing: -/// `max_over_time(m[5m])` every minute over 1M series. As one evaluation -/// per minute, rebuilding the window (P1, 20.8281 cost/s) beats the kept -/// tumbling windows (P2-m2, 32.2630: 224 MB of 56-byte per-series maxima kept) and the -/// tumbling windows maintained at ingestion (P2-m1, 46.2340). As a dashboard over the -/// last hour at 1-minute steps, rebuilding costs 61 times as much -/// (1,270.5151), while the kept tumbling windows add only the per-step merges and 1 h -/// of tumbling windows (460.0410), and B1 472.2760: the kept tumbling windows are selected, and without -/// query-time retention, the ingestion-time tumbling windows. -#[test] -fn stage3_plan_selection_prefers_kept_tumbling_windows_once_the_range_is_declared() { - let query = "max_over_time(m[5m])"; - let deployment = keeping_query_time_results(); - let none = run_explicit_for(&exact_panel(query, 60_000, None), &deployment); - let range = run_explicit_for( - &exact_panel(query, 60_000, Some(LAST_HOUR_BY_MINUTE)), - &deployment, - ); - let kept = with_chain( - &range, - Materialization::MatAtQueryKeptAcrossEvaluations(Medium::Memory), - ); - let ingestion = with_chain(&range, Materialization::MatAtIngestion(Medium::Memory)); - for (run, id, expected) in [ - (&none, "P1", 20.828_117), - (&none, kept.as_str(), 32.262_967), - (&none, ingestion.as_str(), 46.234_035), - (&range, "P1", 1_270.515_127), - (&range, kept.as_str(), 460.040_967), - (&range, ingestion.as_str(), 472.276_035), - ] { - assert!( - (cost(run, id) - expected).abs() < 1e-5, - "{id}: {}", - cost(run, id) - ); - } - assert_eq!(none.selection.selected, "P1"); - assert_eq!(range.selection.selected, kept); - let p1 = range - .selection - .rejected - .iter() - .find(|r| r.id == "P1") - .unwrap(); - assert_eq!(p1.cause, RejectionCause::Costlier); - - let without_retention = run_explicit(&exact_panel(query, 60_000, Some(LAST_HOUR_BY_MINUTE))); - assert_eq!(without_retention.selection.selected, ingestion); -} - -/// (e) A chain over an offset input reaches back its offset further: with -/// no raw data kept, `max_over_time(m[60m] offset 10m)` every 10 minutes -/// keeps six 10-minute tumbling windows ending 10 minutes before each evaluation, so -/// its tumbling windows must be kept for 10 min + 60 min = 4,200,000 ms, and the kept -/// memory Stage 3 prices covers that seventh tumbling window; without the offset the -/// retention is the window, 3,600,000 ms. -#[test] -fn an_offset_chain_is_retained_and_priced_for_its_offset() { - let no_raw_data = DeploymentCapabilities { - raw_data_retained: false, - ..DeploymentCapabilities::default() - }; - let plain = run_explicit_for( - &exact_panel("max_over_time(m[60m])", 600_000, None), - &no_raw_data, - ); - let offset = run_explicit_for( - &exact_panel("max_over_time(m[60m] offset 10m)", 600_000, None), - &no_raw_data, - ); - let retention = |run: &Run| -> Vec { - run.selection - .tumbling_window_retention_ms - .values() - .copied() - .collect() - }; - assert_eq!(retention(&plain), [3_600_000]); - assert_eq!(retention(&offset), [4_200_000]); - let memory = |run: &Run| { - run.selection.costs[&run.selection.selected] - .usage - .average_memory_bytes - }; - assert!( - memory(&offset) > memory(&plain), - "{} vs {}", - memory(&offset), - memory(&plain) - ); -} diff --git a/crates/integration-tests/tests/reuse_rules.rs b/crates/integration-tests/tests/reuse_rules.rs deleted file mode 100644 index 2c8d5a4ad..000000000 --- a/crates/integration-tests/tests/reuse_rules.rs +++ /dev/null @@ -1,589 +0,0 @@ -//! Reuse in Stage 1 (#680): the nested-sum merge (a Pass 1 candidate) is -//! selected over the nested sums and answers the same; the rollup and exact -//! top-k reuse candidates (Pass 2 memo rules) are selected once the finer -//! computation they read is materialized for all its readers, and answer as -//! the computation they replace. Selection prices fixed plans made of the -//! memo's candidates, so no assertion depends on a timed search. -use crate::physical_common; - -use std::collections::{BTreeMap, BTreeSet}; -use std::rc::Rc; - -use asap_devtools::planner_layering_examples::{shared_data_workload, SHARED_ROWS_PER_SECOND}; -use asap_executor::physical_planner::{compile, promql_rows}; -use asap_executor::runtime::Scope; -use asap_executor::values::Value; -use asap_integration_tests::fixtures::lower_promql; -use asap_integration_tests::planning::{ - cheapest_of, compose_memo_plan, expanded_memo, select_among, -}; -use asap_types::ir::physical_export::compile_physical_asap_dag; -use asap_types::ir::properties::AccuracyTarget; -use asap_types::ir::{ - apply_materialization_timings, ASAPOp, MaterializationAssignment, NodeToTimedNodeCache, - NonASAPOp, OperatorNode, QueryRoot, -}; -use asap_types::physical::Materialization; -use asap_types::stages::{root_groups, MemoGroupId}; -use asap_types::stages::{Memo, OriginOfCandidateForMemoGroup}; -use asap_types::workload::{ - DataArrival, DataWorkload, QueryRecurrence, QueryWorkloadEntry, RepeatedDemand, - RepetitionInterval, -}; - -const EVALUATION_MS: i64 = 300_000; - -/// `queries` lowered at `accuracy`, with series identity, as the planner -/// receives them. -fn roots(queries: &[(&str, AccuracyTarget)]) -> Vec { - queries - .iter() - .map(|(query, accuracy)| { - let root = lower_promql(query, accuracy.clone()).unwrap(); - let root = promql_rows::with_series_identity(&root).unwrap(); - QueryRoot::Operator(root) - }) - .collect() -} - -fn exact_queries<'a>(queries: &[&'a str]) -> Vec<(&'a str, AccuracyTarget)> { - queries - .iter() - .map(|query| (*query, AccuracyTarget::exact())) - .collect() -} - -/// The memo Stage 1 builds for `queries`, each with `recurrence`. -fn memo(queries: &[(&str, AccuracyTarget)], recurrence: QueryRecurrence) -> Memo { - expanded_memo(roots(queries), &vec![exact(recurrence); queries.len()]) -} - -/// The workload entry of an exact query evaluated as `recurrence` says. -fn exact(recurrence: QueryRecurrence) -> QueryWorkloadEntry { - QueryWorkloadEntry { - query: Default::default(), - requirements: Default::default(), - predictability: Default::default(), - recurrence, - range_evaluation: None, - } -} - -/// The #509 shared data workload: 1,000,000 series sampled every 15 s. -fn data() -> DataWorkload { - let mut data = shared_data_workload( - DataArrival::Ingesting { - rows_per_second: SHARED_ROWS_PER_SECOND, - }, - "m", - ); - // 100 jobs: each `by (job)` group holds many series, so a ranking - // shared between two top-k saves work. - data.sources.get_mut("m").unwrap().distinct_values_per_label = - [("job".to_string(), 100)].into(); - data -} - -/// Every minute. -fn repeating() -> QueryRecurrence { - QueryRecurrence::Repeated(RepeatedDemand::FixedInterval(RepetitionInterval(60_000))) -} - -/// Two jobs of two series each, a sample every 15 s over [0, 5m). -fn samples() -> Vec<(&'static str, &'static str, i64, f64)> { - let mut rows = vec![]; - for (s, (job, series)) in [("a", "a1"), ("a", "a2"), ("b", "b1"), ("b", "b2")] - .into_iter() - .enumerate() - { - for step in 0..20i64 { - rows.push(( - job, - series, - step * 15_000, - (s as i64 * 100 + step) as f64 + 0.5, - )); - } - } - rows -} - -/// Execute `root` at query time over the `samples` bound as raw data at its -/// `Scan`; each `TimeRange` keeps its own window (a tumbling window's moved -/// back by the `TimeShift` below it). -fn execute(root: &Rc) -> Vec { - let timed = apply_materialization_timings( - root, - &MaterializationAssignment::default(), - &mut NodeToTimedNodeCache::new(), - ) - .unwrap(); - // Tumbling windows run as their tumbling windows, each over its own window. - let timed = - asap_types::ir::operator::tumbling_window_chain::expand_tumbling_window_chains(&timed) - .unwrap(); - let dag = compile_physical_asap_dag(&timed).unwrap(); - let store = physical_common::raw_data_store(&dag, |_, schema| { - samples() - .into_iter() - .map(|(job, series, ts, value)| { - let labels = BTreeMap::from([ - ("__name__".to_string(), "m".to_string()), - ("job".to_string(), job.to_string()), - ("series".to_string(), series.to_string()), - ]); - promql_rows::series_row(schema, &labels, ts, value).unwrap() - }) - .collect() - }); - let plan = compile(&dag, &[], &[dag.roots[0] as u64]).unwrap(); - let mut rows: Vec = physical_common::execute( - &plan, - store, - Scope::Query { - evaluation_time_ms: EVALUATION_MS, - }, - ) - .remove(0) - .iter() - .flat_map(|batch| batch.to_rows().to_vec()) - .map(|row: Vec| format!("{row:?}")) - .collect(); - rows.sort(); - rows -} - -fn summary_builds(root: &Rc) -> usize { - OperatorNode::reachable(root) - .iter() - .filter(|n| matches!(n.asap(), Some(ASAPOp::SummaryAgg { .. }))) - .count() -} - -/// Every candidate of every group of `memo` with origin in `origins`, with -/// the group it implements; groups that answer a query or aggregate only. -fn candidates_of<'a>( - memo: &'a Memo, - origins: &'a [OriginOfCandidateForMemoGroup], -) -> impl Iterator)> + 'a { - let root_groups = root_groups(&memo.query_roots); - memo.candidates - .iter() - .enumerate() - .filter(move |(group, _)| { - let is_root = root_groups.contains(&MemoGroupId(*group as u32)); - let group = &memo.groups[*group]; - is_root - || matches!( - group.logical_node.non_asap(), - Some(NonASAPOp::Aggregate { .. }) - ) - }) - .flat_map(move |(group, candidates)| { - candidates - .iter() - .filter(move |c| origins.contains(&c.origin)) - .map(move |c| (group, &c.root_node)) - }) -} - -/// Plan selection's choice among fixed plans of `memo`'s `queries`: every -/// group on its first candidate (`group`), and `variant{i}` per entry of -/// `variants`, a group's candidate index each. Returns the selected plan's -/// name, its roots, whether it builds some computation once per evaluation -/// for all its readers (`MatAtQuery`), and the selection's lowest prices of -/// `group` and of every variant. -fn select( - memo: &Memo, - queries: usize, - entry: QueryWorkloadEntry, - variants: &[BTreeMap], -) -> (String, Vec>, bool, f64, f64) { - let mut plans = vec![( - "group".to_string(), - compose_memo_plan(memo, queries, &BTreeMap::new()), - )]; - for (i, choice) in variants.iter().enumerate() { - plans.push(( - format!("variant{i}"), - compose_memo_plan(memo, queries, choice), - )); - } - let named: Vec<(&str, Vec)> = plans - .iter() - .map(|(name, roots)| { - ( - name.as_str(), - roots.iter().cloned().map(QueryRoot::Operator).collect(), - ) - }) - .collect(); - let (physical, selection) = select_among(&named, &vec![entry; queries], &data()); - let selected = physical - .iter() - .find(|plan| plan.id == selection.selected) - .expect("the selected plan"); - let name = selected.from_logical.clone(); - let built_once = selected - .materialization - .values() - .any(|m| matches!(m, Materialization::MatAtQuery(_))); - let roots = plans - .iter() - .find(|(n, _)| *n == name) - .map(|(_, roots)| roots.clone()) - .unwrap(); - let variants = (0..variants.len()) - .map(|i| cheapest_of(&selection, &format!("variant{i}"))) - .fold(f64::INFINITY, f64::min); - let own = cheapest_of(&selection, "group"); - // A rejected baseline would make "cheaper than the baseline" hold trivially. - assert!(own.is_finite(), "the plan without reuse is valid"); - (name, roots, built_once, own, variants) -} - -/// `(group, candidate index)` of every candidate [`candidates_of`] yields. -fn choices(memo: &Memo, origins: &[OriginOfCandidateForMemoGroup]) -> Vec<(usize, usize)> { - candidates_of(memo, origins) - .map(|(group, node)| { - let index = memo.candidates[group] - .iter() - .position(|c| Rc::ptr_eq(&c.root_node, node)) - .unwrap(); - (group, index) - }) - .collect() -} - -/// [`candidates_of`], without the candidates that are their group's own -/// node. -fn own_candidates<'a>( - memo: &'a Memo, - origins: &'a [OriginOfCandidateForMemoGroup], -) -> impl Iterator)> + 'a { - candidates_of(memo, origins) - .filter(|(group, node)| !Rc::ptr_eq(node, &memo.groups[*group].logical_node)) -} - -/// `sum by (job) (sum_over_time(m[5m]))`: the nested sums and their merge -/// into one accumulator over the raw data samples. Plan selection picks the -/// merge, which builds no per-series state, and every candidate answers the -/// same per-job sums. -#[test] -fn nested_sum_merge_is_selected_and_answers_as_the_nested_sums() { - // Evaluated once, so no tumbling-window candidates. - let query = "sum by (job) (sum_over_time(m[5m]))"; - let memo = memo(&exact_queries(&[query]), QueryRecurrence::Unknown); - let variants: Vec<_> = choices(&memo, &[OriginOfCandidateForMemoGroup::Pass1]) - .into_iter() - .map(|(group, index)| BTreeMap::from([(group, index)])) - .collect(); - let (_, roots, ..) = select(&memo, 1, exact(QueryRecurrence::Unknown), &variants); - assert_eq!(summary_builds(&roots[0]), 1); - let answer = execute(&roots[0]); - assert_eq!(answer.len(), 2, "one sum per job: {answer:?}"); - every_candidate_answers_alike(query); -} - -/// Kinds of the nodes, other than the source chain, that query 0's and -/// query 1's roots share: what one query reads of the other's computation. -fn shared_kinds(roots: &[Rc]) -> BTreeSet { - let first: BTreeSet<_> = OperatorNode::reachable(&roots[0]) - .iter() - .map(Rc::as_ptr) - .collect(); - OperatorNode::reachable(&roots[1]) - .into_iter() - .filter(|node| first.contains(&Rc::as_ptr(node))) - .map(|node| node.operator.kind_name().to_string()) - .filter(|kind| !["Scan", "TimeRange", "TimeShift", "Project"].contains(&kind.as_str())) - .collect() -} - -/// The nodes the plan selected for `queries` shares between its two -/// queries, among the queries' own roots and the plans reading `origin` -/// candidates (the `i`-th of each group that has some, together). Stage 2 -/// offers each computation two queries read built once per evaluation -/// (`MatAtQuery`); the reuse plan is selected and is cheaper than without it. -fn selected_reuse(queries: &[&str], origin: OriginOfCandidateForMemoGroup) -> BTreeSet { - let memo = memo(&exact_queries(queries), repeating()); - let mut by_group: BTreeMap> = BTreeMap::new(); - for (group, index) in choices(&memo, std::slice::from_ref(&origin)) { - by_group.entry(group).or_default().push(index); - } - let most = by_group.values().map(Vec::len).max().unwrap_or(0); - assert!(most > 0, "{queries:?}: some {origin:?} candidate"); - let variants: Vec> = (0..most) - .map(|i| { - by_group - .iter() - .map(|(group, indices)| (*group, indices[i.min(indices.len() - 1)])) - .collect() - }) - .collect(); - let (name, roots, built_once, own, reuse) = - select(&memo, queries.len(), exact(repeating()), &variants); - assert!(name.starts_with("variant"), "{queries:?}: selected {name}"); - assert!( - built_once, - "{queries:?}: {name} computes its shared state once" - ); - assert!(reuse < own, "{queries:?}: reuse {reuse} vs own {own}"); - shared_kinds(&roots) -} - -/// `sum by (job)` beside `sum by (job, instance)`: with the finer sums -/// computed once, plan selection answers the coarse sum from them. -#[test] -fn rollup_is_selected_when_the_finer_sums_are_computed_once() { - let shared = selected_reuse( - &["sum by (job) (m)", "sum by (job, instance) (m)"], - OriginOfCandidateForMemoGroup::Rollup, - ); - // The coarse sum reads the finer sums' state or finalized values. - assert!( - shared.contains("Aggregate") || shared.contains("SummaryAgg"), - "{shared:?}" - ); -} - -/// Two exact top-k of the per-series rates: with the shared ranking -/// computed once, plan selection reads both from one ranking instead of -/// sorting every series twice. -#[test] -fn exact_topk_reuse_is_selected_when_the_ranking_is_computed_once() { - let shared = selected_reuse( - &[ - "topk by (job) (5, rate(m[1m]))", - "topk by (job) (10, rate(m[1m]))", - ], - OriginOfCandidateForMemoGroup::TopKReuse, - ); - assert!(shared.contains("Sort"), "{shared:?}"); -} - -/// `node` as Stage 2 implements it for `entry` (its first -/// materialization), executed. -fn execute_implemented(node: &Rc, entry: &QueryWorkloadEntry) -> Vec { - let plans = asap_physical_asap_optimization::optimize_physical_asap( - "candidate", - &[QueryRoot::Operator(Rc::clone(node))], - std::slice::from_ref(entry), - &data(), - false, - ) - .expect("Stage 2 implements the candidate"); - match &plans[0].roots[0] { - QueryRoot::Operator(root) => execute(root), - QueryRoot::Scalar(_) => panic!("operator root"), - } -} - -/// Every memo candidate of `query`'s aggregate and root groups other than -/// the group's own node executes to the same rows as that node. -fn every_candidate_answers_alike(query: &str) { - let memo = memo(&exact_queries(&[query]), QueryRecurrence::Unknown); - let entry = exact(QueryRecurrence::Unknown); - let mut checked = 0; - for (group, node) in own_candidates(&memo, &[OriginOfCandidateForMemoGroup::Pass1]) { - assert_eq!( - execute_implemented(node, &entry), - execute_implemented(&memo.groups[group].logical_node, &entry), - "{query}: a candidate of group {group} differs" - ); - checked += 1; - } - assert!( - checked > 0, - "{query}: some candidate other than the group's node" - ); -} - -/// A per-series sum drops the metric name, so grouping by `__name__` over -/// the nested sums groups every series under the empty name; a merge over -/// the raw data samples would group them under `m`. Every candidate answers as -/// the nested sums. -#[test] -fn nested_sum_by_the_metric_name_answers_as_the_nested_sums() { - every_candidate_answers_alike("sum by (__name__) (sum_over_time(m[5m]))"); - every_candidate_answers_alike("sum by (job) (sum_over_time(m[5m]))"); -} - -/// Every rollup or top-k reuse candidate in `memo`, of a group only the -/// queries before `first_approximate_query` read, executes to the rows of -/// the computation it replaces (its group's logical node). -fn reuse_answers_alike_in( - memo: &Memo, - queries: &[(&str, AccuracyTarget)], - first_approximate_query: usize, -) { - let entry = exact(repeating()); - let mut checked = 0; - for (group, node) in candidates_of( - memo, - &[ - OriginOfCandidateForMemoGroup::Rollup, - OriginOfCandidateForMemoGroup::TopKReuse, - ], - ) { - if memo.groups[group] - .reached_by_queries() - .iter() - .any(|&q| q >= first_approximate_query) - { - continue; - } - assert_eq!( - execute_implemented(node, &entry), - execute_implemented(&memo.groups[group].logical_node, &entry), - "{queries:?}: a reuse candidate of group {group} differs" - ); - checked += 1; - } - assert!(checked > 0, "{queries:?}: some reuse candidate"); -} - -/// [`reuse_answers_alike_in`] the memo of the exact `queries`. -fn reuse_answers_alike(queries: &[(&str, AccuracyTarget)]) { - reuse_answers_alike_in(&memo(queries, repeating()), queries, queries.len()); -} - -/// Coarse sums (over merged nested sums), minima and maxima rolled up from -/// the finer ones answer as computed from the raw data samples. -#[test] -fn rollups_answer_as_the_coarse_computations() { - for queries in [ - [ - "sum by (job) (sum_over_time(m[5m]))", - "sum by (job, series) (sum_over_time(m[5m]))", - ], - ["min by (job) (m)", "min by (job, series) (m)"], - ["max by (job) (m)", "max by (job, series) (m)"], - ] { - reuse_answers_alike(&exact_queries(&queries)); - } -} - -/// Top-k read from one larger ranking answer as computed alone. -#[test] -fn exact_topk_reuse_answers_as_each_topk() { - reuse_answers_alike(&exact_queries(&[ - "topk by (job) (1, sum_over_time(m[5m]))", - "topk by (job) (2, sum_over_time(m[5m]))", - ])); -} - -/// Query `q`'s rows when Stage 2 implements `roots` together (its first -/// materialization) for `entries`. -fn execute_plan( - roots: &[Rc], - entries: &[QueryWorkloadEntry], - q: usize, -) -> Vec { - let roots: Vec<_> = roots.iter().cloned().map(QueryRoot::Operator).collect(); - let plans = asap_physical_asap_optimization::optimize_physical_asap( - "plan", - &roots, - entries, - &data(), - false, - ) - .expect("Stage 2 implements the plan"); - match &plans[0].roots[q] { - QueryRoot::Operator(root) => execute(root), - QueryRoot::Scalar(_) => panic!("operator root"), - } -} - -/// A fine and a coarse sum beside two differently grouped quantiles: a plan -/// that shares the quantiles' summary build (summary-capability rule) and -/// rolls the coarse sum up from the fine one (rollup rule) answers each sum -/// as its own computation does (from the raw data samples), and each -/// quantile as the same shared build without the rollup. -#[test] -fn a_rollup_over_a_shared_summary_candidate_answers_as_without_it() { - let quantile = AccuracyTarget::max_error(0.05).failure_probability(0.3); - let queries = [ - ( - "sum by (job) (sum_over_time(m[5m]))", - AccuracyTarget::exact(), - ), - ( - "sum by (job, series) (sum_over_time(m[5m]))", - AccuracyTarget::exact(), - ), - ( - "avg by (job, series) (quantile_over_time(0.99, m[5m]))", - quantile.clone(), - ), - ("avg by (job) (quantile_over_time(0.95, m[5m]))", quantile), - ]; - let memo = memo(&queries, repeating()); - let shared = memo - .groups - .iter() - .find(|group| { - group.reached_by_queries().len() > 1 - && group.reached_by_queries().iter().all(|&q| q >= 2) - && matches!(group.logical_node.asap(), Some(ASAPOp::SummaryAgg { .. })) - }) - .map(|group| Rc::clone(&group.logical_node)) - .expect("the quantiles share a summary build"); - // Each quantile's own candidate that reads the shared build. - let mut sharing: BTreeMap = [2, 3] - .into_iter() - .map(|q| { - memo.candidates - .iter() - .enumerate() - .filter(|(group, _)| memo.groups[*group].reached_by_queries() == [q]) - .flat_map(|(group, candidates)| { - candidates - .iter() - .enumerate() - .map(move |(i, c)| (group, i, c)) - }) - .find(|(_, _, c)| { - OperatorNode::reachable(&c.root_node) - .iter() - .any(|node| Rc::ptr_eq(node, &shared)) - }) - .map(|(group, i, _)| (group, i)) - .unwrap_or_else(|| panic!("Q{} reads the shared build", q + 1)) - }) - .collect(); - let without = compose_memo_plan(&memo, queries.len(), &sharing); - let (group, index) = choices(&memo, &[OriginOfCandidateForMemoGroup::Rollup]) - .into_iter() - .find(|(group, _)| memo.groups[*group].reached_by_queries() == [0]) - .expect("a rollup of the coarse sum"); - sharing.insert(group, index); - let with = compose_memo_plan(&memo, queries.len(), &sharing); - let own = compose_memo_plan(&memo, queries.len(), &BTreeMap::new()); - assert!( - OperatorNode::reachable(&with[2]) - .iter() - .any(|node| OperatorNode::reachable(&with[3]) - .iter() - .any(|other| Rc::ptr_eq(node, other) - && matches!(node.asap(), Some(ASAPOp::SummaryAgg { .. })))), - "both quantiles read one summary build" - ); - let entries = vec![exact(repeating()); queries.len()]; - for q in 0..2 { - assert_eq!( - execute_plan(&with, &entries, q), - execute_plan(&own, &entries, q), - "Q{}", - q + 1 - ); - } - for q in 2..4 { - assert_eq!( - execute_plan(&with, &entries, q), - execute_plan(&without, &entries, q), - "Q{}", - q + 1 - ); - } -} diff --git a/crates/integration-tests/tests/rollup_controlled.rs b/crates/integration-tests/tests/rollup_controlled.rs index d646ded6b..1178513c4 100644 --- a/crates/integration-tests/tests/rollup_controlled.rs +++ b/crates/integration-tests/tests/rollup_controlled.rs @@ -1,18 +1,13 @@ //! Controlled plans for the viewer's actual distinct-count rollup workload. use asap_cost::{CostInputs, CostModel, ResourceUnitCost}; -use asap_integration_tests::planning::{builds_and_rollups, expanded_memo, select_rollup}; -use asap_logical_asap_optimization::test_support::{ - compose_logical_candidate, generate_per_computation_candidates, rollup_candidates, - rollup_pass2_candidates, -}; + use asap_planner::PlanningModels; -use asap_types::stages::HowToComputeAggregateMeasure; + use asap_types::{ ir::{ - schema::{DataType, Field, Schema, SketchAlgorithm}, + schema::{DataType, Field, Schema}, ASAPOp, OperatorNode, QueryRoot, }, - physical::{Materialization, Medium}, workload::PlanningWorkload, }; use std::{collections::HashSet, rc::Rc}; @@ -46,122 +41,6 @@ fn nodes(roots: &[QueryRoot]) -> Vec> { .collect() } -#[test] -fn controlled_distinct_rollup_prices_the_original_workload() { - let (workload, roots) = fixture(); - let entries: Vec<_> = workload.query_workload.entries().collect(); - let data = workload.data_workload.as_ref().unwrap(); - let generated = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&roots), - ) - .unwrap(); - let mut settings = CostInputs::default(); - settings.cost_model = CostModel::UsageOverTime(ResourceUnitCost::aws_fargate_us_east_1( - settings.resource_speed, - )); - let models = PlanningModels::builtin().with_cost_inputs(&settings); - let mut accepted = Vec::new(); - for algorithm in [None, Some(SketchAlgorithm::Hll), Some(SketchAlgorithm::Kmv)] { - let choices: Vec<_> = generated - .targets - .iter() - .map(|target| match &algorithm { - None => 0, - Some(a) => target - .ways_to_compute - .iter() - .position( - |c| matches!(c,HowToComputeAggregateMeasure::Summary(found) if found==a), - ) - .expect("family generated"), - }) - .collect(); - let roots: Vec<_> = compose_logical_candidate(&generated, &choices).unwrap(); - // Identical sub-DAGs made one node, as the memo's groups are. - let roots: Vec<_> = asap_logical_asap_optimization::test_support::share_common_sub_dags( - roots - .iter() - .enumerate() - .map(|(id, root)| match root { - QueryRoot::Operator(node) => (id, Rc::clone(node)), - QueryRoot::Scalar(_) => panic!("operator root"), - }) - .collect(), - ) - .into_iter() - .map(|(_, node)| QueryRoot::Operator(node)) - .collect(); - let mut variants = vec![("separate", roots.clone())]; - if algorithm.is_some() { - let rules = rollup_candidates(&roots); - assert_eq!(rules.len(), 1, "one service rollup for {algorithm:?}"); - for rolled in rollup_pass2_candidates(&roots, &rules).unwrap() { - variants.push(("rollup", rolled.roots.unwrap())); - } - } - for (kind, roots) in variants { - let family = algorithm - .as_ref() - .map_or("exact".to_string(), |a| format!("{a:?}")); - let id = format!("{family}-{kind}"); - let builds: HashSet<_> = nodes(&roots) - .iter() - .filter(|n| matches!(n.asap(), Some(ASAPOp::SummaryAgg { .. }))) - .map(Rc::as_ptr) - .collect(); - let mut plan = - asap_physical_asap_optimization::test_support::optimize_physical_asap_chosen( - &id, - &roots, - &entries, - data, - models.deployment_capabilities.query_time_results_retained, - &mut |node, _, options| { - let mat = Materialization::MatAtQuery(Medium::Memory); - Some(if builds.contains(&node) && options.contains(&mat) { - mat - } else { - Materialization::NotMaterialized - }) - }, - ) - .unwrap() - .expect("fixed query-time implementation"); - plan.id = id.clone(); - let result = asap_devtools::memo_pricing::reference_select( - &[plan.clone()], - &entries, - data, - models, - ); - match result { - Ok(selection) => { - println!( - "{id}: builds={} cost={:?} accepted={:?}", - builds.len(), - selection.costs[&id].cost_per_second, - selection.accepted - ); - accepted.push(plan); - } - Err(error) => println!("{id}: REJECTED {error}"), - } - for n in nodes(&roots) { - if let Some(ASAPOp::SummaryAgg { family, .. }) = n.asap() { - println!("{id} family={family:?}"); - } - } - } - } - let chosen = - asap_devtools::memo_pricing::reference_select(&accepted, &entries, data, models).unwrap(); - println!("CONTROLLED WINNER {}", chosen.selected); - assert_eq!( - chosen.selected, "Kmv-rollup", - "shared KMV must win the controlled cost comparison" - ); -} - // The production entrypoint must find a sketch rollup under the viewer budget. #[test] fn production_distinct_rollup_finds_shared_kmv() { @@ -210,18 +89,3 @@ fn production_distinct_rollup_finds_shared_kmv() { "production search must select the cheaper legal rollup" ); } - -// The production shape without a timed search: among fixed plans of the -// memo (a KMV per query, or both reading one KMV state group), plan -// selection with the built-in models picks one build and one rollup, and -// it costs less than a valid build per query. -#[test] -fn memo_offers_one_kmv_build_and_a_rollup() { - let (workload, roots) = fixture(); - let entries: Vec<_> = workload.query_workload.entries().collect(); - let data = workload.data_workload.as_ref().unwrap(); - let memo = expanded_memo(roots, &entries); - let (selected, separate, shared) = select_rollup(&memo, 2, &entries, data); - assert!(shared < separate, "{shared} vs {separate}"); - assert_eq!(builds_and_rollups(&selected), (1, 1)); -} diff --git a/crates/integration-tests/tests/runtime_handoff.rs b/crates/integration-tests/tests/runtime_handoff.rs deleted file mode 100644 index cadfe8788..000000000 --- a/crates/integration-tests/tests/runtime_handoff.rs +++ /dev/null @@ -1,468 +0,0 @@ -//! What `plan_installable` returns for a deployment runtime (issue #749), -//! checked against the Planner's internals; the runtime's own end-to-end -//! run, with public APIs only, is `crates/planner/tests/runtime_install.rs`. - -use std::collections::BTreeSet; - -use asap_executor::physical_planner::compile; -use asap_logical_asap_optimization::DeploymentCapabilities; -use asap_planner::{ - e2e_plan, plan_installable, test_support::query_time_reference, FrontendInput, - InstallableDagError, InstallablePlan, InstallablePlanError, PlanError, PlanOutput, - PlanningModels, SelectedPlanReport, UserInput, -}; -use asap_types::ir::physical_export::{ - compile_physical_asap_workload, PhysicalASAPDAG, PhysicalASAPNodeId, -}; -use asap_types::ir::properties::ExecutionTiming; -use asap_types::ir::{ASAPOp, Operator}; -use asap_types::physical::Materialization; -use asap_types::workload::PlanningWorkload; - -/// #509 Example 4b: one repeating `quantile_over_time` over a 60-minute -/// window, evaluated every 10 minutes, over ingesting data. -fn example4b() -> PlanningWorkload { - serde_json::from_str( - r#"{ - "query_workload": { - "language": "promql", - "repeating_queries": [{ - "query": "quantile_over_time(0.99, latency_ms[60m])", - "demand": {"fixed_interval": 600000}, - "requirements": { - "accuracy": {"accuracy_metric": "rank", "max_error": 0.01, - "failure_probability": 0.01, "accuracy_scope": "whole_result"}, - "max_latency_ms": 200.0 - }, - "predictability": {"predictable": {"known_at": null}} - }] - }, - "data_workload": { - "arrival": {"ingesting": {"rows_per_second": 1000.0}}, - "sources": {"latency_ms": {"series_count": 1000, "sample_interval": 1000}} - } - }"#, - ) - .expect("Example 4b parses") -} - -/// A deployment that keeps no raw data samples: every summary must be built at -/// ingestion time. -fn without_raw_data() -> DeploymentCapabilities { - DeploymentCapabilities { - raw_data_retained: false, - ..DeploymentCapabilities::default() - } -} - -async fn plan(workload: &PlanningWorkload, capabilities: &DeploymentCapabilities) -> PlanOutput { - let models = PlanningModels::builtin().with_deployment_capabilities(capabilities); - e2e_plan(UserInput::new(workload, FrontendInput::Promql, models)) - .await - .expect("Example 4b plans") -} - -fn materialized_count(dag: &PhysicalASAPDAG) -> usize { - dag.nodes - .iter() - .filter(|n| n.materialization != Materialization::NotMaterialized) - .count() -} - -/// Without raw data Stage 3 selects ingestion-time tumbling windows; the facade's -/// export carries that selection rather than resetting every node to -/// query time. -#[tokio::test] -async fn export_carries_the_selected_materialization() { - let capabilities = without_raw_data(); - let output = plan(&example4b(), &capabilities).await; - let dag = output.physical_asap_dag().expect("exports"); - assert!( - materialized_count(&dag) > 0, - "the selected tumbling windows are stored" - ); - assert!(dag - .nodes - .iter() - .any(|n| n.output_state.timing == ExecutionTiming::IngestionTime)); - // Exactly Stage 3's selected plan, as Stage 3 timed it. - assert!(output.selection.is_some(), "Stage 3 selected"); - assert_eq!( - dag, - compile_physical_asap_workload(&output.roots()).expect("the selected plan exports") - ); -} - -/// What a runtime that keeps no raw data samples gets from one facade call: -/// the plan and Stage 3's report of the selection. -async fn installable_with_report( - workload: &PlanningWorkload, -) -> (InstallablePlan, Option) { - let capabilities = without_raw_data(); - let models = PlanningModels::builtin().with_deployment_capabilities(&capabilities); - plan_installable(UserInput::new(workload, FrontendInput::Promql, models)) - .await - .expect("Example 4b plans") -} - -async fn installable(workload: &PlanningWorkload) -> InstallablePlan { - installable_with_report(workload).await.0 -} - -fn tumbling_window_chain_count(dag: &PhysicalASAPDAG) -> usize { - dag.nodes - .iter() - .filter(|n| { - matches!( - n.payload, - Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) - ) - }) - .count() -} - -/// The installable plan expands its selected tumbling-window width into contiguous -/// tumbling windows, each a materialized view refreshed at ingestion time, and -/// the executor compiles it. The stage export keeps the chain for costing -/// and display. Its one query is answered by the DAG's one root. -#[tokio::test] -async fn installable_plan_has_its_tumbling_window_chains_expanded() { - let workload = example4b(); - let (installed, report) = installable_with_report(&workload).await; - installed.validate().unwrap(); - let plan = &installed; - let dag = &installed.dag; - let views = plan.materialized_views().unwrap(); - assert_eq!(tumbling_window_chain_count(dag), 0); - assert_eq!(materialized_count(dag), views.len()); - let roots: Vec = dag.roots.iter().map(|&r| r as u64).collect(); - let compiled = compile(dag, &views, &roots).expect("the executor compiles it"); - assert_eq!(plan.query_texts.len(), 1); - assert_eq!( - plan.query_texts[0], - "quantile_over_time(0.99, latency_ms[60m])" - ); - // The query reads exactly the materialized views, and no raw data. - let refreshed: BTreeSet = views.iter().map(|v| v.node_id as u64).collect(); - let slots: BTreeSet = compiled.input_slots().map(|(id, _)| id).collect(); - assert_eq!(slots, refreshed); - assert_tumbling_window_coverage(plan, dag.roots[0], 0, 3_600_000); - assert!( - plan.raw_data_retention().unwrap().is_empty(), - "no raw data kept" - ); - let report = report.as_ref().expect("Stage 3 selected"); - assert!(report.cost_per_second.is_some()); - - let staged = plan_for_stages(&workload).await; - assert_eq!( - tumbling_window_chain_count(&staged), - 1, - "the stage export keeps the chain" - ); -} - -async fn plan_for_stages(workload: &PlanningWorkload) -> PhysicalASAPDAG { - plan(workload, &without_raw_data()) - .await - .physical_asap_dag() - .unwrap() -} - -/// Each summary build of `dag` (`SummaryAgg` or `SummaryTumblingWindowChain`) as its JSON -/// payload, sorted: which families with which parameters it builds. -fn summary_builds(dag: &PhysicalASAPDAG) -> Vec { - let mut builds: Vec = dag - .nodes - .iter() - .filter(|n| { - matches!( - n.payload, - Operator::ASAP( - ASAPOp::SummaryAgg { .. } | ASAPOp::SummaryTumblingWindowChain { .. } - ) - ) - }) - .map(|n| serde_json::to_string(&n.payload).unwrap()) - .collect(); - builds.sort(); - builds -} - -/// The correctness reference of the installable plan: the same summaries -/// with the same parameters, every node at query time and nothing materialized, -/// with the same queries bound to its roots. -#[tokio::test] -async fn reference_plan_runs_the_installable_plan_at_query_time() { - let plan = installable(&example4b()).await; - let reference = query_time_reference(&plan).expect("re-times"); - reference.validate().unwrap(); - assert_eq!(materialized_count(&reference.dag), 0); - assert!(reference.materialized_views().unwrap().is_empty()); - // The reference reads the raw data the plan's views were refreshed from. - let [retention] = reference.raw_data_retention().unwrap().try_into().unwrap(); - assert_eq!(retention.lookback_ms(), Some(3_600_000)); - assert!(reference - .dag - .nodes - .iter() - .all(|n| n.output_state.timing == ExecutionTiming::QueryTime)); - assert_eq!(summary_builds(&reference.dag), summary_builds(&plan.dag)); - assert_eq!(reference.query_texts, plan.query_texts); -} - -/// A repeating PromQL workload over ingesting `latency_ms`: `queries`, each -/// every 10 minutes at rank error 0.01, δ 0.01. -fn repeating(queries: &[&str]) -> PlanningWorkload { - let entries: Vec = queries - .iter() - .map(|q| { - format!( - r#"{{"query": {q:?}, "demand": {{"fixed_interval": 600000}}, - "requirements": {{"accuracy": {{"accuracy_metric": "rank", "max_error": 0.01, - "failure_probability": 0.01, "accuracy_scope": "whole_result"}}, - "max_latency_ms": 200.0}}, - "predictability": {{"predictable": {{"known_at": null}}}}}}"# - ) - }) - .collect(); - serde_json::from_str(&format!( - r#"{{"query_workload": {{"language": "promql", "repeating_queries": [{}]}}, - "data_workload": {{"arrival": {{"ingesting": {{"rows_per_second": 1000.0}}}}, - "sources": {{"latency_ms": {{"series_count": 1000, "sample_interval": 1000}}}}}}}}"#, - entries.join(",") - )) - .unwrap() -} - -/// p99 and p50 over the same 60 minutes share one chain of tumbling windows: the -/// installable plan materializes one shared set of tumbling windows. -#[tokio::test] -async fn two_queries_on_one_chain_store_its_tumbling_windows_once() { - let plan = installable(&repeating(&[ - "quantile_over_time(0.99, latency_ms[60m])", - "quantile_over_time(0.5, latency_ms[60m])", - ])) - .await; - assert_eq!(plan.query_texts.len(), 2); - let views = plan.materialized_views().unwrap(); - let views_below = |root| { - let below = nodes_below(&plan.dag, root); - views - .iter() - .filter(|view| below.contains(&view.node_id)) - .map(|view| view.node_id) - .collect::>() - }; - assert_eq!( - views_below(plan.dag.roots[0]), - views_below(plan.dag.roots[1]) - ); - assert_eq!(views_below(plan.dag.roots[0]).len(), views.len()); - assert_tumbling_window_coverage(&plan, plan.dag.roots[0], 0, 3_600_000); -} - -/// Planned with raw data kept (everything at query time), so each query's -/// range is a query-time raw-data window. -async fn installable_at_query_time(query: &str) -> Result { - let workload = repeating(&[query]); - plan_installable(UserInput::new( - &workload, - FrontendInput::Promql, - PlanningModels::builtin(), - )) - .await - .map(|(plan, _)| plan) -} - -/// A raw-data window whose position is not a fixed offset from the -/// evaluation time (a subquery or an offset above the range, an `@` -/// anchor below it), or that ends after the evaluation (a negative offset, -/// which no ingestion run can build in time), is refused with its reason, -/// not passed on with a wrong window or dropped; a plain range is listed. -#[tokio::test] -#[ignore = "fails on main too: the `@` query's selection MILP is infeasible (its scan has no rows_held to price); not a wall-clock issue; #1015"] -async fn raw_data_windows_of_unsupported_shapes_are_refused() { - for (query, reason) in [ - ( - "max_over_time(rate(latency_ms[5m])[1h:1m])", - "a subquery or a TimeShift above the range", - ), - ( - "max_over_time(latency_ms[10m] @ 1000)", - "an @ anchor below the range", - ), - ( - "max_over_time(latency_ms[60m] offset -10m)", - "a negative offset below the range", - ), - ] { - match installable_at_query_time(query).await { - Err(PlanError::Installable(InstallablePlanError::Uninstallable( - InstallableDagError::UnsupportedRawDataWindow { - reason: refused, .. - }, - ))) => assert_eq!(refused, reason, "{query}"), - Err(error) => { - // When planning fails before installation, its no-plan - // diagnostic must keep the concrete refusal reason. - let diagnostic = error.to_string(); - assert!(diagnostic.contains(reason), "{query}: {diagnostic}"); - assert!(diagnostic.contains("raw-data window"), "{diagnostic}"); - } - Ok(plan) => panic!( - "{query}: unexpectedly installed {:?}", - plan.raw_data_retention().unwrap() - ), - } - } - let plain = installable_at_query_time("max_over_time(latency_ms[10m] offset 5m)") - .await - .unwrap(); - // 10 minutes of data ending 5 minutes back reaches 15 minutes back. - let [retention] = plain.raw_data_retention().unwrap().try_into().unwrap(); - assert_eq!(retention.lookback_ms(), Some(900_000)); -} - -/// Example 4b answered as the last hour at 10-minute steps (a range -/// evaluation): each selected tumbling window view is kept for the hour its window reads -/// plus the hour the range spans, 7,200,000 ms, as Stage 3 records for the -/// chain (`Selection::tumbling_window_retention_ms`); without the range, for the hour. -#[tokio::test] -async fn stored_tumbling_windows_carry_stage3_tumbling_window_retention() { - let mut workload = example4b(); - let plain = installable(&workload).await; - assert!(plain - .materialized_views() - .unwrap() - .iter() - .all(|v| v.retention_ms() == 3_600_000)); - let mut json = serde_json::to_value(&workload).unwrap(); - json["query_workload"]["repeating_queries"][0]["range_evaluation"] = - serde_json::json!({"step_ms": 600000, "span_ms": 3600000}); - workload = serde_json::from_value(json).unwrap(); - let ranged = installable(&workload).await; - assert_tumbling_window_coverage(&ranged, ranged.dag.roots[0], 0, 3_600_000); - assert!(ranged - .materialized_views() - .unwrap() - .iter() - .all(|v| v.retention_ms() == 7_200_000)); - let output = plan(&workload, &without_raw_data()).await; - assert_eq!( - output - .tumbling_window_retention_ms - .values() - .copied() - .collect::>(), - [7_200_000] - ); - // A plan file that keeps a tumbling window for less than it reads is refused. - let mut short = ranged.clone(); - let view = ranged.materialized_views().unwrap()[0].node_id; - short - .dag - .nodes - .iter_mut() - .find(|n| n.id == view) - .unwrap() - .retention_ms = Some(0); - assert!(matches!( - short.validate(), - Err(InstallablePlanError::Uninstallable( - InstallableDagError::ViewRetentionTooShort { .. } - )) - )); -} - -/// A chain over an offset input keeps its tumbling windows for its window plus the -/// offset: `[60m] offset 10m` reaches 70 minutes back (4,200,000 ms), the -/// unshifted query's chain 60 minutes (3,600,000 ms). Each materialized view -/// carries its own node's requirement; a runtime that keeps the twins' -/// identical tumbling windows once keeps them for the larger one. -#[tokio::test] -async fn an_offset_chain_keeps_its_tumbling_windows_for_its_window_plus_the_offset() { - let plan = installable(&repeating(&[ - "quantile_over_time(0.99, latency_ms[60m])", - "quantile_over_time(0.99, latency_ms[60m] offset 10m)", - ])) - .await; - assert_tumbling_window_coverage(&plan, plan.dag.roots[0], 0, 3_600_000); - assert_tumbling_window_coverage(&plan, plan.dag.roots[1], 600_000, 4_200_000); - let below: Vec<_> = plan - .dag - .roots - .iter() - .map(|&root| nodes_below(&plan.dag, root)) - .collect(); - for view in &plan.materialized_views().unwrap() { - let required = [3_600_000, 4_200_000] - .into_iter() - .zip(&below) - .filter(|(_, nodes)| nodes.contains(&view.node_id)) - .map(|(retention, _)| retention) - .max() - .expect("materialized tumbling window is consumed"); - assert_eq!(view.retention_ms(), required); - } - plan.validate().unwrap(); -} - -/// Any selected legal tumbling-window width must cover the requested interval exactly. -fn assert_tumbling_window_coverage( - plan: &InstallablePlan, - root: PhysicalASAPNodeId, - start: i64, - end: i64, -) { - let below = nodes_below(&plan.dag, root); - let mut intervals: Vec<_> = plan - .materialized_views() - .unwrap() - .iter() - .filter(|view| below.contains(&view.node_id)) - .map(|view| { - let [window] = view.inputs.raw_data_windows() else { - panic!("tumbling window reads one raw interval: {:?}", view.inputs); - }; - assert!(window.width_ms() > 0 && 600_000 % window.width_ms() == 0); - ( - window.end_offset_ms(), - window.end_offset_ms() + window.width_ms(), - ) - }) - .collect(); - intervals.sort_unstable(); - assert!(!intervals.is_empty()); - assert_eq!(intervals[0].0, start); - assert_eq!(intervals.last().unwrap().1, end); - for pair in intervals.windows(2) { - assert_eq!(pair[0].1, pair[1].0); - } -} - -/// Every node `root` reads, directly or through other nodes. -fn nodes_below(dag: &PhysicalASAPDAG, root: PhysicalASAPNodeId) -> BTreeSet { - let mut below = BTreeSet::new(); - let mut pending = vec![root]; - while let Some(node) = pending.pop() { - for edge in dag.edges.iter().filter(|edge| edge.consumer == node) { - if below.insert(edge.producer) { - pending.push(edge.producer); - } - } - } - below -} - -/// A standalone scalar query (`scalar(sum(latency_ms))`) has no DAG node -/// to answer it through the public API, so it is refused with its entry. -#[tokio::test] -async fn a_scalar_query_is_refused() { - match installable_at_query_time("scalar(sum(latency_ms))").await { - Err(PlanError::Installable(InstallablePlanError::UnsupportedQueryRoot { - entry_index: 0, - })) => {} - other => panic!("{:?}", other.map(|p| p.query_texts)), - } -} diff --git a/crates/integration-tests/tests/sharing_issue_regressions.rs b/crates/integration-tests/tests/sharing_issue_regressions.rs index 04555299e..02221f75a 100644 --- a/crates/integration-tests/tests/sharing_issue_regressions.rs +++ b/crates/integration-tests/tests/sharing_issue_regressions.rs @@ -2,9 +2,9 @@ use asap_cost::CostInputs; use asap_devtools::planner_fuzz::{read_seed_file, FuzzSettings, FuzzWorkload, Planner}; use asap_devtools::planning_stages::run_search_over; -use asap_logical_asap_optimization::generate_dag_of_logical_memo_groups; + use asap_planner::{PlanSearchSettings, PlanningModels}; -use asap_types::stages::HowToComputeAggregateMeasure; + use serde_json::json; use std::path::Path; @@ -48,7 +48,6 @@ fn cost(w: &FuzzWorkload) -> f64 { &w.data, models, PlanSearchSettings::default(), - None, ) .unwrap(); let selection = run.run.selection.as_ref().unwrap(); @@ -58,8 +57,8 @@ fn cost(w: &FuzzWorkload) -> f64 { } /// #923: deriving the coarse sum costs only a sum over the 150 fine groups. -/// Stage 2 offers that plan; Stage 3 keeps a `MatAtQuery` build whose -/// `NotMaterialized` sibling would avoid a second scan (#1142). +/// Stage 2 offers that plan; Stage 3 selects a `OneEvaluation` build whose +/// not-materialized sibling would avoid a second scan (#1142). #[test] #[ignore = "Stage 3 misses the cheaper plan Stage 2 offers (#1142)"] fn exact_coarse_sum_reuses_fine_sum() { @@ -102,7 +101,7 @@ fn slower_repeat_reuses_faster_sum() { ); } -/// #902: the filtered reader avoids its independent scan and aggregation. +/// #902: the filtered query avoids its independent scan and aggregation. #[test] fn grouped_filtered_quantile_plans() { assert!( @@ -111,363 +110,7 @@ fn grouped_filtered_quantile_plans() { ); } -/// #886: SQL count-ranked TopK offers a heap over raw rows, absorbing the count. -#[test] -fn sql_count_topk_has_a_fused_raw_row_heap() { - use asap_logical_asap_optimization::test_support::generate_per_computation_candidates; - let roots = Planner::new(FuzzSettings::default()) - .lower(&workload(886)) - .unwrap(); - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&roots), - ) - .unwrap(); - let (index, target) = inventory - .targets - .iter() - .enumerate() - .find(|(_, target)| target.fused_computation.is_some()) - .expect("SQL TopK must offer a fused raw-row heap"); - for choice in target.ways_to_compute.len()..target.choice_count() { - let mut selected = vec![0; inventory.targets.len()]; - selected[index] = choice; - let composed = asap_logical_asap_optimization::test_support::compose_logical_candidate( - &inventory, &selected, - ) - .unwrap(); - let roots: Vec<_> = composed; - let root = roots[0].as_operator().unwrap(); - for node in asap_types::ir::OperatorNode::reachable(root) { - if let Some(asap_types::ir::ASAPOp::SummaryAgg { - family: asap_types::ir::schema::FieldDataType::Sketch(_, _), - input, - child, - .. - }) = node.asap() - { - assert_eq!( - input.weight, - asap_types::ir::schema::SummaryInputExpr::Constant(1.0) - ); - assert!(!child.contains_asap(), "heap consumes raw rows"); - } - } - let guarded = Planner::new(FuzzSettings::default()).price(&workload(886), &roots, false); - assert!( - guarded.unwrap_err().contains("NoAccuracyGuarantee"), - "frequency intervals must not be claimed as Top-K membership guarantees" - ); - use asap_executor::values::Value; - let rows = (0..7) - .map(|_| { - vec![ - Value::Timestamp(0), - Value::Utf8("NYC".into()), - Value::Utf8("one-host".into()), - Value::Float64(1.0), - ] - }) - .collect(); - let actual = physical_common::execute_raw_rows_at(root, rows, 1000); - assert_eq!(actual.len(), 1, "{actual:?}"); - assert!( - actual[0] - .iter() - .any(|v| matches!(v, Value::Utf8(s) if s.as_ref() == "one-host")), - "{actual:?}" - ); - assert!( - actual[0] - .iter() - .any(|v| matches!(v, Value::Float64(count) if *count == 7.0)), - "{actual:?}" - ); - let compiled = Planner::new(FuzzSettings { - enforce_accuracy_guarantees: false, - ..Default::default() - }) - .price(&workload(886), &roots, false); - assert!(compiled.is_ok(), "fused heap must compile: {compiled:?}"); - } -} - -/// #923: label projection differences below the same per-series average do not -/// prevent an exact sum from rolling up the finer sum's output. -#[test] -fn exact_rollup_catalog_crosses_identical_per_series_average() { - use asap_logical_asap_optimization::test_support::{ - generate_per_computation_candidates, try_populate_pass1, - }; - use asap_types::stages::OriginOfCandidateForMemoGroup; - let roots: Vec<_> = Planner::new(FuzzSettings::default()) - .lower(&workload(923)) - .unwrap() - .into_iter() - .collect(); - let initial = generate_dag_of_logical_memo_groups(&roots); - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&roots), - ) - .unwrap(); - let memo = try_populate_pass1(initial, &inventory).unwrap(); - let expanded = - asap_logical_asap_optimization::test_support::sharing_rules_and_shared_state_groups(memo) - .unwrap(); - let owner = expanded.query_roots[1].as_operator().unwrap().0 as usize; - assert!( - expanded.candidates[owner] - .iter() - .any(|c| c.origin == OriginOfCandidateForMemoGroup::Rollup), - "coarse root has no rollup candidate" - ); -} - -use crate::physical_common; - -/// #902: selecting one group preserves values and SQL's one NULL row when -/// the group is absent, including empty inputs and groups of NULL samples. -#[test] -fn filtered_group_preserves_sql_empty_and_null_semantics() { - use asap_executor::values::Value; - use asap_logical_asap_optimization::test_support::{ - generate_per_computation_candidates, try_populate_pass1, - }; - use asap_types::ir::{operator::AggIntent, NonASAPOp}; - use asap_types::stages::OriginOfCandidateForMemoGroup; - let roots: Vec<_> = Planner::new(FuzzSettings::default()) - .lower(&workload(902)) - .unwrap() - .into_iter() - .collect(); - let initial = generate_dag_of_logical_memo_groups(&roots); - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&roots), - ) - .unwrap(); - let memo = try_populate_pass1(initial, &inventory).unwrap(); - let expanded = - asap_logical_asap_optimization::test_support::sharing_rules_and_shared_state_groups(memo) - .unwrap(); - let (owner, candidate) = expanded.candidates.iter().enumerate().flat_map(|(owner, candidates)| candidates.iter().map(move |c| (owner,c))) - .find(|(_, c)| c.origin == OriginOfCandidateForMemoGroup::Rollup && matches!(c.root_node.non_asap(), Some(NonASAPOp::Aggregate { measures, child, .. }) - if matches!(measures.as_slice(), [AggIntent::Min { .. }]) && matches!(child.non_asap(), Some(NonASAPOp::Filter { .. })))) - .expect("filtered group read candidate"); - // The same readout keeps a KLL's rank bound: it selects a whole group, - // never predicates on an approximate value or combines populations. - use asap_accuracy::propagation::recompose_guarantees; - use asap_logical_asap_optimization::test_support::compose_logical_candidate; - use asap_types::ir::{schema::SketchAlgorithm, QueryRoot}; - use std::rc::Rc; - let filter = candidate.root_node.children()[0]; - let grouped = filter.children()[0]; - let inv = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(Rc::clone(grouped)), - ]), - ) - .unwrap(); - let choices: Vec<_> = inv - .targets - .iter() - .map(|target| { - target - .ways_to_compute - .iter() - .position(|c| { - matches!( - c, - HowToComputeAggregateMeasure::Summary(SketchAlgorithm::Kll) - ) - }) - .unwrap_or(0) - }) - .collect(); - let approximate = compose_logical_candidate(&inv, &choices).unwrap().remove(0); - let approximate = approximate.as_operator().unwrap(); - let filtered = Rc::new( - filter - .with_new_children(|_| Rc::clone(approximate)) - .unwrap(), - ); - let readout = Rc::new( - candidate - .root_node - .with_new_children(|_| Rc::clone(&filtered)) - .unwrap(), - ); - let recomposed = recompose_guarantees(&[QueryRoot::Operator(readout)]).unwrap(); - assert!(approximate.guarantee.is_some()); - assert_eq!( - recomposed[0].as_operator().unwrap().guarantee, - approximate.guarantee - ); - // A range of keys can select multiple populations; their extrema do not - // inherit one selected quantile's rank bound. - let mut many_groups = (*filtered).clone(); - if let asap_types::ir::Operator::NonASAP(NonASAPOp::Filter { pred, .. }) = - &mut many_groups.operator - { - if let asap_types::ir::ScalarExpr::Compare { op, .. } = &mut pred.0 { - *op = asap_types::ir::scalar::CompareOpKind::Gt; - } - } - let many_groups = Rc::new(many_groups); - let invalid = Rc::new( - candidate - .root_node - .with_new_children(|_| Rc::clone(&many_groups)) - .unwrap(), - ); - assert!( - recompose_guarantees(&[QueryRoot::Operator(invalid)]).unwrap()[0] - .as_operator() - .unwrap() - .guarantee - .is_none() - ); - // A filter that also reads the estimate decides by an approximate value - // which group is kept, so the selected group's bound does not hold. - let mut on_estimate = (*filtered).clone(); - if let asap_types::ir::Operator::NonASAP(NonASAPOp::Filter { pred, .. }) = - &mut on_estimate.operator - { - pred.0 = asap_types::ir::ScalarExpr::BoolAnd(vec![ - pred.0.clone(), - asap_types::ir::ScalarExpr::IsNotNull(Box::new(asap_types::ir::ScalarExpr::Column(1))), - ]); - } - let on_estimate = Rc::new(on_estimate); - let invalid = Rc::new( - candidate - .root_node - .with_new_children(|_| Rc::clone(&on_estimate)) - .unwrap(), - ); - assert!( - recompose_guarantees(&[QueryRoot::Operator(invalid)]).unwrap()[0] - .as_operator() - .unwrap() - .guarantee - .is_none() - ); - let row = |city: &str, value| { - vec![ - Value::Timestamp(0), - Value::Utf8(city.into()), - Value::Utf8("host".into()), - value, - ] - }; - for rows in [ - vec![], - vec![row("LA", Value::Float64(5.0))], - vec![row("NYC", Value::Null)], - vec![ - row("NYC", Value::Float64(10.0)), - row("NYC", Value::Float64(30.0)), - row("LA", Value::Float64(90.0)), - ], - ] { - let before = physical_common::execute_raw_rows_at( - &expanded.groups[owner].logical_node, - rows.clone(), - 1000, - ); - let after = physical_common::execute_raw_rows_at(&candidate.root_node, rows, 1000); - assert_eq!(format!("{after:?}"), format!("{before:?}")); - assert_eq!(after.len(), 1); - } -} - -/// #923: SQL counts roll up by merging integer states, without float casts. -#[test] -fn sql_count_rollup_executes_with_integer_results() { - use asap_executor::values::Value; - use asap_logical_asap_optimization::test_support::{ - generate_per_computation_candidates, try_populate_pass1, - }; - use asap_types::ir::ASAPOp; - use asap_types::stages::OriginOfCandidateForMemoGroup; - let mut w = workload(902); - w.queries.truncate(2); - w.queries[0].entry.query.0 = - "SELECT city, host, COUNT(*) AS c FROM metrics GROUP BY city, host".into(); - for query in [ - "SELECT city, COUNT(*) AS c FROM metrics GROUP BY city", - "SELECT COUNT(*) AS c FROM metrics", - ] { - w.queries[1].entry.query.0 = query.into(); - let roots: Vec<_> = Planner::new(FuzzSettings::default()) - .lower(&w) - .unwrap() - .into_iter() - .collect(); - let initial = generate_dag_of_logical_memo_groups(&roots); - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&roots), - ) - .unwrap(); - let memo = try_populate_pass1(initial, &inventory).unwrap(); - let expanded = - asap_logical_asap_optimization::test_support::sharing_rules_and_shared_state_groups( - memo, - ) - .unwrap(); - let (owner, candidate) = expanded - .candidates - .iter() - .enumerate() - .flat_map(|(owner, candidates)| candidates.iter().map(move |c| (owner, c))) - .find(|(_, c)| { - c.origin == OriginOfCandidateForMemoGroup::Rollup - && asap_types::ir::OperatorNode::reachable(&c.root_node) - .iter() - .any(|node| { - matches!(node.asap(), Some(ASAPOp::FinalizeExactAccumulator { child }) - if matches!(child.asap(), Some(ASAPOp::SummaryMerge { group_by:Some(_), .. }))) - }) - }) - .expect("integer count state rollup candidate"); - let rows = vec![ - vec![ - Value::Timestamp(0), - Value::Utf8("NYC".into()), - Value::Utf8("a".into()), - Value::Null, - ], - vec![ - Value::Timestamp(0), - Value::Utf8("NYC".into()), - Value::Utf8("b".into()), - Value::Float64(10.0), - ], - vec![ - Value::Timestamp(0), - Value::Null, - Value::Utf8("a".into()), - Value::Null, - ], - ]; - let before = - physical_common::execute_raw_rows(&expanded.groups[owner].logical_node, rows.clone()); - let after = physical_common::execute_raw_rows(&candidate.root_node, rows); - let sorted = |rows: Vec>| { - let mut rows: Vec<_> = rows.iter().map(|row| format!("{row:?}")).collect(); - rows.sort(); - rows - }; - assert!(after - .iter() - .all(|row| matches!(row.last(), Some(Value::Int64(_))))); - assert_eq!(sorted(before), sorted(after)); - let before_empty = - physical_common::execute_raw_rows(&expanded.groups[owner].logical_node, vec![]); - let after_empty = physical_common::execute_raw_rows(&candidate.root_node, vec![]); - assert_eq!(sorted(before_empty), sorted(after_empty)); - } -} - -/// #922: duplicate exact TopK readers share their result and inner window work. +/// #922: duplicate exact TopK queries share their result and inner window work. #[test] fn duplicate_exact_topk_shares_work() { let mut w = workload(924); @@ -481,7 +124,7 @@ fn duplicate_exact_topk_shares_work() { assert!(duplicate <= single * 1.001, "{duplicate} vs {single}"); } -/// #924/#925 SQL variants: nested windows and slower readers reuse one sum. +/// #924/#925 SQL variants: nested windows and slower queries reuse one sum. #[test] fn sql_sum_shares_nested_windows_and_slower_readers() { let mut w = workload(922); @@ -505,68 +148,3 @@ fn sql_sum_shares_nested_windows_and_slower_readers() { "{repeated_cost} vs {single}" ); } - -/// #923: sketches over an inner range function expose the same finer rollup. -#[test] -fn sketch_rollup_crosses_inner_range_function() { - use asap_logical_asap_optimization::test_support::{ - generate_per_computation_candidates, try_populate_pass1, - }; - use asap_types::stages::OriginOfCandidateForMemoGroup; - let mut w = workload(923); - for (query, keys) in w.queries.iter_mut().zip(["job, region", "job"]) { - query.entry.query.0 = format!("quantile by ({keys}) (0.95, sum_over_time(m_gauge[5m]))"); - query.entry.requirements.accuracy = - asap_types::ir::properties::AccuracyTarget::max_error(0.05).failure_probability(0.01); - } - let roots: Vec<_> = Planner::new(FuzzSettings::default()) - .lower(&w) - .unwrap() - .into_iter() - .collect(); - let initial = generate_dag_of_logical_memo_groups(&roots); - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&roots), - ) - .unwrap(); - let memo = try_populate_pass1(initial, &inventory).unwrap(); - let expanded = - asap_logical_asap_optimization::test_support::sharing_rules_and_shared_state_groups(memo) - .unwrap(); - let owner = expanded.query_roots[1].as_operator().unwrap().0 as usize; - assert!(expanded.candidates[owner] - .iter() - .any(|c| c.origin == OriginOfCandidateForMemoGroup::Rollup)); -} - -/// #886: unsupported per-group SQL count sketches are explicitly pruned; -/// the exact counter and separately typed raw-row TopK fusion remain available. -#[test] -fn sql_grouped_count_sketches_are_not_offered() { - use asap_logical_asap_optimization::test_support::generate_per_computation_candidates; - use asap_types::ir::{operator::AggIntent, NonASAPOp}; - use asap_types::stages::PruneReason; - let w = workload(886); - let roots = Planner::new(FuzzSettings::default()).lower(&w).unwrap(); - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&roots), - ) - .unwrap(); - let (index, target) = inventory.targets.iter().enumerate().find(|(_, target)| matches!(inventory.aggregate(target).non_asap(), Some(NonASAPOp::Aggregate { measures, .. }) if matches!(measures.as_slice(), [AggIntent::Count { .. }]))).unwrap(); - assert!(target - .ways_to_compute - .iter() - .all(|candidate| !matches!(candidate, HowToComputeAggregateMeasure::Summary(_)))); - assert_eq!( - inventory - .pruned_of_target(index, false) - .iter() - .filter(|(_, reason)| *reason == PruneReason::NoRelationalCountSketchRealization) - .count(), - 2 - ); - assert!(inventory - .targets - .iter() - .any(|target| target.fused_computation.is_some())); -} diff --git a/crates/integration-tests/tests/sketch_rollup.rs b/crates/integration-tests/tests/sketch_rollup.rs index a5044bd34..a49628beb 100644 --- a/crates/integration-tests/tests/sketch_rollup.rs +++ b/crates/integration-tests/tests/sketch_rollup.rs @@ -4,7 +4,6 @@ //! the planner offers a SQL grouped query, the rolled-up answer equals the //! direct build's (deterministic merges) or is within the sketch's bound //! (KLL), and its coverage is the direct build's. -use crate::physical_common; use std::collections::BTreeMap; use std::rc::Rc; @@ -84,7 +83,7 @@ fn as_float64(node: &Rc) -> Rc { /// The answers of the logical DAG `root` over [`rows`], by group key. fn execute(root: &Rc) -> BTreeMap { let root = &as_float64(root); - let dag = physical_common::compile_physical_asap_dag(root).unwrap(); + let dag = asap_types::test_support::physical_asap_dag_at_query_time(root).unwrap(); let scan = dag .nodes .iter() @@ -242,14 +241,9 @@ async fn direct_and_rolled_up( continue; }; let merge = rolled_up(&build); - assert_eq!( - asap_types::test_support::definition_id(&merge.coverage().expect("a valid rollup")), - asap_types::test_support::definition_id(&build.coverage().unwrap()), - "{kind:?}: the rolled-up states cover what the direct build does" - ); let rolled = replace(&candidate, &build, &merge); rolled.validate_structure().unwrap(); - out.push((*kind.algorithm(), candidate, rolled)); + out.push((*kind, candidate, rolled)); } out } @@ -364,9 +358,9 @@ async fn invalid_rollups_are_rejected() { assert!(regroup(keys.clone()).is_err(), "{keys:?}"); } // No keys: the global build over the same rows. - assert!(regroup(vec![]).unwrap().coverage().is_some()); + assert!(regroup(vec![]).is_ok()); // Both keys, swapped, are still group keys of the input. - assert!(regroup(vec![1, 0]).unwrap().coverage().is_some()); + assert!(regroup(vec![1, 0]).is_ok()); // Rate states merge consecutive windows of one series, not groups. let Some(ASAPOp::SummaryAgg { child, diff --git a/crates/integration-tests/tests/sketch_rollup_planning.rs b/crates/integration-tests/tests/sketch_rollup_planning.rs deleted file mode 100644 index 2519479b7..000000000 --- a/crates/integration-tests/tests/sketch_rollup_planning.rs +++ /dev/null @@ -1,176 +0,0 @@ -//! Sketch rollup in planning: the memo's rollup rule offers answering a -//! coarse sketch grouping from the finest build of the same sketch over the -//! same rows, and plan selection selects it when one build and a merge cost -//! less than a build per query over the raw rows. Selection prices fixed -//! plans made of the memo's candidates, so no assertion depends on a timed -//! search. -use asap_executor::physical_planner::promql_rows; -use asap_frontend_sql::{lower_sql, SqlCatalog}; -use asap_integration_tests::fixtures::lower_promql; -use asap_integration_tests::planning::{builds_and_rollups, expanded_memo, select_rollup}; -use asap_types::ir::properties::{AccuracyMetric, AccuracyScope, AccuracyTarget}; -use asap_types::ir::schema::{DataType, Field, Schema}; -use asap_types::ir::QueryRoot; -use asap_types::workload::{ - DataWorkload, Predictability, QueryRecurrence, QueryRequirements, QueryWorkloadEntry, - RepeatedDemand, RepetitionInterval, -}; - -fn catalog() -> SqlCatalog { - SqlCatalog::new().with_table( - "http", - Schema::new(vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("region", DataType::Utf8, false), - Field::plain("service", DataType::Utf8, false), - Field::plain("user_id", DataType::Float64, false), - Field::plain("latency", DataType::Float64, false), - ]), - ) -} - -/// A predictable query repeating every `interval_ms` at `accuracy`. -fn entry(accuracy: &AccuracyTarget, interval_ms: u32) -> QueryWorkloadEntry { - QueryWorkloadEntry { - query: Default::default(), - requirements: QueryRequirements { - accuracy: accuracy.clone(), - max_latency_ms: None, - }, - predictability: Predictability::Predictable { known_at: None }, - recurrence: QueryRecurrence::Repeated(RepeatedDemand::FixedInterval(RepetitionInterval( - interval_ms, - ))), - range_evaluation: None, - } -} - -/// `http` arrives at 2e6 rows/s (W-RollupHeavy's table), 10,000 series, -/// a million distinct users at most per evaluation, and holds a minute of -/// rows (the catalog has no time index, so a query's time predicate is a -/// filter over every row held). -fn data() -> DataWorkload { - serde_json::from_value(serde_json::json!({ - "arrival": {"ingesting": {"rows_per_second": 2_000_000.0}}, - "sources": {"http": { - "series_count": 10_000, - "sample_interval": 5, - "values_non_negative": true, - "max_distinct_per_evaluation": 1_000_000, - "rows_held": 120_000_000, - }}, - })) - .unwrap() -} - -/// The SQL `queries` lowered at `accuracy`. -async fn sql_roots(queries: &[&str], accuracy: &AccuracyTarget) -> Vec { - let mut roots = Vec::new(); - for query in queries { - let root = lower_sql(query, &catalog(), accuracy.clone()) - .await - .unwrap(); - roots.push(QueryRoot::Operator(root)); - } - roots -} - -/// Distinct users by (region, service) and by service, both at relative -/// ε = 0.05 per value over the same window (W-RollupHeavy's targets): the -/// memo offers the coarse KMV rolled up from the finer one, and plan -/// selection selects one build and a rollup, cheaper than a build per -/// query from the raw rows. -#[tokio::test] -async fn stage3_selects_a_distinct_count_rollup_when_it_is_cheaper() { - let accuracy = AccuracyTarget::max_error(0.05) - .failure_probability(0.05) - .accuracy_metric(AccuracyMetric::Relative) - .accuracy_scope(AccuracyScope::PerValue); - let window = "WHERE ts >= now() - INTERVAL '5 minutes'"; - let roots = sql_roots( - &[ - &format!( - "SELECT region, service, COUNT(DISTINCT user_id) AS value FROM http {window} \ - GROUP BY region, service" - ), - &format!( - "SELECT service, COUNT(DISTINCT user_id) AS value FROM http {window} \ - GROUP BY service" - ), - ], - &accuracy, - ) - .await; - let entries = vec![entry(&accuracy, 60_000); 2]; - let memo = expanded_memo(roots, &entries); - let (selected, separate, rolled_up) = select_rollup(&memo, 2, &entries, &data()); - assert!(rolled_up < separate, "{rolled_up} vs {separate}"); - assert_eq!(builds_and_rollups(&selected), (1, 1)); -} - -/// p99 latency by (region, service), by region and by service at rank -/// ε = 0.05 (template 13's shape): both coarse groupings roll up from one -/// KLL, and plan selection selects that plan. -#[tokio::test] -async fn coarse_quantile_groupings_roll_up_from_one_kll() { - let accuracy = AccuracyTarget::max_error(0.05) - .failure_probability(0.05) - .accuracy_metric(AccuracyMetric::Rank) - .accuracy_scope(AccuracyScope::PerValue); - let window = "WHERE ts >= now() - INTERVAL '5 minutes'"; - let query = |keys: &str| { - format!( - "SELECT {keys}, approx_percentile_cont(latency, 0.99) AS value FROM http {window} \ - GROUP BY {keys}" - ) - }; - let roots = sql_roots( - &[ - &query("region, service"), - &query("region"), - &query("service"), - ], - &accuracy, - ) - .await; - let entries = vec![entry(&accuracy, 60_000); 3]; - let memo = expanded_memo(roots, &entries); - let (selected, separate, rolled_up) = select_rollup(&memo, 3, &entries, &data()); - assert!(rolled_up < separate, "{rolled_up} vs {separate}"); - assert_eq!(builds_and_rollups(&selected), (1, 2)); -} - -/// #840's `quantile by (host, job)` and `quantile by (job)` over 10,000 -/// series, every 10 s at rank ε = 0.01, δ = 0.001: the memo offers the -/// coarse KLL rolled up from the finer one, and its guarantee meets the -/// target, so plan selection selects it. -#[test] -fn promql_quantile_by_job_rolls_up_from_by_host_job() { - let accuracy = AccuracyTarget::max_error(0.01) - .failure_probability(0.001) - .accuracy_metric(AccuracyMetric::Rank); - let roots: Vec = [ - "quantile by (host, job) (0.5, data)", - "quantile by (job) (0.5, data)", - ] - .iter() - .map(|query| { - let root = lower_promql(query, accuracy.clone()).unwrap(); - QueryRoot::Operator(promql_rows::with_series_identity(&root).unwrap()) - }) - .collect(); - let data: DataWorkload = serde_json::from_value(serde_json::json!({ - "arrival": {"ingesting": {"rows_per_second": 666.67}}, - "sources": {"data": {"series_count": 10_000, "metric_type": "gauge", - "sample_interval": 15_000, - // 15 minutes of raw rows. - "rows_held": 600_000}}, - })) - .unwrap(); - let entry = entry(&accuracy, 10_000); - let entries = [entry.clone(), entry]; - let memo = expanded_memo(roots, &entries); - let (selected, separate, rolled_up) = select_rollup(&memo, 2, &entries, &data); - assert!(rolled_up < separate, "{rolled_up} vs {separate}"); - assert_eq!(builds_and_rollups(&selected), (1, 1)); -} diff --git a/crates/integration-tests/tests/sql_group_bound.rs b/crates/integration-tests/tests/sql_group_bound.rs deleted file mode 100644 index e8497c4fa..000000000 --- a/crates/integration-tests/tests/sql_group_bound.rs +++ /dev/null @@ -1,244 +0,0 @@ -//! A whole-result accuracy target over SQL `GROUP BY` groups (#863): the -//! distinct values the data workload declares for the grouping columns -//! (`SourceFacts::distinct_values_per_label`) bound the groups one -//! evaluation returns, so a per-group sketch can be sized to meet the target -//! over all of them. Without declarations the groups stay unbound and only -//! the exact plan meets it. - -use crate::planner_layering_common; - -use asap_devtools::planning_stages::lower_sql_roots; -use asap_devtools::SqlCatalog; -use asap_types::ir::schema::{DataType, Field, Schema, SketchAlgorithm}; -use asap_types::workload::PlanningWorkload; -use planner_layering_common::*; - -/// The `http` table of the RollupHeavy workload. -fn http_catalog() -> SqlCatalog { - let utf8 = |name| Field::plain(name, DataType::Utf8, false); - let float = |name| Field::plain(name, DataType::Float64, false); - SqlCatalog::new().with_table( - "http", - Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - utf8("region"), - utf8("service"), - utf8("endpoint"), - utf8("status"), - float("user_id"), - float("latency"), - ], - 0, - vec![], - ), - ) -} - -/// `query` every minute with a whole-result target (`metric` error 0.05, -/// failure probability 0.05) over `http`: 10,000 label combinations, and -/// `declared` as each column's distinct values. -fn workload(query: &str, metric: &str, declared: serde_json::Value) -> PlanningWorkload { - let http = serde_json::json!({ - "series_count": 10000, "sample_interval": 5, "values_non_negative": true, - "max_distinct_per_evaluation": 1000000, "distinct_values_per_label": declared, - "rows_held": 1000000000 - }); - serde_json::from_value(serde_json::json!({ - "query_workload": {"language": {"sql": "datafusion_sql"}, "query_batch": null, - "repeating_queries": [{ - "query": query, "demand": {"fixed_interval": 60000}, - "requirements": {"accuracy": {"accuracy_metric": metric, "max_error": 0.05, - "failure_probability": 0.05, "accuracy_scope": "whole_result"}}, - "predictability": {"predictable": {"known_at": null}}}]}, - "data_workload": {"arrival": {"ingesting": {"rows_per_second": 2e6}}, - "sources": {"http": http}} - })) - .expect("workload") -} - -/// Every `http` column's distinct values: 4 regions, 25 services, 25 -/// endpoints, 4 statuses. -fn every_column() -> serde_json::Value { - serde_json::json!({"region": 4, "service": 25, "endpoint": 25, "status": 4}) -} - -/// Evaluate the exact computation and each supported local summary choice. -/// These tests exercise group/error models, not a timed search's visited archive. -fn run_group_choices(workload: &PlanningWorkload) -> Run { - use asap_logical_asap_optimization::test_support::{ - compose_logical_candidate, generate_per_computation_candidates, - }; - let roots = lower_sql_roots(workload, &http_catalog()).expect("lowers"); - let local = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&roots), - ) - .unwrap(); - assert_eq!(local.targets.len(), 1, "one grouped statistic per fixture"); - let candidates = (0..local.targets[0].ways_to_compute.len()) - .map(|choice| compose_logical_candidate(&local, &[choice]).unwrap()) - .collect(); - run_explicit_candidates( - workload, - roots, - candidates, - asap_planner::PlanningModels::builtin(), - ) -} - -/// The sketches the selected plan builds (empty for the exact plan). -fn selected_sketches( - query: &str, - metric: &str, - declared: serde_json::Value, -) -> Vec { - let workload = workload(query, metric, declared); - let run = run_group_choices(&workload); - sketch_builds(run.physical(&run.selection.selected).dag()) - .into_iter() - .map(|(_, algorithm, _)| algorithm) - .collect() -} - -/// Template 14: distinct users per (service, endpoint). -const DISTINCT_USERS: &str = "SELECT service, endpoint, COUNT(DISTINCT user_id) AS value \ - FROM http WHERE ts >= now() - INTERVAL '5 minutes' GROUP BY service, endpoint"; - -/// Template 13: the p99 latency per (region, service). -const P99_LATENCY: &str = "SELECT region, service, approx_percentile_cont(latency, 0.99) AS value \ - FROM http WHERE ts >= now() - INTERVAL '5 minutes' GROUP BY region, service"; - -/// With 25 services and 25 endpoints declared, at most 625 groups: a -/// distinct-count sketch per group, each failing with probability at most -/// 0.05 / 625, meets the whole-result target. -#[test] -fn declared_grouping_columns_bound_the_groups_of_a_distinct_count() { - let sketches = selected_sketches(DISTINCT_USERS, "relative", every_column()); - assert!( - matches!( - sketches.as_slice(), - [SketchAlgorithm::Hll | SketchAlgorithm::Kmv] - ), - "{sketches:?}" - ); -} - -/// With 4 regions and 25 services declared, at most 100 groups: a KLL per -/// group meets the whole-result rank target. -#[test] -fn declared_grouping_columns_bound_the_groups_of_a_quantile() { - let sketches = selected_sketches(P99_LATENCY, "rank", every_column()); - assert_eq!(sketches, [SketchAlgorithm::Kll]); -} - -/// Without declared distinct values the groups are unbound: no sketch -/// meets a whole-result target, and the exact plan is selected. -#[test] -fn undeclared_grouping_columns_leave_only_the_exact_plan() { - for (query, metric) in [(DISTINCT_USERS, "relative"), (P99_LATENCY, "rank")] { - let sketches = selected_sketches(query, metric, serde_json::json!({})); - assert!(sketches.is_empty(), "{query}: {sketches:?}"); - } -} - -/// One grouping column declared is not enough: `service` has no declared -/// count, so the groups of `GROUP BY region, service` stay unbound. -#[test] -fn a_partly_declared_grouping_leaves_only_the_exact_plan() { - let sketches = selected_sketches(P99_LATENCY, "rank", serde_json::json!({"region": 4})); - assert!(sketches.is_empty(), "{sketches:?}"); -} - -/// A computed key named like a declared column no longer holds that -/// column's values: `upper(service) AS service` gives no bound. -#[test] -fn a_computed_grouping_column_leaves_only_the_exact_plan() { - let query = "SELECT service, COUNT(DISTINCT user_id) AS value FROM \ - (SELECT concat(service, endpoint) AS service, user_id, ts FROM http) \ - WHERE ts >= now() - INTERVAL '5 minutes' GROUP BY service"; - let sketches = selected_sketches(query, "relative", every_column()); - assert!(sketches.is_empty(), "{sketches:?}"); -} - -/// The groups every priced plan's grouped builds and aggregates are priced -/// at (`... into N states`, `... into N groups`), and the rows its scans read. -fn priced_groups_and_scan_rows( - workload: PlanningWorkload, -) -> ( - std::collections::BTreeSet, - std::collections::BTreeSet, -) { - let run = run_group_choices(&workload); - let (mut groups, mut scanned) = ( - std::collections::BTreeSet::new(), - std::collections::BTreeSet::new(), - ); - for cost in run.selection.costs.values() { - for node in cost.per_node.values() { - let words: Vec<&str> = node.detail.split([' ', ':', ';']).collect(); - for window in words.windows(3) { - if window[0] == "into" && matches!(window[2], "states" | "groups") { - groups.insert(window[1].parse().expect("a group count")); - } - if window[0] == "scan" && window[2] == "samples" { - scanned.insert(window[1].parse().expect("a row count")); - } - } - } - } - (groups, scanned) -} - -/// A SQL grouping with an undeclared column gets no invented count: each -/// of the table's 10,000 declared groups of rows is in one group, so its -/// builds are priced at 10,000; without `series_count`, each row read is -/// in at most one group, so they are priced at the rows the scan reads. -/// Declared, `GROUP BY service, endpoint` is 25 x 25. -#[test] -fn an_undeclared_sql_grouping_is_priced_at_the_groups_of_rows_or_the_rows_read() { - let undeclared = || workload(DISTINCT_USERS, "relative", serde_json::json!({})); - let (groups, _) = priced_groups_and_scan_rows(undeclared()); - assert_eq!(groups, std::collections::BTreeSet::from([10_000])); - let mut without_series = undeclared(); - let http = without_series - .data_workload - .as_mut() - .unwrap() - .sources - .get_mut("http") - .unwrap(); - http.series_count = None; - http.sample_interval = None; - let (groups, scanned) = priced_groups_and_scan_rows(without_series); - assert_eq!(groups, scanned); - let (groups, _) = - priced_groups_and_scan_rows(workload(DISTINCT_USERS, "relative", every_column())); - assert_eq!(groups, std::collections::BTreeSet::from([625])); -} - -/// Each endpoint is in one service: declaring `(service, endpoint)`'s 625 -/// combinations prices `GROUP BY service, endpoint` at 625 groups, where -/// the per-column product (25 x 625) is capped only by the 10,000 groups of -/// rows. -#[test] -fn a_declared_column_set_prices_a_sql_grouping_at_its_combinations() { - let hierarchical = - serde_json::json!({"region": 4, "service": 25, "endpoint": 625, "status": 4}); - let (groups, _) = - priced_groups_and_scan_rows(workload(DISTINCT_USERS, "relative", hierarchical.clone())); - assert_eq!(groups, std::collections::BTreeSet::from([10_000])); - let mut declared = workload(DISTINCT_USERS, "relative", hierarchical); - declared - .data_workload - .as_mut() - .unwrap() - .sources - .get_mut("http") - .unwrap() - .distinct_values_per_label_set = vec![asap_types::workload::LabelSetDistinctValues { - labels: vec!["service".to_string(), "endpoint".to_string()], - distinct_values: 625, - }]; - let (groups, _) = priced_groups_and_scan_rows(declared); - assert_eq!(groups, std::collections::BTreeSet::from([625])); -} diff --git a/crates/integration-tests/tests/sql_to_physical.rs b/crates/integration-tests/tests/sql_to_physical.rs deleted file mode 100644 index 258821b74..000000000 --- a/crates/integration-tests/tests/sql_to_physical.rs +++ /dev/null @@ -1,142 +0,0 @@ -//! SQL frontend, the planner pipeline's selection, physical compilation and -//! fresh-run execution. -use crate::physical_common; -use asap_executor::{ - physical_planner::compile, - runtime::{Limits, RunContext, Scope}, - sources::InMemoryDataStore, - values::{Batch, Value}, -}; -use asap_frontend_sql::{lower_sql, SqlCatalog}; -use asap_integration_tests::planning::selected_dag; -use asap_types::ir::physical_export::PhysicalASAPOperatorPayload; -use asap_types::ir::properties::AccuracyTarget; -use asap_types::ir::schema::{DataType, Field, FieldDataType, Schema}; -use futures::StreamExt; -use physical_common::compile_physical_asap_dag; -use std::sync::Arc; - -/// SQL filtering and grouped aggregation survive logical/physical lowering; -/// rebinding the compiled DAG runs against new data rather than cached results. -#[tokio::test] -async fn sql_filter_grouped_sum_executes_and_rebinds() { - let catalog = SqlCatalog::new().with_table( - "metrics", - Schema::new(vec![ - Field::plain("service", DataType::Utf8, false), - Field::plain("value", DataType::Float64, true), - ]), - ); - for query in [ - "SELECT service, SUM(value) AS total FROM metrics WHERE value > 1 GROUP BY service", - "SELECT service, SUM(value) AS total FROM metrics GROUP BY service", - ] { - let logical = lower_sql(query, &catalog, AccuracyTarget::exact()) - .await - .unwrap(); - let selected = selected_dag(logical, AccuracyTarget::exact()); - let dag = compile_physical_asap_dag(&selected).unwrap(); - let scan = dag - .nodes - .iter() - .find(|node| { - matches!( - &node.payload, - PhysicalASAPOperatorPayload::NonASAP(asap_types::ir::NonASAPOp::Scan { .. }) - ) - }) - .expect("raw SQL scan"); - let schema = Arc::new(scan.output_schema.clone()); - assert!(schema - .fields - .iter() - .all(|field| matches!(field.dtype, FieldDataType::Plain(_)))); - let plan = compile(&dag, &[], &[dag.roots[0] as u64]).unwrap(); - for multiplier in [1., 2.] { - let rows = [ - ("api", Some(2.)), - ("api", Some(3.)), - ("api", None), - ("batch", Some(4.)), - ("batch", Some(1.)), - ] - .into_iter() - .map(|(service, value)| { - schema - .fields - .iter() - .map(|field| match field.name.as_str() { - "service" => Value::Utf8(service.into()), - "value" => { - value.map_or(Value::Null, |value| Value::Float64(value * multiplier)) - } - _ => panic!("unexpected field {field:?}"), - }) - .collect() - }) - .collect(); - let PhysicalASAPOperatorPayload::NonASAP(asap_types::ir::NonASAPOp::Scan { - source, - predicates: _, - schema: _scan_schema, - }) = &scan.payload - else { - unreachable!() - }; - let store = InMemoryDataStore::default() - .with_raw_data( - source.clone(), - schema.clone(), - vec![Batch::try_new(schema.clone(), rows).unwrap()], - ) - .unwrap(); - let bound = plan.instantiate(Arc::new(store)).unwrap(); - let mut stream = bound - .execute( - plan.roots(), - RunContext::new( - Scope::Query { - evaluation_time_ms: 300_000, - }, - Limits::default(), - ) - .unwrap(), - ) - .unwrap() - .remove(0); - let mut batches = Vec::new(); - while let Some(batch) = stream.next().await { - batches.push(batch.unwrap()); - } - let mut actual: Vec<_> = batches - .iter() - .flat_map(|batch| batch.to_rows()) - .map(|row| { - let Value::Utf8(service) = &row[0] else { - panic!("missing service") - }; - let Value::Float64(value) = row[1] else { - panic!("missing sum") - }; - (service.to_string(), value) - }) - .collect(); - actual.sort_by(|a, b| a.0.cmp(&b.0)); - assert_eq!( - actual, - vec![ - ("api".into(), 5. * multiplier), - ( - "batch".into(), - 4. * multiplier - + if query.contains("WHERE") && multiplier == 1. { - 0. - } else { - multiplier - } - ) - ] - ); - } - } -} diff --git a/crates/integration-tests/tests/sql_to_post_asap.rs b/crates/integration-tests/tests/sql_to_post_asap.rs index 5f894c98e..411f5b612 100644 --- a/crates/integration-tests/tests/sql_to_post_asap.rs +++ b/crates/integration-tests/tests/sql_to_post_asap.rs @@ -30,8 +30,8 @@ use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::scalar::expr_ir::ColumnRef; use asap_types::ir::schema::{DataType, Field, Schema}; use asap_types::ir::schema::{ - ExactKind, ExactParams, FieldDataType, GroupingStrategy, SketchAlgorithm, SketchKind, - SketchParams, SketchStatistic, SummaryUpdate, + ExactKind, ExactParams, FieldDataType, GroupingStrategy, SketchAlgorithm, SketchStatistic, + SummaryUpdate, }; use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode, Predicate, ScalarExpr}; use asap_types::workload::SqlDialect; @@ -593,16 +593,7 @@ async fn sql_quantile_binds_kll_sketch_over_named_column() { }; assert_eq!( family, - &FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: 269, - failure_probability: None - } - ), - GroupingStrategy::default() - ) + &FieldDataType::Sketch(SketchAlgorithm::Kll, GroupingStrategy::default()) ); assert_eq!( input, @@ -619,16 +610,7 @@ async fn sql_quantile_binds_kll_sketch_over_named_column() { ); assert_eq!( summary_input.schema.fields[0].dtype, - FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: 269, - failure_probability: None - } - ), - GroupingStrategy::default() - ) + FieldDataType::Sketch(SketchAlgorithm::Kll, GroupingStrategy::default()) ); assert!( @@ -690,16 +672,7 @@ async fn sql_count_distinct_with_epsilon_binds_hll_over_named_column() { }; assert_eq!( family, - &FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Hll, - SketchParams::Hll { - precision: 18, - relative_error: Some(0.01) - } - ), - GroupingStrategy::default() - ) + &FieldDataType::Sketch(SketchAlgorithm::Hll, GroupingStrategy::default()) ); assert_eq!( input, diff --git a/crates/integration-tests/tests/sql_trailing_window.rs b/crates/integration-tests/tests/sql_trailing_window.rs deleted file mode 100644 index 81a229329..000000000 --- a/crates/integration-tests/tests/sql_trailing_window.rs +++ /dev/null @@ -1,246 +0,0 @@ -//! A SQL trailing window, `WHERE ts >= now() - INTERVAL 'w'` on the table's -//! time index, plans like a PromQL range selector `m[w]`: Pass 2 builds its -//! summaries in tumbling windows, Stage 2 maintains them at ingestion time, -//! a deployment that keeps no raw data still gets a plan, and Stage 3 prices -//! each scan by its window. - -use crate::planner_layering_common; - -use asap_cost::CostInputs; -use asap_frontend_sql::{lower_sql_dialect, SqlCatalog}; -use asap_logical_asap_optimization::DeploymentCapabilities; -use asap_types::ir::properties::{AccuracyMetric, AccuracyScope, AccuracyTarget}; -use asap_types::ir::schema::{DataType, Field, Schema}; -use asap_types::ir::{NonASAPOp, Operator, QueryRoot}; -use asap_types::workload::{ - DataArrival, DataWorkload, PlanningWorkload, Predictability, Query, QueryLanguage, - QueryRequirements, QueryWorkload, RepeatedDemand, RepeatingEntry, RepetitionInterval, - SourceFacts, SqlDialect, -}; -use planner_layering_common::*; - -/// The 5 and 15 minute distinct counts of W-RollupHeavy's template 9. -const QUERIES: [&str; 2] = [ - "SELECT dst_subnet, COUNT(DISTINCT src_addr) AS value FROM flows \ - WHERE ts >= now() - INTERVAL '5 minutes' GROUP BY dst_subnet", - "SELECT dst_subnet, COUNT(DISTINCT src_addr) AS value FROM flows \ - WHERE ts >= now() - INTERVAL '15 minutes' GROUP BY dst_subnet", -]; -const WINDOWS_MS: [u64; 2] = [300_000, 900_000]; - -/// `flows(ts, dst_subnet, src_addr)`, time index `ts`. -fn catalog() -> SqlCatalog { - SqlCatalog::new().with_table( - "flows", - Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("dst_subnet", DataType::Utf8, false), - Field::plain("src_addr", DataType::Float64, false), - ], - 0, - vec![], - ), - ) -} - -/// Both queries every 60 s, relative ε = 0.05 per value, over 1,000 rows/s. -fn workload() -> PlanningWorkload { - let entry = |query: &str| RepeatingEntry { - query: Query(query.into()), - demand: RepeatedDemand::FixedInterval(RepetitionInterval(60_000)), - requirements: QueryRequirements { - accuracy: AccuracyTarget { - accuracy_metric: Some(AccuracyMetric::Relative), - accuracy_scope: AccuracyScope::PerValue, - ..AccuracyTarget::max_error(0.05).failure_probability(0.05) - }, - max_latency_ms: None, - }, - predictability: Predictability::Predictable { known_at: None }, - range_evaluation: None, - }; - PlanningWorkload { - query_workload: QueryWorkload { - language: QueryLanguage::SQL(SqlDialect::DataFusionSQL), - query_batch: None, - repeating_queries: Some(QUERIES.iter().map(|q| entry(q)).collect()), - }, - data_workload: Some(DataWorkload { - arrival: DataArrival::Ingesting { - rows_per_second: 1_000.0, - }, - sources: [( - "flows".to_string(), - SourceFacts { - max_distinct_per_evaluation: Some(1_000_000), - ..SourceFacts::default() - }, - )] - .into(), - }), - } -} - -async fn roots(workload: &PlanningWorkload) -> Vec { - let mut roots = Vec::new(); - for entry in workload.query_workload.entries() { - let root = lower_sql_dialect( - &entry.query.0, - &catalog(), - SqlDialect::DataFusionSQL, - entry.requirements.accuracy.clone(), - ) - .await - .unwrap_or_else(|e| panic!("{}: {e}", entry.query.0)); - roots.push(QueryRoot::Operator(root)); - } - roots -} - -async fn run(deployment: &DeploymentCapabilities) -> Run { - let workload = workload(); - let roots = roots(&workload).await; - use asap_logical_asap_optimization::test_support::{ - compose_logical_candidate, generate_per_computation_candidates, - }; - use asap_logical_asap_optimization::test_support::{ - tumbling_window_candidates, window_composition_pass2_candidates, - }; - use asap_types::stages::HowToComputeAggregateMeasure; - let local = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&roots), - ) - .unwrap(); - let compose = |choices: &[usize]| compose_logical_candidate(&local, choices).unwrap(); - let exact = compose(&vec![0; local.targets.len()]); - let choices = local - .targets - .iter() - .map(|target| { - target - .ways_to_compute - .iter() - .position(|choice| { - matches!( - choice, - HowToComputeAggregateMeasure::Summary( - asap_types::ir::schema::SketchAlgorithm::Kmv - ) - ) - }) - .expect("KMV distinct-count sketch with a failure bound") - }) - .collect::>(); - let summary = compose(&choices); - let grids = workload - .query_workload - .entries() - .map(|entry| entry.evaluation_grid_ms()) - .collect::>(); - let tumbling_windows = tumbling_window_candidates(&summary, &grids); - assert_eq!(tumbling_windows.len(), WINDOWS_MS.len()); - let mut candidates = vec![exact, summary.clone()]; - candidates.extend( - window_composition_pass2_candidates(&summary, &tumbling_windows) - .unwrap() - .into_iter() - .filter(|candidate| { - candidate.tumbling_window_candidates.len() == tumbling_windows.len() - }) - .map(|candidate| candidate.roots.unwrap()), - ); - let settings = CostInputs::default(); - run_explicit_candidates( - &workload, - roots, - candidates, - asap_planner::PlanningModels::builtin() - .with_cost_inputs(&settings) - .with_deployment_capabilities(deployment), - ) -} - -/// Pass 2 offers each query's distinct-count sketch in tumbling windows, and -/// Stage 2 maintains a tumbling-window chain at ingestion time, as for `m[5m]`. -#[tokio::test] -async fn sql_windows_get_tumbling_windows_and_ingestion_time_builds() { - let run = run(&DeploymentCapabilities::default()).await; - for (query, window_ms) in WINDOWS_MS.iter().enumerate() { - let tumbling = run.logical.iter().any(|c| { - sketch_builds(&c.dag).iter().any(|(build, _, _)| { - readers(&c.dag, &c.query_roots, *build).contains(&query) - && matches!(window_form(&c.dag, *build), WindowForm::Tumbling { .. }) - }) - }); - assert!( - tumbling, - "query {query} ({window_ms} ms) has no tumbling-window candidate" - ); - } - let at_ingestion = run.physical.iter().any(|p| { - p.dag().nodes.iter().any(|n| { - matches!( - n.payload, - Operator::ASAP(asap_types::ir::ASAPOp::SummaryTumblingWindowChain { .. }) - ) && runs_at_ingestion(p, n.id) - }) - }); - assert!( - at_ingestion, - "no tumbling-window chain maintained at ingestion time" - ); -} - -/// Without raw data at query time, the ingestion-time builds still answer -/// both queries, so the workload has a plan (`run_stages_with` panics when -/// none is selected) and it reads no raw scan at query time. -#[tokio::test] -async fn sql_windows_have_a_plan_without_raw_data() { - let deployment = DeploymentCapabilities { - raw_data_retained: false, - ..DeploymentCapabilities::default() - }; - let run = run(&deployment).await; - let selected = run.physical(&run.selection.selected); - for n in &selected.dag().nodes { - if matches!(n.payload, Operator::NonASAP(NonASAPOp::Scan { .. })) { - assert!(runs_at_ingestion(selected, n.id), "{}", selected.id); - } - } -} - -/// Each query-time scan is priced by the window above it: the 15 minute -/// query's scan reads 3x the rows of the 5 minute one. -#[tokio::test] -async fn sql_scans_are_priced_by_their_window() { - let run = run(&DeploymentCapabilities::default()).await; - let mut scanned_rows = std::collections::BTreeSet::new(); - for (id, cost) in &run.selection.costs { - let p = run.physical(id); - for (node, node_cost) in &cost.per_node { - let is_query_time_scan = p.dag().nodes.iter().any(|n| { - n.id == *node - && matches!(n.payload, Operator::NonASAP(NonASAPOp::Scan { .. })) - && !runs_at_ingestion(p, n.id) - }); - if !is_query_time_scan { - continue; - } - let rows: u64 = node_cost - .detail - .strip_prefix("scan ") - .and_then(|d| d.split_whitespace().next()) - .and_then(|n| n.parse().ok()) - .unwrap_or_else(|| panic!("scan detail: {}", node_cost.detail)); - scanned_rows.insert(rows); - } - } - // 1 row per ms over each window. - for window_ms in WINDOWS_MS { - assert!( - scanned_rows.contains(&window_ms), - "no scan of {window_ms} rows: {scanned_rows:?}" - ); - } -} diff --git a/crates/integration-tests/tests/sql_two_percentiles.rs b/crates/integration-tests/tests/sql_two_percentiles.rs deleted file mode 100644 index 6ae9ff645..000000000 --- a/crates/integration-tests/tests/sql_two_percentiles.rs +++ /dev/null @@ -1,555 +0,0 @@ -//! Repeating SQL percentiles over one table, every 10 s at ε = 0.01. Q1, -//! the overall p50 and p90 over the last 5 minutes: every candidate that -//! computes one of its percentiles exactly and the other from a sketch -//! compiles, is priced, and answers the exact one as `percentile_cont` does -//! and the sketched one within the sketch's bound. With Q2, the p90 per -//! city over the last 20 minutes, Q1 and Q2 share the table scan, and Q2's -//! per-city tumbling-window chain serves Q1 through its newest tumbling windows rolled up across -//! cities. - -use crate::physical_common; -use crate::planner_layering_common; - -use asap_cost::CostInputs; -use asap_executor::values::Value; -use asap_frontend_sql::{lower_sql_dialect, SqlCatalog}; -use asap_types::ir::operator::AggIntent; -use asap_types::ir::schema::{DataType, Field, FieldDataType, Schema, SketchAlgorithm}; -use asap_types::ir::{ASAPOp, NonASAPOp, OperatorNode, QueryRoot}; -use asap_types::workload::{PlanningWorkload, SqlDialect}; -use planner_layering_common::*; -use std::rc::Rc; - -const Q1: &str = "SELECT percentile_cont(0.5) WITHIN GROUP (ORDER BY value), \ - percentile_cont(0.9) WITHIN GROUP (ORDER BY value) FROM metrics \ - WHERE ts > now() - INTERVAL '5' MINUTE"; -const Q2: &str = "SELECT city, percentile_cont(0.9) WITHIN GROUP (ORDER BY value) \ - FROM metrics WHERE ts > now() - INTERVAL '20' MINUTE GROUP BY city"; -/// Every query's accuracy target. -const EPSILON: f64 = 0.01; -const MINUTE_MS: i64 = 60_000; -/// The evaluation time of every run. -const EVALUATION_TIME_MS: i64 = 1_800_000_000_000; - -/// `metrics(ts, city, value)`, time index `ts`. -fn catalog() -> SqlCatalog { - SqlCatalog::new().with_table( - "metrics", - Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("city", DataType::Utf8, true), - Field::plain("value", DataType::Float64, true), - ], - 0, - vec![], - ), - ) -} - -/// `queries`, each every 10 s at ε = [`EPSILON`], over `metrics` ingesting -/// 666.67 rows per second. -fn workload(queries: &[&str]) -> PlanningWorkload { - let entry = |query: &&str| { - format!( - r#"{{"query": {query:?}, "demand": {{"fixed_interval": 10000}}, - "requirements": {{"accuracy": {{"max_error": {EPSILON}, "failure_probability": 0.001}}}}, - "predictability": {{"predictable": {{"known_at": null}}}}}}"# - ) - }; - serde_json::from_str(&format!( - r#"{{"query_workload": {{"language": {{"sql": "datafusion_sql"}}, "query_batch": null, - "repeating_queries": [{}]}}, - "data_workload": {{"arrival": {{"ingesting": {{"rows_per_second": 666.67}}}}, - "sources": {{"metrics": {{"series_count": 100}}}}}}}}"#, - queries.iter().map(entry).collect::>().join(", ") - )) - .unwrap() -} - -/// Every stage over `queries`. -async fn run(queries: &[&str]) -> Run { - let workload = workload(queries); - let mut roots = Vec::new(); - for entry in workload.query_workload.entries() { - let root = lower_sql_dialect( - &entry.query.0, - &catalog(), - SqlDialect::DataFusionSQL, - entry.requirements.accuracy.clone(), - ) - .await - .unwrap_or_else(|e| panic!("{}: {e}", entry.query.0)); - roots.push(QueryRoot::Operator(root)); - } - // Every output accepted whatever its guarantee: this test checks the - // answers against the bound. - let settings = CostInputs { - ..CostInputs::default() - }; - let candidates = stage1(queries) - .await - .into_iter() - .map(|(_, roots)| roots) - .collect(); - run_explicit_candidates( - &workload, - roots, - candidates, - asap_planner::PlanningModels::builtin() - .with_cost_inputs(&settings) - .with_ignore_accuracy_guarantees(true), - ) -} - -/// Two rows a second over the 20 minutes before the evaluation: city -/// `c{i % 3}`, or none for every seventh row, and a heavy-tailed value from -/// a fixed linear congruential generator. -fn rows() -> Vec<(i64, Option, f64)> { - let mut state: u64 = 0x2545_f491_4f6c_dd1d; - let mut next = move || { - state = state - .wrapping_mul(6_364_136_223_846_793_005) - .wrapping_add(1_442_695_040_888_963_407); - (state >> 33) as f64 / (1u64 << 31) as f64 - }; - (0..20 * 60 * 2) - .map(|i: i64| { - let ts = EVALUATION_TIME_MS - 20 * MINUTE_MS + (i + 1) * 500; - let city = (i % 7 != 0).then(|| format!("c{}", i % 3)); - (ts, city, 10.0 / (1.0 - next()).max(1e-6).sqrt()) - }) - .collect() -} - -/// The values of [`rows`] in `(T - window, T]` of the city `city` (every -/// city for `None`), sorted. -fn values_in(window_ms: i64, city: Option<&str>) -> Vec { - let mut values: Vec = rows() - .into_iter() - .filter(|(ts, row_city, _)| { - *ts > EVALUATION_TIME_MS - window_ms - && city.is_none_or(|c| row_city.as_deref() == Some(c)) - }) - .map(|(_, _, value)| value) - .collect(); - values.sort_by(f64::total_cmp); - values -} - -/// SQL `percentile_cont` of sorted `values`. -fn percentile_cont(q: f64, values: &[f64]) -> f64 { - let rank = q * (values.len() - 1) as f64; - let (low, high) = (rank.floor() as usize, rank.ceil() as usize); - values[low] + (values[high] - values[low]) * (rank - low as f64) -} - -/// Whether `answer` is the `q` percentile of sorted `values` exactly, or -/// within `algorithm`'s bound: a KLL's rank error, a DDSketch's relative -/// value error. -fn within_bound(algorithm: Option<&SketchAlgorithm>, q: f64, answer: f64, values: &[f64]) -> bool { - let exact = percentile_cont(q, values); - if (answer - exact).abs() <= 1e-9 * exact.abs() { - return true; - } - match algorithm { - Some(SketchAlgorithm::Kll) => { - let n = values.len() as f64; - let below = values.partition_point(|&v| v < answer) as f64 / n; - let at_most = values.partition_point(|&v| v <= answer) as f64 / n; - below - EPSILON - 1.0 / n <= q && q <= at_most + EPSILON + 1.0 / n - } - Some(SketchAlgorithm::DDSketch) => (answer - exact).abs() <= EPSILON * exact.abs(), - _ => false, - } -} - -/// `root`'s answer rows at [`EVALUATION_TIME_MS`] over [`rows`], every node at -/// query time, each tumbling-window chain as its tumbling windows. -fn answers(root: &Rc) -> Vec> { - answers_over(root, rows()) -} - -/// [`answers`] over the table rows `table` (time, city, value). -fn answers_over( - root: &Rc, - table: Vec<(i64, Option, f64)>, -) -> Vec> { - let table = table - .into_iter() - .map(|(ts, city, value)| { - vec![ - Value::Timestamp(ts), - city.map_or(Value::Null, |city| Value::Utf8(city.into())), - Value::Float64(value), - ] - }) - .collect(); - // The executor runs a tumbling-window chain as its tumbling windows. - let root = asap_types::ir::operator::tumbling_window_chain::expand_tumbling_window_chains(root) - .unwrap(); - physical_common::execute_raw_rows_at(&root, table, EVALUATION_TIME_MS) -} - -fn as_f64(value: &Value) -> f64 { - match value { - Value::Float64(value) => *value, - other => panic!("not a Float64: {other:?}"), - } -} - -/// The sketch algorithm `root` reads, if any. -fn sketch_of(root: &Rc) -> Option { - OperatorNode::reachable(root) - .into_iter() - .find_map(|node| match node.asap() { - Some( - ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), - .. - } - | ASAPOp::SummaryTumblingWindowChain { - family: FieldDataType::Sketch(kind, _), - .. - }, - ) => Some(*kind.algorithm()), - _ => None, - }) -} - -fn operator(root: &QueryRoot) -> &Rc { - match root { - QueryRoot::Operator(node) => node, - QueryRoot::Scalar(_) => panic!("operator root expected"), - } -} - -/// Q1 with one percentile exact and the other from a sketch: every such -/// plan compiles and is priced (an exact aggregate split from a sketched -/// measure names its one output), and answers each percentile exactly or -/// within the sketch's bound. -#[tokio::test] -async fn q1_with_one_exact_and_one_sketched_percentile_compiles_and_matches() { - let run = run(&[Q1]).await; - let mixed: Vec<&Logical> = run - .logical - .iter() - .filter(|c| { - let reached = OperatorNode::reachable(operator(&c.roots[0])); - reached - .iter() - .any(|n| matches!(n.non_asap(), Some(NonASAPOp::Aggregate { .. }))) - && reached - .iter() - .any(|n| matches!(n.asap(), Some(ASAPOp::SummaryEstimate { .. }))) - }) - .collect(); - assert!(!mixed.is_empty(), "no mixed exact and sketch plan for Q1"); - for candidate in &mixed { - assert!( - run.physical_of(candidate) - .any(|p| run.cost(&p.id).is_some()), - "{}: no physical candidate priced: {:#?}", - candidate.id, - run.selection - .rejected - .iter() - .filter(|r| run.physical_of(candidate).any(|p| p.id == r.id)) - .map(|r| format!("{}: {:?} {}", r.id, r.cause, r.reason)) - .collect::>() - ); - } - let values = values_in(5 * MINUTE_MS, None); - for algorithm in [SketchAlgorithm::Kll, SketchAlgorithm::DDSketch] { - let candidate = mixed - .iter() - .find(|c| sketch_of(operator(&c.roots[0])) == Some(algorithm)) - .unwrap_or_else(|| panic!("no mixed {algorithm:?} plan")); - let root = operator(&candidate.roots[0]); - // The percentile the exact aggregate computes. - let exact_q = OperatorNode::reachable(root) - .iter() - .find_map(|n| match n.non_asap() { - Some(NonASAPOp::Aggregate { measures, .. }) => match measures.as_slice() { - [AggIntent::Quantile { q, .. }] => Some(*q), - _ => None, - }, - _ => None, - }) - .expect("one exact percentile"); - let rows = answers(root); - assert_eq!(rows.len(), 1, "{}", candidate.id); - for (column, q) in [0.5, 0.9].into_iter().enumerate() { - let answer = as_f64(&rows[0][column]); - let bound = (q != exact_q).then_some(&algorithm); - assert!( - within_bound(bound, q, answer, &values), - "{}: p{} = {answer}, exact {}", - candidate.id, - q * 100.0, - percentile_cont(q, &values) - ); - } - } -} - -/// The id and query roots of every Stage 1 candidate of `queries`. -async fn stage1(queries: &[&str]) -> Vec<(String, Vec)> { - use asap_logical_asap_optimization::test_support::{ - compose_logical_candidate, generate_per_computation_candidates, - }; - use asap_logical_asap_optimization::test_support::{ - tumbling_window_candidates, window_composition_pass2_candidates, - }; - use asap_types::stages::HowToComputeAggregateMeasure; - let workload = workload(queries); - let entries: Vec<_> = workload.query_workload.entries().collect(); - let mut roots = Vec::new(); - for entry in &entries { - roots.push(QueryRoot::Operator( - lower_sql_dialect( - &entry.query.0, - &catalog(), - SqlDialect::DataFusionSQL, - entry.requirements.accuracy.clone(), - ) - .await - .unwrap(), - )); - } - let local = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&roots), - ) - .unwrap(); - // Explicit fixture configurations: exact; one local substitution at a - // time (mixed percentiles); and all quantiles using each supported family. - // Do not enumerate a Cartesian product of complete workload choices. - let mut choices = std::collections::BTreeSet::from([vec![0; local.targets.len()]]); - for (target, options) in local.targets.iter().enumerate() { - for option in 1..options.choice_count() { - let mut choice = vec![0; local.targets.len()]; - choice[target] = option; - choices.insert(choice); - } - } - for algorithm in [SketchAlgorithm::Kll, SketchAlgorithm::DDSketch] { - let together: Option> = local.targets.iter().map(|target| target.ways_to_compute.iter() - .position(|candidate| matches!(candidate, HowToComputeAggregateMeasure::Summary(family) if *family == algorithm))).collect(); - if let Some(choice) = together { - choices.insert(choice); - } - } - let share = |roots: Vec| { - asap_logical_asap_optimization::test_support::share_common_sub_dags( - roots - .into_iter() - .enumerate() - .map(|(id, root)| match root { - QueryRoot::Operator(node) => (id, node), - _ => unreachable!(), - }) - .collect(), - ) - .into_iter() - .map(|(_, node)| QueryRoot::Operator(node)) - .collect::>() - }; - let grids = entries - .iter() - .map(|entry| entry.evaluation_grid_ms()) - .collect::>(); - let mut candidates = Vec::new(); - for choice in choices { - let roots = share(compose_logical_candidate(&local, &choice).unwrap()); - candidates.push(roots.clone()); - let tumbling_windows = tumbling_window_candidates(&roots, &grids); - if !tumbling_windows.is_empty() { - candidates.extend( - window_composition_pass2_candidates(&roots, &tumbling_windows) - .unwrap() - .into_iter() - .filter(|candidate| { - candidate.tumbling_window_candidates.len() == tumbling_windows.len() - }) - .filter_map(|candidate| candidate.roots.ok()) - .map(share), - ); - } - } - candidates - .into_iter() - .enumerate() - .map(|(index, roots)| (format!("explicit-{}", index + 1), roots)) - .collect() -} - -/// Q1 and Q2 read the same rows of `metrics`: a candidate reads them with -/// one scan. A table scan has no unique key, and is shared all the same: -/// each reader reads every row it outputs. -#[tokio::test] -async fn q1_and_q2_share_the_table_scan() { - let scans = |root: &QueryRoot| -> Vec<*const OperatorNode> { - OperatorNode::reachable(operator(root)) - .iter() - .filter(|n| matches!(n.non_asap(), Some(NonASAPOp::Scan { .. }))) - .map(Rc::as_ptr) - .collect() - }; - let candidates = stage1(&[Q1, Q2]).await; - assert!( - candidates.iter().any(|(_, roots)| { - let q2_scans = scans(&roots[1]); - scans(&roots[0]).iter().any(|scan| q2_scans.contains(scan)) - }), - "no candidate shares the scan" - ); -} - -/// Whether Q2's per-city tumbling-window chain (120 tumbling windows of 10 s) serves both of -/// Q1's percentiles in `roots`: Q1 merges each city's newest 30 tumbling windows (its -/// 5 minutes), then rolls the cities up into one state, which both of its -/// estimates read. -fn newest_tumbling_windows_rolled_up(roots: &[QueryRoot]) -> bool { - let (q1, q2) = (operator(&roots[0]), operator(&roots[1])); - let q2_chains: Vec<*const OperatorNode> = OperatorNode::reachable(q2) - .iter() - .filter(|n| { - matches!( - n.asap(), - Some(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_count: 120, - .. - }) - ) - }) - .map(Rc::as_ptr) - .collect(); - let rollups: Vec> = OperatorNode::reachable(q1) - .into_iter() - .filter(|n| match n.asap() { - Some(ASAPOp::SummaryMerge { - children, - group_by: Some(keys), - newest_tumbling_windows: None, - }) => { - keys.is_empty() - && matches!( - children[0].asap(), - Some(ASAPOp::SummaryMerge { - children: chain, - newest_tumbling_windows: Some(30), - group_by: None, - }) if q2_chains.contains(&Rc::as_ptr(&chain[0])) - ) - } - _ => false, - }) - .collect(); - OperatorNode::reachable(q1) - .iter() - .filter(|n| match n.asap() { - Some(ASAPOp::SummaryEstimate { summary_input, .. }) => { - rollups.iter().any(|r| Rc::ptr_eq(r, summary_input)) - } - _ => false, - }) - .count() - == 2 -} - -/// The values of [`rows`] of city `city` (no city for `None`) in the last -/// 20 minutes, sorted. -fn city_values(city: Option<&str>) -> Vec { - let mut values: Vec = rows() - .into_iter() - .filter(|(ts, row_city, _)| { - *ts > EVALUATION_TIME_MS - 20 * MINUTE_MS && row_city.as_deref() == city - }) - .map(|(_, _, value)| value) - .collect(); - values.sort_by(f64::total_cmp); - values -} - -/// Q2's per-city chain serves both of Q1's percentiles through its newest -/// 30 tumbling windows rolled up across cities. Each query's answers are within the -/// sketch's bound of the exact `percentile_cont`, and over a window with -/// no row Q1 still answers one NULL row, as the global SQL aggregate the -/// rollup stands for. -#[tokio::test] -async fn q2_tumbling_window_chain_serves_q1_through_newest_tumbling_windows_and_a_rollup() { - let candidates = stage1(&[Q1, Q2]).await; - for algorithm in [SketchAlgorithm::Kll, SketchAlgorithm::DDSketch] { - let (id, roots) = candidates - .iter() - .find(|(_, roots)| { - newest_tumbling_windows_rolled_up(roots) - && sketch_of(operator(&roots[0])) == Some(algorithm) - }) - .unwrap_or_else(|| panic!("no {algorithm:?} newest-tumbling-windows rollup candidate")); - let q1 = answers(operator(&roots[0])); - let values = values_in(5 * MINUTE_MS, None); - assert_eq!(q1.len(), 1, "{id}"); - for (column, q) in [0.5, 0.9].into_iter().enumerate() { - let answer = as_f64(&q1[0][column]); - assert!( - within_bound(Some(&algorithm), q, answer, &values), - "{id}: Q1 p{} = {answer}, exact {}", - q * 100.0, - percentile_cont(q, &values) - ); - } - let older = rows() - .into_iter() - .filter(|(ts, _, _)| *ts <= EVALUATION_TIME_MS - 5 * MINUTE_MS) - .collect(); - let empty = answers_over(operator(&roots[0]), older); - assert!( - matches!(empty.as_slice(), [row] if matches!(row.as_slice(), [Value::Null, Value::Null])), - "{id}: {empty:?}" - ); - let q2 = answers(operator(&roots[1])); - assert_eq!(q2.len(), 4, "{id}: three cities and none"); - for row in q2 { - let city = match &row[0] { - Value::Utf8(city) => Some(city.to_string()), - Value::Null => None, - other => panic!("city: {other:?}"), - }; - let values = city_values(city.as_deref()); - let answer = as_f64(&row[1]); - assert!( - within_bound(Some(&algorithm), 0.9, answer, &values), - "{id}: Q2 p90 of {city:?} = {answer}, exact {}", - percentile_cont(0.9, &values) - ); - } - } -} - -/// A global SQL `COUNT(*)` is 0 over no rows, but a rollup of the per-city -/// counts would sum no count into NULL. A rollup to no keys over a table is -/// offered, yet no candidate answers the global count from the per-city -/// counts: an exact rollup never sums Int64 counts again. -#[tokio::test] -async fn a_global_count_is_not_rolled_up_from_grouped_counts() { - let global = "SELECT COUNT(*) FROM metrics WHERE ts > now() - INTERVAL '5' MINUTE"; - let by_city = "SELECT city, COUNT(*) FROM metrics \ - WHERE ts > now() - INTERVAL '5' MINUTE GROUP BY city"; - let finalized = |root: &QueryRoot| -> Vec<*const OperatorNode> { - OperatorNode::reachable(operator(root)) - .iter() - .filter(|n| matches!(n.asap(), Some(ASAPOp::FinalizeExactAccumulator { .. }))) - .map(Rc::as_ptr) - .collect() - }; - for (id, roots) in stage1(&[global, by_city]).await { - let by_city_values = finalized(&roots[1]); - assert!( - !finalized(&roots[0]) - .iter() - .any(|value| by_city_values.contains(value)), - "{id}: the global count reads the per-city counts" - ); - } -} diff --git a/crates/integration-tests/tests/stage1_logical_asap_optimization.rs b/crates/integration-tests/tests/stage1_logical_asap_optimization.rs index 25117bb35..fab612462 100644 --- a/crates/integration-tests/tests/stage1_logical_asap_optimization.rs +++ b/crates/integration-tests/tests/stage1_logical_asap_optimization.rs @@ -30,8 +30,6 @@ use asap_types::workload::{ }; use futures::StreamExt; -use crate::physical_common; - fn accuracy() -> AccuracyTarget { AccuracyTarget::max_error(0.01).failure_probability(0.01) } @@ -100,7 +98,7 @@ fn output_columns(root: &QueryRoot) -> Vec<(String, String, bool)> { fn compose_all(label: &str, root: QueryRoot) -> usize { let expected = output_columns(&root); let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&[root]), + &asap_logical_asap_optimization::memo_of_queries(&[root]).unwrap(), ) .unwrap(); let radix: Vec<_> = inventory @@ -179,7 +177,7 @@ fn promql_candidates_compose() { /// choice, every other target taking its first. fn composed_estimates(root: QueryRoot) -> Vec> { let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&[root]), + &asap_logical_asap_optimization::memo_of_queries(&[root]).unwrap(), ) .unwrap(); let targets = inventory.targets.len(); @@ -220,7 +218,7 @@ fn update_of(estimate: &OperatorNode) -> (SketchAlgorithm, SummaryUpdate) { else { panic!("sketch state expected") }; - (*kind.algorithm(), input.clone()) + (*kind, input.clone()) } /// A heap sketch's bound is per item frequency, not Top-K membership, and @@ -269,7 +267,7 @@ async fn subquery_in_a_measure_filter_is_substituted() { let query = "SELECT src, count(*) FILTER (WHERE bytes > (SELECT approx_percentile_cont(bytes, 0.5) FROM flows)) \ FROM flows GROUP BY src"; let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&[sql(query).await]), + &asap_logical_asap_optimization::memo_of_queries(&[sql(query).await]).unwrap(), ) .unwrap(); assert_eq!(inventory.targets.len(), 2); @@ -325,7 +323,7 @@ async fn subquery_in_a_measure_filter_is_substituted() { /// Run the logical DAG `root` over `rows` of its one scanned table, each row /// given by column name. Rows come back shown and sorted. fn execute(root: &Rc, rows: &[Vec<(&str, Value)>]) -> Vec> { - let dag = physical_common::compile_physical_asap_dag(root).unwrap(); + let dag = asap_types::test_support::physical_asap_dag_at_query_time(root).unwrap(); let scan = dag .nodes .iter() @@ -403,9 +401,8 @@ async fn avg_as_sum_and_count( ) -> (Rc, Rc) { let exact = lower_sql(query, catalog, accuracy()).await.unwrap(); let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(exact.clone()), - ]), + &asap_logical_asap_optimization::memo_of_queries(&[QueryRoot::Operator(exact.clone())]) + .unwrap(), ) .unwrap(); let choice: Vec<_> = inventory @@ -539,9 +536,7 @@ async fn filtered_sql_avg_is_null_without_matching_rows() { #[test] fn promql_avg_divides_sum_and_count_vectors() { let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&[promql( - "avg by (job) (m)", - )]), + &asap_logical_asap_optimization::memo_of_queries(&[promql("avg by (job) (m)")]).unwrap(), ) .unwrap(); assert_eq!( @@ -583,9 +578,7 @@ async fn every_candidate( )> { let root = lower_sql(query, catalog, accuracy()).await.unwrap(); let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(root), - ]), + &asap_logical_asap_optimization::memo_of_queries(&[QueryRoot::Operator(root)]).unwrap(), ) .unwrap(); assert_eq!(inventory.targets.len(), 1, "{query}"); @@ -695,46 +688,6 @@ async fn sql_avg_of_an_int64_column_keeps_the_exact_columns() { assert_eq!(execute(&exact, &rows), want); } -/// `count(*)` counts a non-null column, never a nullable one; executed, it -/// counts the rows whose nullable column is NULL too. -#[tokio::test] -async fn sql_count_star_counts_a_non_null_column() { - let catalog = table(&[("v", true), ("n", false)]); - let query = "SELECT k, count(*) FROM t GROUP BY k"; - for (candidate, composed) in every_candidate(&catalog, query).await { - let composed = composed.unwrap_or_else(|e| panic!("{candidate:?}: {e}")); - composed[0].validate_structure().unwrap(); - let updates = count_updates(&composed[0]); - assert_eq!(updates.len(), 1, "{candidate:?}"); - assert!(reads_only(&updates[0], "n"), "{candidate:?}: {updates:?}"); - } - let (exact, root) = { - let exact = lower_sql(query, &catalog, accuracy()).await.unwrap(); - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(exact.clone()), - ]), - ) - .unwrap(); - let QueryRoot::Operator(root) = compose_logical_candidate(&inventory, &[0]) - .unwrap() - .remove(0) - else { - panic!("operator root expected") - }; - (exact, root) - }; - let row = |k: &str, v: Option| { - vec![ - ("k", Value::Utf8(k.into())), - ("v", v.map_or(Value::Null, Value::Float64)), - ("n", Value::Float64(0.)), - ] - }; - let rows = vec![row("a", None), row("a", Some(1.)), row("b", None)]; - assert_eq!(execute(&root, &rows), execute(&exact, &rows)); -} - /// `count(*)` with no column known to be non-null (one nullable numeric /// column, or several and none named `value`) is unsupported for every /// candidate, not a count of a nullable column or an invalid DAG. diff --git a/crates/integration-tests/tests/stage2_physical_asap_optimization_sql.rs b/crates/integration-tests/tests/stage2_physical_asap_optimization_sql.rs deleted file mode 100644 index dbe386cac..000000000 --- a/crates/integration-tests/tests/stage2_physical_asap_optimization_sql.rs +++ /dev/null @@ -1,202 +0,0 @@ -//! SQL heavy-hitter queries through Stage 1 (every Pass 1 choice composed) -//! and physical ASAP-aware optimization: exact top-k sorts by the ranked -//! measure, descending, whatever the group keys are called or typed, no -//! valid candidate is pruned, and every candidate answers the ranked value -//! then the group keys naming each ranked item. - -use asap_frontend_sql::{lower_sql, SqlCatalog}; -use asap_logical_asap_optimization::test_support::{ - compose_logical_candidate, generate_per_computation_candidates, -}; -use asap_physical_asap_optimization::optimize_physical_asap; -use asap_types::ir::operator::AggIntent; -use asap_types::ir::properties::AccuracyTarget; -use asap_types::ir::schema::{DataType, Field, Schema}; -use asap_types::ir::{NonASAPOp, OperatorNode, QueryRoot, ScalarExpr}; -use asap_types::stages::HowToComputeAggregateMeasure; - -fn approximate() -> AccuracyTarget { - AccuracyTarget::max_error(0.01).failure_probability(0.001) -} - -fn names(node: &OperatorNode) -> Vec { - node.schema.fields.iter().map(|f| f.name.clone()).collect() -} - -/// The `(sort column name, ascending)` of an exact top-k's `Sort`, if `root` -/// is one (a heavy-hitter summary candidate has none). -fn exact_topk_sort(root: &OperatorNode) -> Option<(String, bool)> { - let NonASAPOp::Project { child: limit, .. } = root.non_asap()? else { - return None; - }; - let NonASAPOp::Limit { - n: Some(5), - child: sort, - .. - } = limit.non_asap()? - else { - return None; - }; - let NonASAPOp::Sort { keys, .. } = sort.expect_non_asap() else { - panic!("sort below limit") - }; - let [key] = keys.as_slice() else { - panic!("one sort key") - }; - let ScalarExpr::Column(column) = key.expr else { - panic!("sort by a column") - }; - Some((sort.schema.fields[column].name.clone(), key.ascending)) -} - -/// Every Pass 1 choice of `sql` under `accuracy`, composed and physically -/// optimized: each must succeed, every exact top-k must sort by `ranked` -/// descending, and every root must output `root_names`. Returns how many -/// candidates had an exact top-k. -async fn check_every_choice( - sql: &str, - accuracy: AccuracyTarget, - ranked: &str, - root_names: &[&str], -) -> usize { - let catalog = SqlCatalog::new().with_table( - "metrics", - Schema::new(vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("value", DataType::Float64, false), - Field::plain("host", DataType::Utf8, false), - Field::plain("port", DataType::Int64, false), - ]), - ); - let root = lower_sql(sql, &catalog, accuracy).await.unwrap(); - let NonASAPOp::Aggregate { measures, .. } = root.expect_non_asap() else { - panic!("{sql}: heavy-hitter aggregate root") - }; - assert!( - matches!(measures.as_slice(), [AggIntent::TopK { k: 5, .. }]), - "{sql}" - ); - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(root), - ]), - ) - .unwrap(); - // The cartesian product of every target's candidates. - let mut choices = vec![vec![]]; - for target in &inventory.targets { - choices = choices - .into_iter() - .flat_map(|prefix: Vec| { - (0..target.choice_count()).map(move |i| { - let mut choice = prefix.clone(); - choice.push(i); - choice - }) - }) - .collect(); - } - let mut exact = 0; - for choice in choices { - let composed = match compose_logical_candidate(&inventory, &choice) { - Ok(composed) => composed, - // Stage 1 cannot yet compose a heavy-hitter summary over some - // keys (an integer or several key columns); Stage 2 never sees - // that candidate. Every other choice must compose. - Err(e) => { - let summarized_topk = inventory.targets.iter().zip(&choice).any(|(t, &i)| { - let NonASAPOp::Aggregate { measures, .. } = - inventory.aggregate(t).expect_non_asap() - else { - return false; - }; - matches!(measures[t.measure_index], AggIntent::TopK { .. }) - && t.computation_for(i).map(|(c, _)| c) - != Some(&HowToComputeAggregateMeasure::Exact) - }); - assert!(summarized_topk, "{sql}, choice {choice:?}: {e}"); - continue; - } - }; - let roots: Vec<_> = composed; - let candidate = optimize_physical_asap( - "L1", - &roots, - &[], - &asap_types::workload::DataWorkload::default(), - false, - ) - .unwrap_or_else(|e| panic!("{sql}, choice {choice:?} pruned: {e}")) - .remove(0); - let QueryRoot::Operator(physical) = &candidate.roots[0] else { - panic!("operator root") - }; - assert_eq!(names(physical), root_names, "{sql}, choice {choice:?}"); - if let Some(sort) = exact_topk_sort(physical) { - assert_eq!( - sort, - (ranked.to_string(), false), - "{sql}, choice {choice:?}" - ); - exact += 1; - } - } - exact -} - -/// Each query under an exact and an approximate target; every exact top-k -/// sorts by the ranked measure. -async fn check(sql: &str, ranked: &str, root_names: &[&str]) { - for accuracy in [AccuracyTarget::exact(), approximate()] { - let exact = check_every_choice(sql, accuracy.clone(), ranked, root_names).await; - assert!( - exact > 0, - "{sql} under {accuracy:?}: an exact top-k candidate" - ); - } -} - -/// Grouping by a column named `value` still ranks by the count. -#[tokio::test] -async fn group_key_named_value_ranks_by_count() { - check( - "SELECT value, COUNT(*) AS c FROM metrics GROUP BY value ORDER BY c DESC LIMIT 5", - "count(Int64(1))", - &["topk_5", "value"], - ) - .await; -} - -/// An integer group key leaves no `value` column; the count is still ranked. -#[tokio::test] -async fn integer_group_key_ranks_by_count() { - check( - "SELECT port, COUNT(*) AS c FROM metrics GROUP BY port ORDER BY c DESC LIMIT 5", - "count(Int64(1))", - &["topk_5", "port"], - ) - .await; -} - -/// Two group keys, one of them an integer. -#[tokio::test] -async fn multi_key_ranks_by_count() { - check( - "SELECT host, port, COUNT(*) AS c FROM metrics GROUP BY host, port \ - ORDER BY c DESC LIMIT 5", - "count(Int64(1))", - &["topk_5", "host", "port"], - ) - .await; -} - -/// A SUM ranking sorts by the sum. -#[tokio::test] -async fn sum_ranking_ranks_by_sum() { - check( - "SELECT port, SUM(value) AS s FROM metrics GROUP BY port ORDER BY s DESC LIMIT 5", - "sum(metrics.value)", - &["topk_5", "port"], - ) - .await; -} diff --git a/crates/integration-tests/tests/summary_coverage_e2e.rs b/crates/integration-tests/tests/summary_coverage_e2e.rs deleted file mode 100644 index f2ce4aad3..000000000 --- a/crates/integration-tests/tests/summary_coverage_e2e.rs +++ /dev/null @@ -1,262 +0,0 @@ -//! Query string → planned summary state → `coverage()`, for the examples of -//! the ASAP primitive schema design doc (#573 §4.2.2). Each query is lowered -//! and its logical candidates composed, and the quantile's KLL `SummaryAgg` -//! is checked: what its `definition` keeps and what its `selection` takes, -//! row by row of §4.2.2's "More examples". The tests read the candidates, not -//! plan selection's choice, so they do not depend on which candidate is -//! cheapest. - -use std::ops::Bound; -use std::rc::Rc; - -use asap_frontend_sql::{lower_sql, SqlCatalog}; -use asap_integration_tests::fixtures::lower_promql; -use asap_integration_tests::planning::explicit_summary_candidate; -use asap_types::ir::properties::summary_coverage::{ColumnIdentity, Constraint, SelectionBox}; -use asap_types::ir::properties::AccuracyTarget; -use asap_types::ir::scalar::expr_ir::ScalarValue; -use asap_types::ir::schema::{DataType, Field, Schema}; -use asap_types::ir::schema::{FieldDataType, SketchAlgorithm}; -use asap_types::ir::{ASAPOp, NonASAPOp, OperatorNode, Predicate, ScalarExpr}; - -fn accuracy() -> AccuracyTarget { - AccuracyTarget::max_error(0.01).failure_probability(0.01) -} - -/// The single KLL `SummaryAgg` in an explicit KLL rule fixture for `pre`. -/// Coverage semantics do not depend on the capacities visited by search. -fn kll_state(pre: Rc) -> Rc { - let dag = explicit_summary_candidate(pre, SketchAlgorithm::Kll); - let mut states: Vec<_> = OperatorNode::reachable(&dag) - .into_iter() - .filter(|node| { - matches!(node.asap(), Some(ASAPOp::SummaryAgg { family: FieldDataType::Sketch(kind, _), .. }) - if kind.algorithm() == &SketchAlgorithm::Kll) - }) - .collect(); - assert_eq!( - states.len(), - 1, - "expected one KLL candidate state: {states:#?}" - ); - states.pop().unwrap() -} - -async fn sql_state(sql: &str) -> Rc { - let catalog = SqlCatalog::new().with_table( - "t", - Schema::new(vec![ - Field::plain("job", DataType::Utf8, false), - Field::plain("region", DataType::Utf8, false), - Field::plain("latency", DataType::Float64, false), - ]), - ); - let pre = lower_sql(sql, &catalog, accuracy()) - .await - .unwrap_or_else(|e| panic!("lower failed for {sql:?}: {e}")); - kll_state(pre) -} - -fn promql_state(query: &str) -> Rc { - kll_state(lower_promql(query, accuracy()).expect("lowering failed")) -} - -fn column(table: Option<&str>, name: &str) -> ColumnIdentity { - ColumnIdentity::new(table.map(str::to_string), name.to_string()) -} - -fn utf8(value: &str) -> ScalarValue { - ScalarValue::Utf8(value.to_string()) -} - -/// The child of the definition's `SummaryAgg`. -fn definition_child(state: &OperatorNode) -> Rc { - state.coverage().unwrap().definition().children()[0].clone() -} - -/// The definition's `SummaryAgg` carries no filter of its own. -fn assert_definition_unfiltered(state: &OperatorNode) { - let coverage = state.coverage().unwrap(); - let definition = coverage.definition(); - assert!( - matches!( - definition.asap(), - Some(ASAPOp::SummaryAgg { filter: None, .. }) - ), - "{definition:#?}" - ); -} - -/// The definition keeps the whole sub-DAG below the state: nothing moved -/// into the selection, which is the one unrestricted box. -fn assert_nothing_selected(state: &OperatorNode) { - assert_eq!( - state.coverage().unwrap().selection(), - vec![SelectionBox::default()] - ); - assert!(Rc::ptr_eq(&definition_child(state), state.children()[0])); -} - -#[tokio::test] -async fn sql_where_value_set_moves_to_selection() { - let state = - sql_state("SELECT approx_percentile_cont(latency, 0.99) FROM t WHERE region = 'us'").await; - let coverage = state.coverage().unwrap(); - assert_eq!( - coverage.selection(), - vec![SelectionBox::new( - [( - column(Some("t"), "region"), - Constraint::In(vec![utf8("us")]) - )] - .into(), - None - )] - ); - assert_definition_unfiltered(&state); - let child = definition_child(&state); - assert!( - matches!(child.non_asap(), Some(NonASAPOp::Scan { predicates, .. }) if predicates.is_empty()) - ); -} - -#[tokio::test] -async fn sql_where_range_moves_to_selection() { - let state = - sql_state("SELECT approx_percentile_cont(latency, 0.99) FROM t WHERE latency < 100").await; - assert_eq!( - state.coverage().unwrap().selection(), - vec![SelectionBox::new( - [( - column(Some("t"), "latency"), - Constraint::Interval { - lower: Bound::Unbounded, - upper: Bound::Excluded(ScalarValue::Float64(100.0)), - } - )] - .into(), - None - )] - ); - assert_definition_unfiltered(&state); - let child = definition_child(&state); - assert!( - matches!(child.non_asap(), Some(NonASAPOp::Scan { predicates, .. }) if predicates.is_empty()) - ); -} - -/// The worked example without its `FILTER` clause: the rename carries -/// `region = 'us'` into the selection as `r`, and the expression condition -/// stays in the definition's scan. -#[tokio::test] -async fn sql_renamed_column_moves_and_expression_stays() { - let state = sql_state( - "SELECT job, approx_percentile_cont(latency, 0.99) \ - FROM (SELECT job, region AS r, latency FROM t \ - WHERE region = 'us' AND latency * 2 > 10) \ - GROUP BY job", - ) - .await; - assert_eq!( - state.coverage().unwrap().selection(), - vec![SelectionBox::new( - [(column(None, "r"), Constraint::In(vec![utf8("us")]))].into(), - None - )] - ); - assert_definition_unfiltered(&state); - let project = definition_child(&state); - assert!(matches!( - project.non_asap(), - Some(NonASAPOp::Project { .. }) - )); - let scan = project.children()[0].clone(); - let Some(NonASAPOp::Scan { predicates, .. }) = scan.non_asap() else { - panic!("expected a scan, got {scan:#?}"); - }; - assert_eq!(predicates.len(), 1, "{predicates:#?}"); - assert!( - matches!(&predicates[0], Predicate(ScalarExpr::Compare { left, .. }) - if matches!(**left, ScalarExpr::Arithmetic { .. })), - "the expression condition should stay: {predicates:#?}" - ); -} - -#[test] -fn promql_offset_window_is_relative_time() { - let state = promql_state("quantile_over_time(0.99, m[1m] offset 2m)"); - assert_eq!( - state.coverage().unwrap().selection(), - vec![SelectionBox::new( - Default::default(), - Some((Bound::Excluded(-180_000), Bound::Included(-120_000))) - )] - ); - assert!(matches!( - definition_child(&state).non_asap(), - Some(NonASAPOp::Scan { .. }) - )); -} - -/// The label matcher is below `rate`, which the walk does not pass: the -/// definition keeps all of it, the matcher included. -#[test] -fn promql_matcher_below_rate_stays_in_definition() { - let state = promql_state("quantile(0.99, rate(m{job=\"api\"}[5m]))"); - assert_nothing_selected(&state); - let scans: Vec<_> = OperatorNode::reachable(&definition_child(&state)) - .into_iter() - .filter_map(|node| match node.non_asap() { - Some(NonASAPOp::Scan { predicates, .. }) => Some(predicates.len()), - _ => None, - }) - .collect(); - assert_eq!(scans, vec![1], "the matcher stays in the scan"); -} - -/// A PromQL comparison lowers to a `Filter` above `rate`, so it picks the -/// rate outputs the state reads; the definition keeps the `rate` below it. -#[test] -fn promql_comparison_above_rate_moves_to_selection() { - let state = promql_state("quantile(0.99, rate(m[5m]) > 0)"); - let filter = state.children()[0]; - assert!(matches!(filter.non_asap(), Some(NonASAPOp::Filter { .. }))); - assert!(Rc::ptr_eq(&definition_child(&state), filter.children()[0])); - assert_eq!( - state.coverage().unwrap().selection(), - vec![SelectionBox::new( - [( - column(None, "value"), - Constraint::Interval { - lower: Bound::Excluded(ScalarValue::Float64(0.0)), - upper: Bound::Unbounded, - }, - )] - .into(), - None - )] - ); -} - -/// A vector/vector comparison is a `BinaryOp`, which the walk does not enter. -#[test] -fn promql_vector_comparison_stays_in_definition() { - let state = promql_state("quantile(0.99, rate(a[5m]) > rate(b[5m]))"); - assert_nothing_selected(&state); - assert!(matches!( - definition_child(&state).non_asap(), - Some(NonASAPOp::BinaryOp { .. }) - )); -} - -/// A `bool` comparison lowers to a `Project` (values become 0 or 1), which -/// drops no row, so it stays in the definition. -#[test] -fn promql_bool_comparison_stays_in_definition() { - let state = promql_state("quantile(0.99, rate(m[5m]) > bool 0)"); - assert_nothing_selected(&state); - assert!(matches!( - definition_child(&state).non_asap(), - Some(NonASAPOp::Project { .. }) - )); -} diff --git a/crates/integration-tests/tests/tumbling_window_merge.rs b/crates/integration-tests/tests/tumbling_window_merge.rs deleted file mode 100644 index 7f5c7cff2..000000000 --- a/crates/integration-tests/tests/tumbling_window_merge.rs +++ /dev/null @@ -1,887 +0,0 @@ -//! Tumbling windows merged by `SummaryMerge` answer the same per-series query -//! as one build over the whole window (#509 Example 3/4, Pattern B): tumbling windows -//! written by hand, and the planner's window-composition rule's -//! `SummaryTumblingWindowChain`, expanded into the tumbling windows the executor runs. -use crate::physical_common; - -use std::collections::BTreeMap; -use std::rc::Rc; -use std::sync::Arc; - -use asap_executor::physical_planner::test_support::views_at; -use asap_executor::physical_planner::{compile, promql_rows, CompiledPhysicalDAG}; -use asap_executor::runtime::Scope; -use asap_executor::sources::InMemoryDataStore; -use asap_executor::values::{Batch, Value}; -use asap_integration_tests::fixtures::lower_promql; -use asap_integration_tests::planning::{explicit_local_candidates, explicit_summary_candidate}; -use asap_logical_asap_optimization::test_support::{ - tumbling_window_candidates, window_composition_pass2_candidates, - window_composition_shared_chain_candidates, -}; -use asap_types::ir::operator::operator_properties::TimeShift; -use asap_types::ir::operator::tumbling_window_chain::expand_tumbling_window_chains; -use asap_types::ir::physical_export::PhysicalASAPDAG; -use asap_types::ir::properties::{AccuracyMetric, AccuracyTarget}; -use asap_types::ir::schema::Schema; -use asap_types::ir::{ - apply_materialization_timings, ASAPOp, MaterializationAssignment, NodeToTimedNodeCache, - NonASAPOp, Operator, OperatorNode, QueryRoot, -}; -use asap_types::physical::{Materialization, Medium}; -use asap_types::test_support::compile_physical_asap_dag_with_node_ids; - -const EVALUATION_MS: i64 = 300_000; -const TUMBLING_WINDOW_WIDTH_MS: i64 = 60_000; -const TUMBLING_WINDOW_COUNT: i64 = 5; - -/// An explicit supported single-build fixture for the tumbling-window merge rules. -fn single_build(query: &str, accuracy: AccuracyTarget) -> Rc { - let root = lower_promql(query, accuracy.clone()).unwrap(); - let root = promql_rows::with_series_identity(&root).unwrap(); - if accuracy.is_exact() { - explicit_local_candidates(root) - .into_iter() - .find(|candidate| { - OperatorNode::reachable(candidate) - .iter() - .any(|node| matches!(node.asap(), Some(ASAPOp::SummaryAgg { .. }))) - }) - .expect("an exact accumulator fixture") - } else { - // Pin DDS so the actual positive-sample interpolation and tumbling-window merge - // checks do not depend on the optimizer's preferred implementation. - explicit_summary_candidate(root, asap_types::ir::schema::SketchAlgorithm::DDSketch) - } -} - -/// The per-entity `SummaryAgg` of `root` over its raw `TimeRange`. -fn summary_agg(root: &Rc) -> Rc { - OperatorNode::reachable(root) - .into_iter() - .find(|node| matches!(node.asap(), Some(ASAPOp::SummaryAgg { .. }))) - .expect("a summary build") -} - -/// Rewrite `root` so its summary is built in tumbling windows: tumbling window `i` is -/// `TimeRange(1m)` over `TimeShift(i·1m)` over the same `Scan`, and a -/// `SummaryMerge` combines the tumbling windows. Returns the root and the tumbling windows. -fn tumbling_windows(root: &Rc) -> (Rc, Vec>) { - let agg = summary_agg(root); - let Some(NonASAPOp::TimeRange { kind, child, .. }) = agg.children()[0].non_asap() else { - panic!("expected a raw time range under the summary"); - }; - let tumbling_windows: Vec<_> = (0..TUMBLING_WINDOW_COUNT) - .map(|i| { - let shifted = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeShift { - shift: TimeShift { - offset_ms: i * TUMBLING_WINDOW_WIDTH_MS, - at: None, - }, - child: child.clone(), - })) - .unwrap(); - let range = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeRange { - range: std::time::Duration::from_millis(TUMBLING_WINDOW_WIDTH_MS as u64), - kind: *kind, - child: shifted, - })) - .unwrap(); - OperatorNode::new_shared(agg.operator.map_children(|_| range.clone())).unwrap() - }) - .collect(); - let merged = OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryMerge { - children: tumbling_windows.clone(), - newest_tumbling_windows: None, - group_by: None, - })) - .unwrap(); - let root = replace(root, &agg, &merged); - root.validate_structure().unwrap(); - (root, tumbling_windows) -} - -/// `root` with its summary built in tumbling windows by the planner's -/// window-composition rule, for a query repeating every minute: the -/// planned root (a `SummaryMerge` over one `SummaryTumblingWindowChain`) and its expansion -/// into the tumbling windows the executor runs. -fn planned_tumbling_windows(root: &Rc) -> (Rc, Rc) { - let roots = [QueryRoot::Operator(Rc::clone(root))]; - let rule_candidates = - tumbling_window_candidates(&roots, &[Some(TUMBLING_WINDOW_WIDTH_MS as u64)]); - let [rule_candidate] = rule_candidates.as_slice() else { - panic!("one build to split: {rule_candidates:?}") - }; - assert_eq!( - ( - rule_candidate.tumbling_window_width_ms, - rule_candidate.tumbling_window_count - ), - (TUMBLING_WINDOW_WIDTH_MS as u64, 5) - ); - let mut pass2_candidates = - window_composition_pass2_candidates(&roots, &rule_candidates).unwrap(); - let pass2_candidate = pass2_candidates.pop().unwrap(); - assert!(pass2_candidates.is_empty()); - let QueryRoot::Operator(planned) = pass2_candidate.roots.unwrap().remove(0) else { - panic!("an operator root") - }; - planned.validate_structure().unwrap(); - assert_eq!( - planned.guarantee, root.guarantee, - "merged tumbling windows keep the whole build's guarantee" - ); - let expanded = expand_tumbling_window_chains(&planned).unwrap(); - expanded.validate_structure().unwrap(); - (planned, expanded) -} - -/// `node` with `from` replaced by `to`. -fn replace( - node: &Rc, - from: &Rc, - to: &Rc, -) -> Rc { - if Rc::ptr_eq(node, from) { - return to.clone(); - } - Rc::new( - node.with_new_children(|child| replace(child, from, to)) - .unwrap() - .with_guarantee(node.guarantee.clone()), - ) -} - -/// One raw data sample: series, timestamp (ms) and value. -type RawDataSample = (&'static str, i64, f64); - -/// Two series, four positive samples per minute in [0, 5m), each -/// `phase_ms` after its 15-s step (0: on the tumbling-window boundaries). -fn samples(phase_ms: i64) -> Vec { - let mut rows = vec![]; - for (s, series) in ["a", "b"].into_iter().enumerate() { - for minute in 0..TUMBLING_WINDOW_COUNT { - for j in 0..4 { - let ts = minute * TUMBLING_WINDOW_WIDTH_MS + j * 15_000 + phase_ms; - rows.push((series, ts, (s * 100) as f64 + (minute * 4 + j + 1) as f64)); - } - } - } - rows -} - -struct Exported { - dag: PhysicalASAPDAG, - /// Each `SummaryAgg` node with its tumbling window. - builds: Vec<(u64, i64)>, -} - -fn export(root: &Rc, assignment: &MaterializationAssignment) -> Exported { - let timed = - apply_materialization_timings(root, assignment, &mut NodeToTimedNodeCache::new()).unwrap(); - let compiled = compile_physical_asap_dag_with_node_ids(&timed).unwrap(); - let tumbling_window = |node: &OperatorNode| -> i64 { - match node.children()[0].non_asap() { - Some(NonASAPOp::TimeShift { shift, .. }) => shift.offset_ms / TUMBLING_WINDOW_WIDTH_MS, - _ => 0, - } - }; - let mut builds = vec![]; - for node in &compiled.dag.nodes { - let logical = compiled.node_ids.operator_node(node.id).unwrap(); - if let Operator::ASAP(ASAPOp::SummaryAgg { child, .. }) = &logical.operator { - builds.push((node.id as u64, tumbling_window(child))); - } - } - Exported { - dag: compiled.dag, - builds, - } -} - -fn schema_of(dag: &PhysicalASAPDAG, id: u64) -> Arc { - Arc::new( - dag.nodes - .iter() - .find(|node| node.id as u64 == id) - .unwrap() - .output_schema - .clone(), - ) -} - -/// A store holding every sample as raw data at the `Scan` of `m`; each -/// `TimeRange` keeps its own window of them. -fn raw_store(dag: &PhysicalASAPDAG, samples: &[RawDataSample]) -> InMemoryDataStore { - physical_common::raw_data_store(dag, |_, schema| { - samples - .iter() - .map(|&(series, ts, value)| { - let labels = BTreeMap::from([ - ("__name__".to_string(), "m".to_string()), - ("series".to_string(), series.to_string()), - ]); - promql_rows::series_row(schema, &labels, ts, value).unwrap() - }) - .collect() - }) -} - -fn run(plan: &CompiledPhysicalDAG, store: InMemoryDataStore, scope: Scope) -> Vec> { - let mut rows: Vec> = physical_common::execute(plan, store, scope) - .remove(0) - .iter() - .flat_map(|batch| batch.to_rows().to_vec()) - .collect(); - rows.sort_by_key(|row| format!("{row:?}")); - rows -} - -/// `Value` has no equality; its debug form identifies every scalar exactly. -fn render(rows: &[Vec]) -> Vec { - rows.iter().map(|row| format!("{row:?}")).collect() -} - -fn has_timestamp(row: &[Value], ms: i64) -> bool { - row.iter() - .any(|v| matches!(v, Value::Timestamp(t) if *t == ms)) -} - -fn query_scope() -> Scope { - Scope::Query { - evaluation_time_ms: EVALUATION_MS, - } -} - -/// Compile `root` with everything at query time (Pattern B2: tumbling windows rebuilt -/// from raw rows at query time). -fn compile_at_query_time( - root: &Rc, -) -> Result<(Exported, CompiledPhysicalDAG), String> { - let exported = export(root, &MaterializationAssignment::default()); - let plan = - compile(&exported.dag, &[], &[exported.dag.roots[0] as u64]).map_err(|e| e.to_string())?; - Ok((exported, plan)) -} - -fn execute(root: &Rc, samples: &[RawDataSample]) -> Vec> { - let (exported, plan) = compile_at_query_time(root).unwrap(); - run(&plan, raw_store(&exported.dag, samples), query_scope()) -} - -/// Build the tumbling windows in ingestion runs (stored `MatAtIngestion`, each with its -/// own build timestamp), then merge them in the query run (Pattern B1). -fn execute_retained( - root: &Rc, - tumbling_windows: &[Rc], - samples: &[RawDataSample], -) -> Vec> { - let mut assignment = MaterializationAssignment::default(); - for tumbling_window in tumbling_windows { - assignment.set( - tumbling_window, - Materialization::MatAtIngestion(Medium::Memory), - ); - } - execute_retained_as(root, &assignment, samples) -} - -/// The planner's tumbling-window chain maintained at ingestion time, as Stage 2's B1 -/// option assigns it (`MatAtIngestion` on the chain), then expanded: each -/// tumbling window carries the chain's timing and materialization, so the tumbling windows are -/// built in ingestion runs and merged in the query run. -fn execute_maintained_chain( - planned: &Rc, - samples: &[RawDataSample], -) -> Vec> { - let chain = OperatorNode::reachable(planned) - .into_iter() - .find(|n| matches!(n.asap(), Some(ASAPOp::SummaryTumblingWindowChain { .. }))) - .expect("a tumbling-window chain"); - let mut assignment = MaterializationAssignment::default(); - assignment.set(&chain, Materialization::MatAtIngestion(Medium::Memory)); - let timed = - apply_materialization_timings(planned, &assignment, &mut NodeToTimedNodeCache::new()) - .unwrap(); - let expanded = expand_tumbling_window_chains(&timed).unwrap(); - for tumbling_window in OperatorNode::reachable(&expanded) - .iter() - .filter(|n| matches!(n.asap(), Some(ASAPOp::SummaryAgg { .. }))) - { - assert_eq!( - tumbling_window.materialization, - Materialization::MatAtIngestion(Medium::Memory) - ); - } - execute_retained_as(&expanded, &MaterializationAssignment::default(), samples) -} - -fn execute_retained_as( - root: &Rc, - assignment: &MaterializationAssignment, - samples: &[RawDataSample], -) -> Vec> { - let mut exported = export(root, assignment); - let builds: Vec = exported.builds.iter().map(|&(build, _)| build).collect(); - let views = views_at(&mut exported.dag, &builds).unwrap(); - // Each tumbling window's ingestion run refreshes its view from the raw samples. - let mut store = InMemoryDataStore::default(); - for &(build, tumbling_window) in &exported.builds { - let plan = compile(&exported.dag, &[], &[build]).unwrap(); - let end = EVALUATION_MS - tumbling_window * TUMBLING_WINDOW_WIDTH_MS; - let rows = run( - &plan, - raw_store(&exported.dag, samples), - Scope::Ingestion { - window_start_ms: end - TUMBLING_WINDOW_WIDTH_MS, - window_end_ms: end, - }, - ); - assert!(rows.iter().all(|row| has_timestamp(row, end))); - store = store - .with_materialized_view( - build as usize, - vec![Batch::try_new(schema_of(&exported.dag, build), rows).unwrap()], - ) - .unwrap(); - } - let plan = compile(&exported.dag, &views, &[exported.dag.roots[0] as u64]).unwrap(); - run(&plan, store, query_scope()) -} - -fn values(rows: &[Vec]) -> Vec { - rows.iter() - .flat_map(|row| { - row.iter().filter_map(|v| match v { - Value::Float64(v) => Some(*v), - _ => None, - }) - }) - .collect() -} - -/// Returns the per-series values, which every execution path agrees on. -fn assert_tumbling_window_merge_matches_single_build( - query: &str, - accuracy: AccuracyTarget, - samples: &[RawDataSample], -) -> Vec { - let single = single_build(query, accuracy); - let (tumbling_windows, tumbling_window_builds) = tumbling_windows(&single); - let expected = execute(&single, samples); - assert_eq!(expected.len(), 2, "{expected:?}"); - assert!(expected.iter().all(|row| has_timestamp(row, EVALUATION_MS))); - let result = values(&expected); - let expected = render(&expected); - assert_eq!( - render(&execute(&tumbling_windows, samples)), - expected, - "tumbling windows rebuilt at query time" - ); - assert_eq!( - render(&execute_retained( - &tumbling_windows, - &tumbling_window_builds, - samples - )), - expected, - "retained tumbling windows" - ); - let (planned, expanded) = planned_tumbling_windows(&single); - assert_eq!( - render(&execute(&expanded, samples)), - expected, - "the planner's tumbling-window chain, expanded" - ); - assert_eq!( - render(&execute_maintained_chain(&planned, samples)), - expected, - "the planner's tumbling-window chain, maintained at ingestion time" - ); - let unexpanded = compile_at_query_time(&planned).err().unwrap_or_default(); - assert!( - unexpanded.contains("SummaryTumblingWindowChain"), - "{unexpanded}" - ); - result -} - -/// Exact per-series Sum over 5 merged 1-minute tumbling windows equals one 5-minute -/// build, timestamped at the evaluation time. -#[test] -fn exact_sum_over_tumbling_windows_matches_single_build() { - let sums = assert_tumbling_window_merge_matches_single_build( - "sum_over_time(m[5m])", - AccuracyTarget::exact(), - // 1 ms past each step: the window `(0, 5m]` holds all 20 samples. - &samples(1), - ); - assert_eq!(sums, vec![210., 2210.]); -} - -/// The selected quantile sketch over 5 merged 1-minute tumbling windows equals one -/// 5-minute build, within the target of the exact p99 (20 and 120). -#[test] -fn quantile_over_tumbling_windows_matches_single_build() { - let p99 = assert_tumbling_window_merge_matches_single_build( - "quantile_over_time(0.99, m[5m])", - AccuracyTarget::max_error(0.05).accuracy_metric(AccuracyMetric::Relative), - &samples(0), - ); - let mut p99 = p99; - p99.sort_by(f64::total_cmp); - assert_eq!(p99.len(), 2); - for (estimate, exact) in p99.into_iter().zip([20., 120.]) { - assert!( - (estimate - exact).abs() <= 0.05 * exact, - "{estimate} vs {exact}" - ); - } -} - -/// Exact counter Rate over 5 merged 1-minute tumbling windows equals one 5-minute -/// build: the merged increase is divided by the whole 5-minute window, not -/// by one tumbling window's minute. -#[test] -fn exact_rate_over_tumbling_windows_matches_single_build() { - let rates = assert_tumbling_window_merge_matches_single_build( - "rate(m[5m])", - AccuracyTarget::exact(), - &samples(0), - ); - assert_eq!(rates.len(), 2); - // 19 counted over 285 s of samples, extrapolated to the 5-min window. - for rate in rates { - assert!((rate - 20.0 / 300.0).abs() < 1e-9, "{rate}"); - } -} - -/// With samples off the tumbling-window boundaries, Increase extrapolates to the -/// start of the whole 5-minute window, as one build does, not to the start -/// of the newest tumbling window. -#[test] -fn exact_increase_off_the_tumbling_window_boundaries_matches_single_build() { - let increases = assert_tumbling_window_merge_matches_single_build( - "increase(m[5m])", - AccuracyTarget::exact(), - &samples(7_000), - ); - assert_eq!(increases.len(), 2); -} - -/// The single build of `query`, evaluated at `evaluation_ms` over every -/// sample, as values. -fn whole_build_values_at(query: &str, evaluation_ms: i64, samples: &[RawDataSample]) -> Vec { - let root = single_build(query, AccuracyTarget::exact()); - let (exported, plan) = compile_at_query_time(&root).unwrap(); - values(&run( - &plan, - raw_store(&exported.dag, samples), - Scope::Query { - evaluation_time_ms: evaluation_ms, - }, - )) -} - -/// A counter over `[5m] offset 1m` evaluated a minute later reads the same -/// window as one without the offset: the counter window ends at the -/// evaluation time minus the offset, not at the evaluation time. -#[test] -fn offset_counter_window_ends_at_the_offset() { - let samples = samples(7_000); - for function in ["rate", "increase"] { - let plain = whole_build_values_at(&format!("{function}(m[5m])"), EVALUATION_MS, &samples); - let shifted = whole_build_values_at( - &format!("{function}(m[5m] offset 1m)"), - EVALUATION_MS + TUMBLING_WINDOW_WIDTH_MS, - &samples, - ); - assert_eq!(shifted, plain, "{function}"); - } -} - -/// A counter over `[5m] @ 300` reads the window ending at 300 s whatever the -/// evaluation time, and `@ 300 offset 1m` the one ending at 240 s: the same -/// values as the unanchored counter evaluated at 300 s and at 240 s. Today -/// it reads the window ending at the evaluation time (#748). -#[test] -#[ignore = "the counter window drops an @ anchor and its offset: #748"] -fn anchored_counter_window_ends_at_the_anchor() { - let samples = samples(7_000); - for function in ["rate", "increase"] { - let at_300 = whole_build_values_at(&format!("{function}(m[5m])"), EVALUATION_MS, &samples); - let at_240 = whole_build_values_at( - &format!("{function}(m[5m])"), - EVALUATION_MS - TUMBLING_WINDOW_WIDTH_MS, - &samples, - ); - assert_ne!(at_300, at_240, "{function}: the two windows differ"); - let anchored = whole_build_values_at( - &format!("{function}(m[5m] @ 300)"), - EVALUATION_MS + TUMBLING_WINDOW_WIDTH_MS, - &samples, - ); - assert_eq!(anchored, at_300, "{function} @ 300"); - let anchored_offset = whole_build_values_at( - &format!("{function}(m[5m] @ 300 offset 1m)"), - EVALUATION_MS + TUMBLING_WINDOW_WIDTH_MS, - &samples, - ); - assert_eq!(anchored_offset, at_240, "{function} @ 300 offset 1m"); - } -} - -/// Tumbling windows whose ranges read the scan through two stacked `TimeShift`s (the -/// tumbling window's shift over a zero shift) compile and answer as the single build: -/// the per-entity build looks through every shift to its raw source. -#[test] -fn tumbling_windows_over_stacked_time_shifts_match_single_build() { - let single = single_build("increase(m[5m])", AccuracyTarget::exact()); - let (tumbling_windows, _) = tumbling_windows(&single); - let agg = summary_agg(&single); - let Some(NonASAPOp::TimeRange { child: scan, .. }) = agg.children()[0].non_asap() else { - panic!("expected a raw time range under the summary"); - }; - let zero_shift = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeShift { - shift: TimeShift { - offset_ms: 0, - at: None, - }, - child: scan.clone(), - })) - .unwrap(); - let stacked = replace(&tumbling_windows, scan, &zero_shift); - let expected = render(&execute(&single, &samples(0))); - assert_eq!(expected.len(), 2, "{expected:?}"); - assert_eq!(render(&execute(&stacked, &samples(0))), expected); -} - -/// Exact counter Increase over 5 merged 1-minute tumbling windows equals one 5-minute -/// build: the merge joins consecutive tumbling windows across their boundaries. -#[test] -fn exact_increase_over_tumbling_windows_matches_single_build() { - let increases = assert_tumbling_window_merge_matches_single_build( - "increase(m[5m])", - AccuracyTarget::exact(), - &samples(0), - ); - assert_eq!(increases.len(), 2); -} - -/// A middle tumbling window with no samples, for one series or for every series: the -/// merge joins the tumbling windows on either side of the empty one, and Increase -/// matches one 5-minute build. -#[test] -fn exact_increase_over_an_empty_middle_tumbling_window_matches_single_build() { - let middle = 2 * TUMBLING_WINDOW_WIDTH_MS..3 * TUMBLING_WINDOW_WIDTH_MS; - for emptied in [&["a"][..], &["a", "b"]] { - let gapped: Vec = samples(0) - .into_iter() - .filter(|(series, ts, _)| !(emptied.contains(series) && middle.contains(ts))) - .collect(); - let increases = assert_tumbling_window_merge_matches_single_build( - "increase(m[5m])", - AccuracyTarget::exact(), - &gapped, - ); - assert_eq!(increases.len(), 2, "{emptied:?}"); - } -} - -/// A counter over `[5m] offset 1m` split into the planner's tumbling windows, read -/// at query time or maintained at ingestion time, matches one build: the -/// tumbling windows' shifts fold the outer offset, and the merged counter window ends -/// at the evaluation time minus the offset. -#[test] -fn a_counter_over_tumbling_windows_with_an_outer_offset_matches_single_build() { - let samples = samples(7_000); - for function in ["rate", "increase"] { - let single = single_build( - &format!("{function}(m[5m] offset 1m)"), - AccuracyTarget::exact(), - ); - let expected = render(&execute(&single, &samples)); - assert_eq!(expected.len(), 2, "{function}: {expected:?}"); - let (planned, expanded) = planned_tumbling_windows(&single); - assert_eq!( - render(&execute(&expanded, &samples)), - expected, - "{function}: expanded tumbling windows" - ); - assert_eq!( - render(&execute_maintained_chain(&planned, &samples)), - expected, - "{function}: tumbling windows maintained at ingestion time" - ); - } -} - -/// `root`'s summary built from tumbling windows `(shift, width)` merged by one -/// `SummaryMerge`, both in minutes, or the IR's refusal of that merge. -fn tumbling_windows_at( - root: &Rc, - tumbling_windows: &[(i64, i64)], -) -> Result, String> { - let agg = summary_agg(root); - let Some(NonASAPOp::TimeRange { kind, child, .. }) = agg.children()[0].non_asap() else { - panic!("expected a raw time range under the summary"); - }; - let children = tumbling_windows - .iter() - .map(|&(shift, width)| { - let shifted = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeShift { - shift: TimeShift { - offset_ms: shift * TUMBLING_WINDOW_WIDTH_MS, - at: None, - }, - child: child.clone(), - })) - .unwrap(); - let range = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeRange { - range: std::time::Duration::from_millis((width * TUMBLING_WINDOW_WIDTH_MS) as u64), - kind: *kind, - child: shifted, - })) - .unwrap(); - OperatorNode::new_shared(agg.operator.map_children(|_| range.clone())).unwrap() - }) - .collect(); - let merged = OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryMerge { - children, - newest_tumbling_windows: None, - group_by: None, - })) - .map_err(|error| format!("{error:?}"))?; - Ok(replace(root, &agg, &merged)) -} - -/// Counter tumbling windows form one window only back to back: with a gap between -/// them, compiling the counter evaluation is an error; overlapping tumbling windows -/// are already refused by the `SummaryMerge`, whose counter inputs must be -/// disjoint in time. -#[test] -fn counter_tumbling_windows_with_a_gap_or_an_overlap_are_rejected() { - let single = single_build("rate(m[5m])", AccuracyTarget::exact()); - let gapped = tumbling_windows_at(&single, &[(0, 1), (2, 1)]).unwrap(); - let error = compile_at_query_time(&gapped).err().unwrap_or_default(); - assert!(error.contains("ambiguous logical windows"), "{error}"); - let overlap = tumbling_windows_at(&single, &[(0, 2), (1, 2)]) - .err() - .unwrap_or_default(); - assert!(overlap.contains("InputsOverlap"), "{overlap}"); -} - -/// A 2-min and a 5-min quantile, both every minute, on 1-min tumbling windows: the -/// window-composition rule offers one 5-tumbling-window chain both read, the 2-min -/// query merging only its newest 2 tumbling windows. Expanded, rebuilt at query time -/// or maintained at ingestion time, each query's answer from the shared -/// chain equals its answer from its own chain. -#[test] -fn newest_tumbling_windows_of_a_shared_chain_match_a_separate_chain() { - let accuracy = AccuracyTarget::max_error(0.05).accuracy_metric(AccuracyMetric::Relative); - let roots = [ - QueryRoot::Operator(single_build( - "quantile_over_time(0.5, m[2m])", - accuracy.clone(), - )), - QueryRoot::Operator(single_build("quantile_over_time(0.99, m[5m])", accuracy)), - ]; - let grid = Some(TUMBLING_WINDOW_WIDTH_MS as u64); - let rule_candidates = tumbling_window_candidates(&roots, &[grid, grid]); - assert_eq!(rule_candidates.len(), 2, "{rule_candidates:?}"); - let pass2_candidates = window_composition_pass2_candidates(&roots, &rule_candidates).unwrap(); - let both = |shared: bool| -> Vec> { - pass2_candidates - .iter() - .find(|c| { - c.tumbling_window_candidates.len() == 2 && c.shared_chains.is_empty() != shared - }) - .expect("both builds in tumbling windows") - .roots - .as_ref() - .unwrap() - .iter() - .map(|root| match root { - QueryRoot::Operator(node) => Rc::clone(node), - QueryRoot::Scalar(_) => panic!("an operator root"), - }) - .collect() - }; - let (shared, separate) = (both(true), both(false)); - let newest = OperatorNode::reachable(&shared[0]) - .into_iter() - .find_map(|n| match n.asap() { - Some(ASAPOp::SummaryMerge { - newest_tumbling_windows, - .. - }) => Some(*newest_tumbling_windows), - _ => None, - }); - assert_eq!(newest, Some(Some(2))); - let data = samples(0); - for (query, (shared, separate)) in shared.iter().zip(&separate).enumerate() { - let expected = execute(&expand_tumbling_window_chains(separate).unwrap(), &data); - assert!(!expected.is_empty()); - assert_eq!( - render(&execute( - &expand_tumbling_window_chains(shared).unwrap(), - &data - )), - render(&expected), - "Q{}: the shared chain rebuilt at query time", - query + 1 - ); - assert_eq!( - render(&execute_maintained_chain(shared, &data)), - render(&expected), - "Q{}: the shared chain maintained at ingestion time", - query + 1 - ); - } -} - -/// Runs `root` at query time at `evaluation_ms` over every sample; each -/// `TimeRange` keeps its own window `(T - s - w, T - s]` for a range of -/// width `w` over a shift of `s` (none for an unshifted range). -fn execute_ranges_at_their_windows( - root: &Rc, - samples: &[RawDataSample], - evaluation_ms: i64, -) -> Vec> { - let (exported, plan) = compile_at_query_time(root).unwrap(); - let scope = Scope::Query { - evaluation_time_ms: evaluation_ms, - }; - run(&plan, raw_store(&exported.dag, samples), scope) -} - -/// A 5-min quantile every 30 s (10 × 30-s tumbling windows) and a 5-min quantile -/// every minute (5 × 1-min tumbling windows): the rule offers one chain of 30-s -/// tumbling windows both read in full, each at its own interval. Expanded, each -/// query's answer from the shared chain equals one build over its window, -/// at 5 min, and for the 30-s query also at 4.5 min, off the 1-min grid. -#[test] -fn queries_at_different_intervals_read_one_chain_of_the_finer_tumbling_windows() { - let accuracy = AccuracyTarget::max_error(0.05).accuracy_metric(AccuracyMetric::Relative); - let singles = [ - single_build("quantile_over_time(0.5, m[5m])", accuracy.clone()), - single_build("quantile_over_time(0.99, m[5m])", accuracy), - ]; - let roots: Vec = singles - .iter() - .map(|root| QueryRoot::Operator(Rc::clone(root))) - .collect(); - let rule_candidates = tumbling_window_candidates( - &roots, - &[Some(30_000), Some(TUMBLING_WINDOW_WIDTH_MS as u64)], - ); - let widths: Vec = rule_candidates - .iter() - .map(|c| c.tumbling_window_width_ms) - .collect(); - assert_eq!(widths, [30_000, TUMBLING_WINDOW_WIDTH_MS as u64]); - let shared = window_composition_pass2_candidates(&roots, &rule_candidates) - .unwrap() - .into_iter() - .find(|c| c.tumbling_window_candidates.len() == 2 && !c.shared_chains.is_empty()) - .expect("one shared chain"); - // The whole set's shared chains, built alone, are the same candidates. - let alone = window_composition_shared_chain_candidates(&roots, &rule_candidates); - let first = alone.first().expect("one shared chain alone"); - assert_eq!(first.shared_chains, shared.shared_chains); - assert_eq!(first.roots.as_ref().ok(), shared.roots.as_ref().ok()); - assert_eq!( - ( - shared.shared_chains[0].tumbling_window_width_ms, - shared.shared_chains[0].tumbling_window_count - ), - (30_000, 10) - ); - let data = samples(0); - let shared_roots: Vec> = shared - .roots - .unwrap() - .into_iter() - .map(|root| match root { - QueryRoot::Operator(node) => expand_tumbling_window_chains(&node).unwrap(), - QueryRoot::Scalar(_) => panic!("an operator root"), - }) - .collect(); - for (query, evaluation_ms) in [(0, EVALUATION_MS), (1, EVALUATION_MS), (0, 270_000)] { - let expected = execute_ranges_at_their_windows(&singles[query], &data, evaluation_ms); - assert_eq!(expected.len(), 2, "{expected:?}"); - assert_eq!( - render(&execute_ranges_at_their_windows( - &shared_roots[query], - &data, - evaluation_ms - )), - render(&expected), - "Q{} at {evaluation_ms} ms", - query + 1 - ); - } -} - -/// A 5-min quantile every minute (5 × 1-min tumbling windows) and a 3-min one every -/// 90 s (2 × 90-s tumbling windows): neither width divides the other, so the rule -/// offers one chain at their gcd, 10 × 30-s tumbling windows, the 3-min query merging -/// the newest 6. Expanded, each query's answer from the shared chain equals -/// one build over its window, at times on each query's own grid. -#[test] -fn queries_at_60_and_90_seconds_read_one_chain_of_30_second_tumbling_windows() { - let accuracy = AccuracyTarget::max_error(0.05).accuracy_metric(AccuracyMetric::Relative); - let singles = [ - single_build("quantile_over_time(0.5, m[5m])", accuracy.clone()), - single_build("quantile_over_time(0.99, m[3m])", accuracy), - ]; - let roots: Vec = singles - .iter() - .map(|root| QueryRoot::Operator(Rc::clone(root))) - .collect(); - let rule_candidates = tumbling_window_candidates( - &roots, - &[Some(TUMBLING_WINDOW_WIDTH_MS as u64), Some(90_000)], - ); - let widths: Vec<(u64, u32)> = rule_candidates - .iter() - .map(|c| (c.tumbling_window_width_ms, c.tumbling_window_count)) - .collect(); - assert_eq!(widths, [(TUMBLING_WINDOW_WIDTH_MS as u64, 5), (90_000, 2)]); - let shared = window_composition_pass2_candidates(&roots, &rule_candidates) - .unwrap() - .into_iter() - .find(|c| c.tumbling_window_candidates.len() == 2 && !c.shared_chains.is_empty()) - .expect("one shared chain at the gcd"); - assert_eq!( - ( - shared.shared_chains[0].tumbling_window_width_ms, - shared.shared_chains[0].tumbling_window_count - ), - (30_000, 10) - ); - let data = samples(0); - let shared_roots: Vec> = shared - .roots - .unwrap() - .into_iter() - .map(|root| match root { - QueryRoot::Operator(node) => expand_tumbling_window_chains(&node).unwrap(), - QueryRoot::Scalar(_) => panic!("an operator root"), - }) - .collect(); - for (query, evaluation_ms) in [(0, EVALUATION_MS), (1, 270_000), (1, 180_000)] { - let expected = execute_ranges_at_their_windows(&singles[query], &data, evaluation_ms); - assert_eq!(expected.len(), 2, "{expected:?}"); - assert_eq!( - render(&execute_ranges_at_their_windows( - &shared_roots[query], - &data, - evaluation_ms - )), - render(&expected), - "Q{} at {evaluation_ms} ms", - query + 1 - ); - } -} diff --git a/crates/logical-asap-optimization/Cargo.toml b/crates/logical-asap-optimization/Cargo.toml index 665246936..3154d7233 100644 --- a/crates/logical-asap-optimization/Cargo.toml +++ b/crates/logical-asap-optimization/Cargo.toml @@ -21,7 +21,7 @@ asap-frontend-promql = { path = "../frontend-promql", optional = true } [features] # The test helpers (`test_support`) the later stage crates' tests share. -test-support = ["dep:asap-frontend-promql"] +test-support = ["dep:asap-frontend-promql", "asap-types/test-support"] [dev-dependencies] asap-types = { path = "../types", features = ["test-support"] } diff --git a/crates/logical-asap-optimization/src/deployment_capabilities.rs b/crates/logical-asap-optimization/src/deployment_capabilities.rs index f449513b6..add7d3ddd 100644 --- a/crates/logical-asap-optimization/src/deployment_capabilities.rs +++ b/crates/logical-asap-optimization/src/deployment_capabilities.rs @@ -13,17 +13,17 @@ //! accumulators and plain exact execution are always supported; //! * whether raw data samples are retained. If they are, Stage 3 counts them on //! disk once per source, at the longest window a query-time scan of it -//! reads. If they are not, raw data samples are not kept after they arrive, so +//! reads. If they are not, raw data samples are not retained after they arrive, so //! a candidate that scans raw data at query time is invalid (the //! rejection lists every such scan): only ingestion-time work reads raw //! rows. Stored data (at rest or mixed) with no raw data retained is a //! contradiction, refused as bad input ([`DeploymentCapabilities::validate`]); //! * whether query-time results are retained across evaluations. If they -//! are, Stage 2 also offers a tumbling-window chain whose readers evaluate -//! together at its tumbling-window width as `MatAtQueryKeptAcrossEvaluations`: each +//! are, Stage 2 also offers a tumbling-window chain whose queries evaluate +//! together at its tumbling-window width as query-time `AcrossEvaluations`: each //! evaluation builds one new tumbling window and keeps the others for the next ones, -//! covering the readers' longest look-back. A chain two or more queries -//! read keeps its `MatAtQuery` option too (shared within one evaluation), +//! covering the queries' longest look-back. A chain two or more queries +//! read keeps its `OneEvaluation` option too (shared within one evaluation), //! and Stage 3 picks the cheaper. If they are not, a query-time result is //! reused only within one evaluation. //! @@ -50,10 +50,10 @@ pub struct DeploymentCapabilities { /// it can answer from one; `None` (the default) supports every family /// and readout the planner can build. pub supported_summaries: Option>, - /// Whether raw data samples are kept for query-time scans; default `true`. + /// Whether raw data samples are retained for query-time scans; default `true`. pub raw_data_retained: bool, - /// Whether a query-time result may be kept across evaluations - /// (`MatAtQueryKeptAcrossEvaluations`); default `false`. + /// Whether a query-time result may be materialized across evaluations + /// (query-time `AcrossEvaluations`); default `false`. pub query_time_results_retained: bool, } diff --git a/crates/logical-asap-optimization/src/lib.rs b/crates/logical-asap-optimization/src/lib.rs index 3864772dc..a94771df1 100644 --- a/crates/logical-asap-optimization/src/lib.rs +++ b/crates/logical-asap-optimization/src/lib.rs @@ -4,7 +4,7 @@ //! (`asap_types::stages::LogicalASAPMemoCandidates`) that Stage 2 implements. //! //! Identical-sub-DAG sharing over the logical DAG (Stage 0, the `stage0` -//! module, behind [`generate_dag_of_logical_memo_groups`]) runs before the +//! module, behind [`memo_of_queries`]) runs before the //! stages; Stage 1 adds the ASAP-aware sharing that is invisible at the //! logical level (Pass 2, the `asap_aware_cse` module), e.g. //! `Quantile(x, 0.99)` and `Quantile(x, 0.95)` reading one built sketch. @@ -13,8 +13,7 @@ //! (`asap-accuracy`). It depends on no other stage crate, and no other //! stage crate depends on it except in their tests (CI checks the crate //! graph): the stage-boundary types (its input -//! `DagOfLogicalMemoGroups`, Pass 1's `PerComputationLogicalMemoCandidates` -//! and Pass 2's `LogicalASAPMemoCandidates`) are in `asap_types::stages`, and +//! `Memo`, Pass 1's `Memo` and Pass 2's `LogicalASAPMemoCandidates`) are in `asap_types::stages`, and //! the planner facade calls each stage in turn. It depends on no //! query-language frontend either, except with the `test-support` feature, //! whose test helpers (`test_support`) lower PromQL test queries through @@ -33,12 +32,11 @@ //! re-exports the internals other crates' tests call. //! //! - `stage0` — Stage 0: hash-consing the workload's logical DAG into memo -//! groups (`stage0::cse`), behind [`generate_dag_of_logical_memo_groups`]. +//! groups through `Memo::find_or_add_group`, behind [`memo_of_queries`]. //! - `pass1::per_computation_candidates` — Pass 1: per-computation candidate //! generation and composition, behind -//! [`generate_per_computation_logical_memo_candidates`]. -//! - `pass2::asap_aware_cse` and `pass2::memo` — Pass 2: ASAP-aware -//! common-subexpression elimination and the memo rules, behind +//! [`pass1_memo`]. +//! - `pass2::memo` — Pass 2: the rules over groups, behind //! [`generate_logical_asap_memo_candidates`]. //! - `pass1::realization` — the summary families that can serve an aggregate //! intent, the intent's accuracy target and `(ε, δ)` budget, and default @@ -57,7 +55,7 @@ //! candidate. Pass 1 takes the deployment's `AccuracyModel` and evidence //! only to prune summaries no query reading them can use (when they cannot //! accept an output without its guarantee) and sizes every summary for its -//! readers; plan selection (Stage 3) checks each selected candidate's +//! consumers; plan selection (Stage 3) checks each selected candidate's //! guarantee against its query's `AccuracyTarget` and never resizes. /// The deployment's capabilities: supported summaries and what it retains. @@ -78,25 +76,15 @@ pub mod test_support; pub use deployment_capabilities::{ DeploymentCapabilities, DeploymentCapabilitiesError, SupportedSummary, }; -// Stage 0: the memo groups of the workload's query roots -// (`asap_types::stages::DagOfLogicalMemoGroups`). -pub use stage0::generate_dag_of_logical_memo_groups; +// Stage 0: the memo groups of the workload's query roots, without +// candidates (`asap_types::stages::Memo`). +pub use stage0::memo_of_queries; // Pass 1: its entry and its errors. Its output and the types the output // holds are in `asap_types::stages`. -pub use pass1::{ - generate_per_computation_logical_memo_candidates, PerComputationCandidateError, - PerComputationLogicalMemoCandidatesError, -}; +pub use pass1::{pass1_memo, Pass1Error, PerComputationCandidateError}; // Pass 2: its entry and its error. -pub use pass1::memo_population::CandidatePopulationError; pub use pass2::logical_memo_candidates::{ generate_logical_asap_memo_candidates, pass2_after_local_window_composition, pass2_local_window_composition, }; -pub use pass2::memo::state_groups::Pass2RuleError; -// The CandidateLogicalASAPDAGs a memo stands for (developer tools' -// reference pricing). -pub use pass2::memo::enumeration::{ - build_logical_asap_dags_from_chosen_candidates, enumerate, AllChosenCandidatesPerMemoGroup, - ChosenCandidatePerMemoGroup, ErrorBuildingLogicalASAPDagsFromChosenCandidates, -}; +pub use pass2::memo::rules::Pass2RuleError; diff --git a/crates/logical-asap-optimization/src/pass1.rs b/crates/logical-asap-optimization/src/pass1.rs index 2f115053b..6c2f87b43 100644 --- a/crates/logical-asap-optimization/src/pass1.rs +++ b/crates/logical-asap-optimization/src/pass1.rs @@ -1,14 +1,10 @@ //! Stage 1, Pass 1: per-computation candidate generation and composition, //! and the summary realizations it offers. Its one entry is -//! [`generate_per_computation_logical_memo_candidates`], whose output -//! ([`PerComputationLogicalMemoCandidates`](asap_types::stages::PerComputationLogicalMemoCandidates)) -//! Pass 2 extends. +//! [`pass1_memo`], whose output +//! memo Pass 2 extends. -/// Stage 1, Pass 1's entry and output: the per-computation candidates on -/// the memo groups ([`PerComputationLogicalMemoCandidates`](asap_types::stages::PerComputationLogicalMemoCandidates)). +/// Stage 1, Pass 1's entry: the initial candidates of the memo groups. pub(crate) mod logical_memo_candidates; -/// Pass 1's candidates on Stage 0's memo groups (`try_populate_pass1`). -pub(crate) mod memo_population; /// Stage 1, Pass 1's computation targets, their candidates and their /// composition. No execution timing is assigned: that is a Stage 2 /// materialization decision. Its target list @@ -20,7 +16,5 @@ pub(crate) mod per_computation_candidates; mod per_computation_candidates_tests; pub(crate) mod realization; -pub use logical_memo_candidates::{ - generate_per_computation_logical_memo_candidates, PerComputationLogicalMemoCandidatesError, -}; +pub use logical_memo_candidates::{pass1_memo, Pass1Error}; pub use per_computation_candidates::PerComputationCandidateError; diff --git a/crates/logical-asap-optimization/src/pass1/logical_memo_candidates.rs b/crates/logical-asap-optimization/src/pass1/logical_memo_candidates.rs index 1d20e5e79..066f41724 100644 --- a/crates/logical-asap-optimization/src/pass1/logical_memo_candidates.rs +++ b/crates/logical-asap-optimization/src/pass1/logical_memo_candidates.rs @@ -1,310 +1,53 @@ -//! Stage 1, Pass 1's entry and output: the per-computation candidates attached to the -//! memo groups of the workload's hash-consed DAG -//! ([`PerComputationLogicalMemoCandidates`]), which Pass 2 extends. - -use std::rc::Rc; +//! Stage 1, Pass 1's entry and output: each group's initial candidates +//! (exact, exact accumulator, one summary build per listed parameter +//! configuration, fused builds), with every intermediate result its own +//! group (a [`Memo`]), which Pass 2 extends. use asap_types::ir::properties::AccuracyTarget; -use asap_types::ir::{QueryRoot, SchemaDerivationError}; use asap_types::workload::DataWorkload; -use asap_accuracy::sizing::{ - per_input_failure_probability, with_summaries_sized_for_failure_probability, -}; use asap_accuracy::{AccuracyEvidenceProvider, AccuracyModel}; -use super::memo_population::try_populate_pass1; -use super::memo_population::CandidatePopulationError; use crate::deployment_capabilities::SupportedSummary; use crate::pass1::per_computation_candidates::{ generate_per_computation_candidates, PerComputationCandidateError, }; -use asap_types::stages::{ - DagOfLogicalMemoGroups, Memo, PerComputationLogicalMemoCandidates, PrunedCandidate, -}; +use asap_types::stages::Memo; -/// Why [`generate_per_computation_logical_memo_candidates`] failed. +/// Why [`pass1_memo`] failed. #[derive(Debug, thiserror::Error)] -pub enum PerComputationLogicalMemoCandidatesError { +pub enum Pass1Error { /// A query's DAG cannot be planned: it already holds ASAP or physical /// annotations, or a measure's accuracy target is invalid. #[error(transparent)] PerComputation(#[from] PerComputationCandidateError), - /// The targets do not belong to the DAG of logical memo groups. - #[error(transparent)] - Population(#[from] CandidatePopulationError), - /// A candidate's summaries sized for its readers' failure probability - /// did not rebuild. - #[error(transparent)] - Sizing(#[from] SchemaDerivationError), } -/// Stage 1, Pass 1: the per-computation candidates on the memo groups of -/// `dag_of_logical_memo_groups`. It finds one computation target per +/// Stage 1, Pass 1: the initial candidates of the memo groups of `memo` +/// (Stage 0's, without candidates). It finds one computation target per /// measure of each aggregate group with a realization besides exact /// execution, drops the summaries the deployment does not support /// (`supported_summaries`, the deployment's /// `DeploymentCapabilities::supported_summaries`; `None` supports every /// one) and, unless `accuracy` or `evidence` may accept an output without /// its guarantee, the summaries no query reading them can use (a query -/// `q`'s target is `accuracy_targets[q]`). It then attaches each group's -/// candidates, each with its summaries sized for its readers' failure -/// probabilities (`size_for_failure_probability`): one sized candidate -/// per accuracy-budget split. Every pruned way to compute a measure is in -/// the output's `pruned_candidates`. -pub fn generate_per_computation_logical_memo_candidates( - dag_of_logical_memo_groups: DagOfLogicalMemoGroups, +/// `q`'s target is `accuracy_targets[q]`). It then adds each way to compute +/// a group as candidates, through `Memo::find_or_add_group` and +/// `Memo::add_candidate`. +pub fn pass1_memo( + memo: Memo, accuracy_targets: &[AccuracyTarget], accuracy: &dyn AccuracyModel, evidence: &dyn AccuracyEvidenceProvider, data: &DataWorkload, supported_summaries: Option<&[SupportedSummary]>, -) -> Result { - let mut per_computation = generate_per_computation_candidates(&dag_of_logical_memo_groups)? +) -> Result { + let mut per_computation = generate_per_computation_candidates(&memo)? .without_unsupported_summaries(supported_summaries); if !accuracy.may_accept_regardless_of_guarantee() && !evidence.may_provide_evidence() { per_computation = per_computation.without_summaries_no_query_can_use(accuracy_targets, data, accuracy); } - let mut memo = try_populate_pass1(dag_of_logical_memo_groups, &per_computation)?; - size_for_failure_probability(&mut memo, &per_computation.roots, accuracy_targets, data)?; - let pruned_candidates = per_computation - .pruned_ways_to_compute - .into_iter() - .map(|pruned| PrunedCandidate { - group: per_computation.targets[pruned.target].aggregate_group, - candidate: pruned.way_to_compute, - reason: pruned.reason, - }) - .collect(); - Ok(PerComputationLogicalMemoCandidates { - memo, - pruned_candidates, - }) -} - -/// Sizes every candidate of `memo` for its readers' failure probabilities, -/// in place: each approximate input of a candidate with `n` -/// accuracy budget parts (`accuracy_budget_parts`) gets the smallest -/// [`per_input_failure_probability`] `δ_q / (n · V_q)` among the queries `q` -/// reaching its group, `V_q` the values `q`'s target covers (`G` of its -/// root, `roots[q]`, for the whole result) -/// ([`with_summaries_sized_for_failure_probability`]). These are the candidates' final -/// sizes: Pass 2 inherits them or takes the strictest shared reader's -/// ([`size_for_all_readers`](asap_accuracy::estimators::size_for_all_readers)), -/// and Stage 3 only checks the composed accuracy. A candidate no reader -/// gives a failure probability (exact, per-value with no `δ`, or a -/// whole-result target over an unbound `G`) keeps its Pass 1 size. -fn size_for_failure_probability( - memo: &mut Memo, - roots: &[QueryRoot], - accuracy_targets: &[AccuracyTarget], - data: &DataWorkload, -) -> Result<(), PerComputationLogicalMemoCandidatesError> { - for (group, candidates) in memo.groups.iter().zip(&mut memo.candidates) { - for candidate in candidates.iter_mut() { - let Some(failure_probability) = group - .reached_by_queries() - .iter() - .filter_map(|&query| { - per_input_failure_probability( - accuracy_targets.get(query)?, - candidate.accuracy_budget_parts, - roots.get(query)?, - data, - ) - }) - .reduce(f64::min) - else { - continue; - }; - let sized = with_summaries_sized_for_failure_probability( - &[QueryRoot::Operator(Rc::clone(&candidate.root_node))], - failure_probability, - data, - )?; - let [QueryRoot::Operator(node)] = &sized[..] else { - unreachable!("one operator root in, one out") - }; - candidate.root_node = Rc::clone(node); - } - } - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::stage0::generate_dag_of_logical_memo_groups; - use crate::test_support::{ - lower_promql, shared_summary_state_groups, sharing_rules_and_shared_state_groups, - }; - use asap_accuracy::{IgnoreAccuracyGuarantees, NoAccuracyEvidence}; - use asap_types::ir::properties::{AccuracyMetric, AccuracyScope, AccuracyTarget}; - use asap_types::ir::schema::{FieldDataType, SketchAlgorithm, SketchParams}; - use asap_types::ir::{ASAPOp, OperatorNode}; - use asap_types::stages::CandidateForMemoGroup; - use asap_types::workload::DurationMs; - - /// Rank error 0.01 at δ = 0.01 over `scope`. - fn rank_target(scope: AccuracyScope) -> AccuracyTarget { - AccuracyTarget::max_error(0.01) - .failure_probability(0.01) - .accuracy_metric(AccuracyMetric::Rank) - .accuracy_scope(scope) - } - - /// Metric `x` with `series` series (`None`: undeclared). - fn data(series: Option) -> DataWorkload { - let data = DataWorkload::sampled_every(["x".to_string()], DurationMs(1_000)); - match series { - Some(series) => data.with_series_count(series), - None => data, - } - } - - /// Pass 1's memo for `queries`, each lowered and entered with its target. - fn pass1(queries: &[(&str, AccuracyTarget)], data: &DataWorkload) -> Memo { - let roots: Vec = queries - .iter() - .map(|(query, target)| { - QueryRoot::Operator(lower_promql( - query, - target.clone().accuracy_scope(AccuracyScope::PerValue), - )) - }) - .collect(); - let targets: Vec = - queries.iter().map(|(_, target)| target.clone()).collect(); - generate_per_computation_logical_memo_candidates( - generate_dag_of_logical_memo_groups(&roots), - &targets, - &IgnoreAccuracyGuarantees, - &NoAccuracyEvidence, - data, - None, - ) - .unwrap() - .memo - } - - /// The failure probability of every KLL `candidates` build. - fn kll_failure_probabilities<'a>( - candidates: impl IntoIterator, - ) -> Vec> { - candidates - .into_iter() - .flat_map(|candidate| OperatorNode::reachable(&candidate.root_node)) - .filter_map(|node| match node.asap() { - Some(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), - .. - }) if *kind.algorithm() == SketchAlgorithm::Kll => match kind.params() { - SketchParams::Kll { - failure_probability, - .. - } => Some(*failure_probability), - _ => None, - }, - _ => None, - }) - .collect() - } - - /// The failure probability of every KLL build of `memo`'s candidates - /// for one accuracy budget part (the whole budget). - fn whole_share_kll(memo: &Memo) -> Vec> { - kll_failure_probabilities( - memo.candidates - .iter() - .flatten() - .filter(|candidate| candidate.accuracy_budget_parts == 1), - ) - } - - /// Pass 1 sizes each sketch for `δ / V`, `V` the values its query's - /// target covers: one for a per-value target, `⌊1 / share⌋` = 20 for - /// the groups holding at least 5% of the records (whether or not `G` is - /// declared), `G` = 10 000 series for the whole result. A whole-result - /// target over an undeclared `G` leaves the per-value size (KLL's - /// default failure probability, `None`), which Stage 3 then rejects. - #[test] - fn pass1_sizes_each_sketch_for_the_values_its_target_covers() { - let query = "quantile_over_time(0.99, x[5m])"; - let five_percent = AccuracyScope::GroupsWithRecordShareAtLeast(0.05); - for (scope, series, failure_probability) in [ - (AccuracyScope::PerValue, Some(10_000), None), - (five_percent, None, Some(0.01 / 20.0)), - (five_percent, Some(10_000), Some(0.01 / 20.0)), - ( - AccuracyScope::WholeResult, - Some(10_000), - Some(0.01 / 10_000.0), - ), - (AccuracyScope::WholeResult, None, None), - ] { - let memo = pass1(&[(query, rank_target(scope))], &data(series)); - let found = whole_share_kll(&memo); - assert!(!found.is_empty(), "{scope:?}"); - assert!( - found.iter().all(|f| *f == failure_probability), - "{scope:?} {series:?}: {found:?}" - ); - } - } - - /// A group several queries reach is sized for the strictest of them: - /// the per-series quantiles are both a whole-result query's value (over - /// `G` = 10 000 series) and the input of a global `max` (one value), so - /// they take `δ / 10 000`. - #[test] - fn a_group_several_queries_reach_takes_the_strictest_failure_probability() { - let whole = rank_target(AccuracyScope::WholeResult); - let memo = pass1( - &[ - ("quantile_over_time(0.99, x[5m])", whole.clone()), - ("max(quantile_over_time(0.99, x[5m]))", whole), - ], - &data(Some(10_000)), - ); - let found = whole_share_kll(&memo); - assert!(!found.is_empty()); - assert!( - found.iter().all(|f| *f == Some(0.01 / 10_000.0)), - "{found:?}" - ); - } - - /// Pass 2 shares one build among readers sized differently in Pass 1 - /// at the strictest reader's size: a per-value 0.95 quantile (δ) and a - /// whole-result 0.99 quantile over `G` = 10 000 series (δ / 10 000) - /// share a KLL sized for δ / 10 000. Stage 3 does not resize it. - #[test] - fn a_shared_build_takes_its_strictest_readers_size() { - let memo = pass1( - &[ - ( - "quantile_over_time(0.95, x[5m])", - rank_target(AccuracyScope::PerValue), - ), - ( - "quantile_over_time(0.99, x[5m])", - rank_target(AccuracyScope::WholeResult), - ), - ], - &data(Some(10_000)), - ); - let memo = sharing_rules_and_shared_state_groups(memo).unwrap(); - let found = kll_failure_probabilities( - shared_summary_state_groups(&memo) - .iter() - .filter(|group| memo.groups[group.0 as usize].reached_by_queries() == [0, 1]) - .flat_map(|group| &memo.candidates[group.0 as usize]), - ); - assert!(!found.is_empty()); - assert!( - found.iter().all(|f| *f == Some(0.01 / 10_000.0)), - "{found:?}" - ); - } + let _ = (memo, per_computation); + todo!("#1207 PR 3: add each way to compute as candidates, one per listed θ") } diff --git a/crates/logical-asap-optimization/src/pass1/memo_population.rs b/crates/logical-asap-optimization/src/pass1/memo_population.rs deleted file mode 100644 index b3b411f1a..000000000 --- a/crates/logical-asap-optimization/src/pass1/memo_population.rs +++ /dev/null @@ -1,1231 +0,0 @@ -//! The hash-consed group DAG and Pass 1's implementation candidates per -//! group (design steps 1 and 2, Pass 1 part). -//! -//! Groups are the workload's logical DAG interned by -//! [`crate::stage0::cse::share_common_sub_dags`], the identical-expression rule's legality -//! included: structurally equal nodes are one group when sharing them is -//! legal: an ASAP node, or a non-ASAP node that returns the same rows for -//! the same input rows (`NonASAPOp::is_shareable_when_equal`). -//! A unique key is not needed; an unordered `LIMIT` stays one group per -//! reader. -//! -//! A Pass 1 target's aggregate group gets one candidate per combination of -//! its measures' Pass 1 candidates: the aggregate alone composed by -//! [`super::per_computation_candidates::compose_logical_candidate`] with that choice, every target beneath it -//! exact. Its local nodes are the composed nodes above the first node -//! equal to one of the group's descendant groups, which are its children. -//! The local nodes read each child group through its node (the same -//! `Rc`), and the candidate outputs the group's schema, so a parent reads -//! it in the group's place (a query's top-k root, which no parent reads, -//! may output the ranked value beside the key). -//! -//! A fused computation is one more candidate of the outer group, whatever -//! else reads the groups it absorbs: those keep their own candidates for -//! their other readers. A candidate equal to one the group already has (the -//! same structural id, input groups, fused groups and accuracy budget parts) is not -//! added again. -//! -//! A query whose value combines `n` approximate inputs splits its budget -//! into `n` equal shares, so a summary's size depends on its readers: each -//! sketch candidate is offered once per share `1..=n`, `n` the most -//! approximate inputs a query reaching the group can have -//! ([`CandidateForMemoGroup::accuracy_budget_parts`]). This records the sizing -//! configuration used during generation; the concrete sketch parameters carry -//! the resulting error contract. The connection planner may select a conservative -//! larger configuration, and validates the recomposed query-root guarantee. - -use asap_types::stages::HowToComputeAggregateMeasure; -use std::collections::{HashMap, HashSet}; -use std::rc::Rc; - -use super::per_computation_candidates::{ - compose_logical_candidate_with_accuracy_budget_parts, PerComputationCandidates, -}; -use asap_types::ir::schema::FieldDataType; -use asap_types::ir::{ASAPOp, OperatorNode, QueryRoot}; - -use asap_types::stages::{ - CandidateForMemoGroup, DagOfLogicalMemoGroups, Memo, MemoGroup, MemoGroupId, - OriginOfCandidateForMemoGroup, -}; - -/// A Pass 1 inventory does not belong to the initial logical memo. -#[derive(Debug, thiserror::Error, PartialEq, Eq)] -pub enum CandidatePopulationError { - #[error("Pass 1 root dependencies differ from the initial logical memo")] - RootDependenciesChanged, - #[error("Pass 1 target {target} is not a valid measure of an initial aggregate group")] - InvalidTarget { target: usize }, - #[error("Pass 1 target {target} references missing absorbed target {absorbed}")] - InvalidAbsorption { target: usize, absorbed: usize }, -} - -/// Populate implementation candidates without changing initial group identities. -/// The inventory must have walked `initial`'s groups -/// (`PerComputationCandidates::walked`); its targets name them by id. -pub fn try_populate_pass1( - initial: DagOfLogicalMemoGroups, - per_computation: &PerComputationCandidates, -) -> Result { - if !per_computation.walked(&initial) { - return Err(CandidatePopulationError::RootDependenciesChanged); - } - for (index, target) in per_computation.targets.iter().enumerate() { - if !matches!( - per_computation.aggregate(target).non_asap(), - Some(asap_types::ir::NonASAPOp::Aggregate { measures, .. }) if target.measure_index < measures.len() - ) { - return Err(CandidatePopulationError::InvalidTarget { target: index }); - } - if let Some(fused) = &target.fused_computation { - if let Some(&absorbed) = fused - .absorbed_targets - .iter() - .find(|&&absorbed| absorbed >= per_computation.targets.len()) - { - return Err(CandidatePopulationError::InvalidAbsorption { - target: index, - absorbed, - }); - } - } - } - let candidates = pass1_candidates(per_computation, &initial.groups); - Ok(Memo { - groups: initial.groups, - query_roots: initial.query_roots, - candidates, - }) -} - -/// `g` and every group below it. -fn reach(groups: &[MemoGroup], g: MemoGroupId, reached: &mut HashSet) { - if reached.insert(g) { - for &child in &groups[g.0 as usize].child_groups { - reach(groups, child, reached); - } - } -} - -/// Every group's Pass 1 candidates, by group index. -fn pass1_candidates( - per_computation: &PerComputationCandidates, - groups: &[MemoGroup], -) -> Vec> { - // Each target aggregate group's targets, one per measure: Pass 1 walks - // the groups, so an aggregate several queries read has one target per - // measure. - let mut targets_of: HashMap> = HashMap::new(); - for (t, target) in per_computation.targets.iter().enumerate() { - targets_of - .entry(target.aggregate_group) - .or_default() - .push(t); - } - // The groups' own nodes, which the composed candidates read. - let logical_nodes: Rc<[Rc]> = groups - .iter() - .map(|group| Rc::clone(&group.logical_node)) - .collect(); - let own = |g: usize| CandidateForMemoGroup { - fused_groups: Vec::new(), - root_node: Rc::clone(&groups[g].logical_node), - input_groups: groups[g].child_groups.clone(), - origin: OriginOfCandidateForMemoGroup::Pass1, - accuracy_budget_parts: 1, - }; - // Each query's most approximate inputs: its targets that offer a - // summary, each one sketch state at most. - let offers_a_summary = |t: usize| { - let target = &per_computation.targets[t]; - let summary = |c: &HowToComputeAggregateMeasure| { - matches!(c, HowToComputeAggregateMeasure::Summary(_)) - }; - target.ways_to_compute.iter().any(summary) - || target - .fused_computation - .as_ref() - .is_some_and(|fused| fused.ways_to_compute.iter().any(summary)) - }; - let mut most_approximate_inputs = vec![0usize; per_computation.roots.len()]; - for (g, targets) in &targets_of { - let summaries = targets.iter().filter(|&&t| offers_a_summary(t)).count(); - for &q in &groups[g.0 as usize].queries { - most_approximate_inputs[q] += summaries; - } - } - (0..groups.len()) - .map(|g| match targets_of.get(&MemoGroupId(g as u32)) { - None => vec![own(g)], - Some(targets) => target_candidates( - per_computation, - groups, - &logical_nodes, - MemoGroupId(g as u32), - targets, - groups[g] - .queries - .iter() - .map(|&q| most_approximate_inputs[q]) - .max() - .unwrap_or(1) - .max(1), - ) - .unwrap_or_else(|| vec![own(g)]), - }) - .collect() -} - -/// The candidates of target aggregate group `g`, whose measures' targets -/// are `targets`: one per combination of their Pass 1 candidates (the last -/// target varying fastest), each the aggregate composed alone with that -/// choice. `None` when none composes. -fn target_candidates( - per_computation: &PerComputationCandidates, - groups: &[MemoGroup], - logical_nodes: &Rc<[Rc]>, - g: MemoGroupId, - targets: &[usize], - max_accuracy_budget_parts: usize, -) -> Option> { - let aggregate = Rc::clone(&groups[g.0 as usize].logical_node); - // The groups a candidate's private nodes may bottom out at, by key. - let mut below = HashSet::new(); - for &child in &groups[g.0 as usize].child_groups { - reach(groups, child, &mut below); - } - let mut structural_ids = NodeToStructuralIdCache::default(); - let mut descendant: HashMap = HashMap::new(); - let mut ordered: Vec = below.into_iter().collect(); - ordered.sort_unstable(); - for d in ordered { - descendant - .entry(structural_ids.structural_id(&groups[d.0 as usize].logical_node)) - .or_insert(d); - } - let own_key = structural_ids.structural_id(&groups[g.0 as usize].logical_node); - let mut candidates = Vec::new(); - // Each candidate's identity in the memo: its node's structural id, its - // input groups, the groups it fuses and its accuracy budget parts. A candidate - // equal to one already listed is not added again. - let mut known = HashSet::new(); - let mut choice = vec![0; per_computation.targets.len()]; - loop { - let mut implements = vec![g]; - for &t in targets { - if let Some((_, Some(absorbs))) = per_computation.targets[t].computation_for(choice[t]) - { - for &a in absorbs { - let absorbed = per_computation.targets[a].aggregate_group; - if !implements.contains(&absorbed) { - implements.push(absorbed); - } - } - } - } - for accuracy_budget_parts in 1..=max_accuracy_budget_parts { - let Ok(composed) = compose_alone( - per_computation, - logical_nodes, - &aggregate, - targets, - &choice, - accuracy_budget_parts, - ) else { - continue; - }; - let Some(QueryRoot::Operator(root)) = composed.into_iter().next() else { - continue; - }; - if accuracy_budget_parts > 1 && !builds_a_sketch(&root) { - break; - } - let (node, children) = if structural_ids.structural_id(&root) == own_key { - ( - Rc::clone(&groups[g.0 as usize].logical_node), - groups[g.0 as usize].child_groups.clone(), - ) - } else { - let mut children = Vec::new(); - let node = over_groups( - &root, - groups, - &mut structural_ids, - &descendant, - &mut children, - &mut HashMap::new(), - ); - // A parent reads the candidate in the group's place, so it - // outputs the group's schema when it has the same columns, - // with the group's metadata (composition may derive a closed - // label set where the frontend's node leaves it open). A - // query's top-k root may output other columns (the ranked - // value beside the key): it keeps its own. - let group_node = &groups[g.0 as usize].logical_node; - let same_columns = node.schema.fields == group_node.schema.fields - && node.schema.time_index == group_node.schema.time_index; - let node = if !same_columns || node.schema == group_node.schema { - node - } else { - Rc::new(OperatorNode { - schema: group_node.schema.clone(), - ..(*node).clone() - }) - }; - (node, children) - }; - if !known.insert(( - structural_ids.structural_id(&node), - children.clone(), - implements[1..].to_vec(), - accuracy_budget_parts, - )) { - continue; - } - candidates.push(CandidateForMemoGroup { - fused_groups: implements[1..].to_vec(), - root_node: node, - input_groups: children, - origin: OriginOfCandidateForMemoGroup::Pass1, - accuracy_budget_parts, - }); - } - // Mixed-radix over this group's targets only. - let mut advanced = false; - for &t in targets.iter().rev() { - choice[t] += 1; - if choice[t] < per_computation.targets[t].choice_count() { - advanced = true; - break; - } - choice[t] = 0; - } - if !advanced { - break; - } - } - (!candidates.is_empty()).then_some(candidates) -} - -/// `aggregate` composed alone with `choice[t]` for each of its measures' -/// `targets`, every other target left exact: the inventory holds only -/// `targets`, and the targets a chosen fused computation absorbs (not -/// computed then). -fn compose_alone( - per_computation: &PerComputationCandidates, - logical_nodes: &Rc<[Rc]>, - aggregate: &Rc, - targets: &[usize], - choice: &[usize], - accuracy_budget_parts: usize, -) -> Result, super::per_computation_candidates::PerComputationCandidateError> { - let mut kept: Vec = targets.to_vec(); - for &t in targets { - if let Some((_, Some(absorbs))) = per_computation.targets[t].computation_for(choice[t]) { - for &a in absorbs { - if !kept.contains(&a) { - kept.push(a); - } - } - } - } - let position = |t: usize| kept.iter().position(|&k| k == t); - let alone = PerComputationCandidates { - roots: vec![QueryRoot::Operator(Rc::clone(aggregate))], - logical_nodes: Rc::clone(logical_nodes), - targets: kept - .iter() - .map(|&t| { - let mut target = per_computation.targets[t].clone(); - if let Some(fused) = &mut target.fused_computation { - // Absorbed targets not kept are never absorbed here: the - // fusion is kept only when chosen, with all of them. - match fused - .absorbed_targets - .iter() - .map(|&a| position(a)) - .collect() - { - Some(absorbs) => fused.absorbed_targets = absorbs, - None => target.fused_computation = None, - } - } - target - }) - .collect(), - pruned_ways_to_compute: Vec::new(), - }; - let alone_choice: Vec = kept - .iter() - .map(|&t| if targets.contains(&t) { choice[t] } else { 0 }) - .collect(); - compose_logical_candidate_with_accuracy_budget_parts( - &alone, - &alone_choice, - accuracy_budget_parts, - ) -} - -/// `node` with each first node on a path down from its children that -/// equals a group in `descendant` replaced by that group's node (the same -/// `Rc`), and those groups pushed to `children` in the order met: the -/// candidate's private nodes over its child groups. -fn over_groups( - node: &Rc, - groups: &[MemoGroup], - structural_ids: &mut NodeToStructuralIdCache, - descendant: &HashMap, - children: &mut Vec, - rebuilt: &mut HashMap<*const OperatorNode, Rc>, -) -> Rc { - let mut replaced: HashMap<*const OperatorNode, Rc> = HashMap::new(); - for child in node.children() { - let key = Rc::as_ptr(child); - if replaced.contains_key(&key) { - continue; - } - let new = match descendant.get(&structural_ids.structural_id(child)) { - Some(&group) => { - if !children.contains(&group) { - children.push(group); - } - Rc::clone(&groups[group.0 as usize].logical_node) - } - None => match rebuilt.get(&key) { - Some(done) => Rc::clone(done), - None => { - let done = - over_groups(child, groups, structural_ids, descendant, children, rebuilt); - rebuilt.insert(key, Rc::clone(&done)); - done - } - }, - }; - replaced.insert(key, new); - } - if node - .children() - .into_iter() - .all(|child| Rc::ptr_eq(child, &replaced[&Rc::as_ptr(child)])) - { - return Rc::clone(node); - } - Rc::new(OperatorNode { - operator: node - .operator - .map_children(|child| Rc::clone(&replaced[&Rc::as_ptr(child)])), - ..(**node).clone() - }) -} - -/// Whether `node`'s private nodes build a sketch, which its accuracy budget parts size. -fn builds_a_sketch(node: &Rc) -> bool { - OperatorNode::reachable(node).iter().any(|n| { - matches!( - n.asap(), - Some( - ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(..), - .. - } | ASAPOp::SummaryTumblingWindowChain { - family: FieldDataType::Sketch(..), - .. - } - ) - ) - }) -} - -/// Interned structural identities: the operator over its children's identities, -/// without the schema, guarantee, timing and materialization composition -/// derives. Memoized per node (the cache keeps each node alive, so its -/// address is not reused). -#[derive(Default)] -pub(crate) struct NodeToStructuralIdCache { - id_of_node: HashMap<*const OperatorNode, (Rc, u64)>, - id_of_encoded_operator: HashMap, u64>, -} - -impl NodeToStructuralIdCache { - pub(crate) fn structural_id(&mut self, node: &Rc) -> u64 { - if let Some((_, key)) = self.id_of_node.get(&Rc::as_ptr(node)) { - return *key; - } - let children: HashMap<*const OperatorNode, u64> = node - .children() - .into_iter() - .map(|child| (Rc::as_ptr(child), self.structural_id(child))) - .collect(); - let operator = node.operator.map_children(|c| children[&Rc::as_ptr(c)]); - // MessagePack preserves non-finite floats and signed zero. HashMap - // compares the complete byte key, so a hash collision cannot merge - // different computations. Cloned/recomposed NaN literals retain identity. - let encoded = - rmp_serde::to_vec(&operator).expect("an IR operator over child IDs must serialize"); - let next = self.id_of_encoded_operator.len() as u64; - let key = *self.id_of_encoded_operator.entry(encoded).or_insert(next); - self.id_of_node - .insert(Rc::as_ptr(node), (Rc::clone(node), key)); - key - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::stage0::generate_dag_of_logical_memo_groups; - - #[test] - fn structural_keys_preserve_nonfinite_and_signed_zero_literals() { - let input = crate::test_support::lower_promql( - "m", - asap_types::ir::properties::AccuracyTarget::exact(), - ); - let literal = |value| { - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP( - asap_types::ir::NonASAPOp::Project { - cols: vec![asap_types::ir::ProjectItem { - alias: Some("value".into()), - expr: asap_types::ir::ScalarExpr::literal_f64(value), - }], - qualifier: None, - child: Rc::clone(&input), - }, - )) - .unwrap() - }; - let mut structural_ids = NodeToStructuralIdCache::default(); - for (left, right) in [ - (f64::NAN, f64::INFINITY), - (f64::INFINITY, f64::NEG_INFINITY), - (0.0, -0.0), - ] { - assert_ne!( - structural_ids.structural_id(&literal(left)), - structural_ids.structural_id(&literal(right)) - ); - } - assert_eq!( - structural_ids.structural_id(&literal(1.0)), - structural_ids.structural_id(&literal(1.0)) - ); - let node = literal(f64::NAN); - assert_eq!( - structural_ids.structural_id(&node), - structural_ids.structural_id(&literal(f64::NAN)) - ); - } - - fn populate_pass1( - initial: DagOfLogicalMemoGroups, - inventory: &PerComputationCandidates, - ) -> Memo { - try_populate_pass1(initial, inventory).unwrap() - } - - fn memo_of(inventory: &PerComputationCandidates) -> Memo { - let roots: Vec<_> = inventory.roots.clone(); - populate_pass1(generate_dag_of_logical_memo_groups(&roots), inventory) - } - use crate::pass1::per_computation_candidates::{ - compose_logical_candidate, generate_per_computation_candidates, - }; - use crate::test_support::lower_promql; - use asap_types::ir::properties::AccuracyTarget; - - use asap_types::ir::ASAPOp; - - /// The cross-stage API rejects mismatched root identities without panicking. - #[test] - fn population_rejects_an_inventory_from_other_roots() { - let roots = vec![QueryRoot::Operator(lower_promql( - "quantile_over_time(0.95, m[5m])", - AccuracyTarget::max_error(0.01), - ))]; - let initial = generate_dag_of_logical_memo_groups(&roots); - // An inventory is tied to its roots by structure: the same query - // text lowered again is the same DAG, so the other roots must differ. - let other_roots = vec![QueryRoot::Operator(lower_promql( - "quantile_over_time(0.5, n[1m])", - AccuracyTarget::max_error(0.01), - ))]; - let inventory = - generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&other_roots)) - .unwrap(); - assert!(matches!( - try_populate_pass1(initial, &inventory), - Err(CandidatePopulationError::RootDependenciesChanged) - )); - } - - /// Invalid target and fusion indices fail at the stage boundary, not in composition. - #[test] - fn population_checks_target_and_absorption_indices() { - let roots = vec![QueryRoot::Operator(lower_promql( - "quantile_over_time(0.95, m[5m])", - AccuracyTarget::max_error(0.01), - ))]; - let initial = generate_dag_of_logical_memo_groups(&roots); - let inventory = - generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&roots)) - .unwrap(); - let mut invalid_measure = inventory.clone(); - invalid_measure.targets[0].measure_index = usize::MAX; - assert!(matches!( - try_populate_pass1(initial.clone(), &invalid_measure), - Err(CandidatePopulationError::InvalidTarget { target: 0 }) - )); - let mut invalid_absorption = inventory; - let missing = invalid_absorption.targets.len(); - invalid_absorption.targets[0].fused_computation = - Some(asap_types::stages::FusedComputation { - absorbed_targets: vec![missing], - ways_to_compute: Vec::new(), - }); - assert!(matches!(try_populate_pass1(initial, &invalid_absorption), - Err(CandidatePopulationError::InvalidAbsorption { target: 0, absorbed }) if absorbed == missing)); - } - - /// Capability pruning must change implementations, not logical identities. - #[test] - fn initial_groups_survive_pass1_expansion_and_pruning() { - let roots: Vec<_> = [ - "quantile_over_time(0.95, m[5m])", - "quantile_over_time(0.99, m[5m])", - ] - .iter() - .map(|text| QueryRoot::Operator(lower_promql(text, AccuracyTarget::max_error(0.01)))) - .collect(); - let initial = generate_dag_of_logical_memo_groups(&roots); - let original_groups = initial.groups.clone(); - let original_roots = initial.query_roots.clone(); - let inventory = generate_per_computation_candidates(&initial).unwrap(); - let expanded = populate_pass1(initial.clone(), &inventory); - let restricted_inventory = inventory.without_unsupported_summaries(Some(&[])); - let restricted = populate_pass1(initial, &restricted_inventory); - for memo in [&expanded, &restricted] { - assert_eq!(memo.groups.len(), original_groups.len()); - for (before, after) in original_groups.iter().zip(&memo.groups) { - assert!(Rc::ptr_eq(&before.logical_node, &after.logical_node)); - assert_eq!(before.child_groups, after.child_groups); - assert_eq!(before.parents, after.parents); - assert_eq!(before.queries, after.queries); - } - assert_eq!(memo.query_roots, original_roots); - assert_ne!( - group_of_kind(memo, 0, "Aggregate"), - group_of_kind(memo, 1, "Aggregate") - ); - assert!(memo.candidates.iter().all(|c| !c.is_empty())); - } - assert!( - expanded.candidates.iter().map(Vec::len).sum::() - > restricted.candidates.iter().map(Vec::len).sum::() - ); - } - - /// The memo of PromQL `queries`, each with accuracy target `target`. - fn memo(queries: &[&str], target: AccuracyTarget) -> Memo { - let roots: Vec<_> = queries - .iter() - .map(|q| QueryRoot::Operator(lower_promql(q, target.clone()))) - .collect(); - memo_of( - &generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&roots)) - .unwrap(), - ) - } - - /// The group of the first node of `kind` reached from query `q`'s root, - /// walking down the groups' first children. - fn group_of_kind(memo: &Memo, q: usize, kind: &str) -> usize { - let mut g = memo.query_roots[q].as_operator().unwrap().0 as usize; - while memo.groups[g].logical_node.operator.kind_name() != kind { - g = memo.groups[g].child_groups[0].0 as usize; - } - g - } - - /// The summary algorithms a candidate's private nodes build. - fn algorithms(candidate: &CandidateForMemoGroup) -> Vec { - OperatorNode::reachable(&candidate.root_node) - .iter() - .filter_map(|node| match node.asap() { - Some(ASAPOp::SummaryAgg { family, .. }) => Some(format!("{family:?}")), - _ => None, - }) - .collect() - } - - /// The per-job rate sum both queries compute is one group, reached by - /// both, the root of the first and read by the second's arithmetic. - #[test] - fn identical_sub_expressions_of_two_queries_are_one_group() { - let memo = memo( - &[ - "sum by (job) (rate(m[5m]))", - "sum by (job) (rate(m[5m])) * 2", - ], - AccuracyTarget::exact(), - ); - let sum = group_of_kind(&memo, 0, "Aggregate"); - assert_eq!(group_of_kind(&memo, 1, "Aggregate"), sum); - let group = &memo.groups[sum]; - assert_eq!(group.queries, vec![0, 1]); - assert_eq!( - asap_types::stages::queries_rooted_at(&memo.query_roots, MemoGroupId(sum as u32)), - vec![0] - ); - let product = memo.query_roots[1].as_operator().unwrap().0 as usize; - assert_eq!(group.parents, vec![MemoGroupId(product as u32)]); - assert_eq!( - memo.groups[product].child_groups, - vec![MemoGroupId(sum as u32)] - ); - // Children come before parents. - for (g, group) in memo.groups.iter().enumerate() { - assert!(group.child_groups.iter().all(|c| (c.0 as usize) < g)); - } - } - - /// A quantile's aggregate group has an exact candidate (its own node over - /// its input's group) and a KLL and a DDSketch candidate over the same - /// input group. - #[test] - fn a_quantile_target_group_has_exact_kll_and_ddsketch_candidates() { - let memo = memo( - &["quantile_over_time(0.9, m[5m])"], - AccuracyTarget::max_error(0.01), - ); - let quantile = group_of_kind(&memo, 0, "Aggregate"); - let input = memo.groups[quantile].child_groups.clone(); - let candidates = &memo.candidates[quantile]; - assert_eq!(candidates.len(), 3); - assert!(Rc::ptr_eq( - &candidates[0].root_node, - &memo.groups[quantile].logical_node - )); - assert!(algorithms(&candidates[0]).is_empty()); - assert!(algorithms(&candidates[1])[0].contains("Kll")); - assert!(algorithms(&candidates[2])[0].contains("DDSketch")); - for candidate in candidates { - assert!(candidate.fused_groups.is_empty()); - assert_eq!(candidate.input_groups, input); - assert_eq!(candidate.origin, OriginOfCandidateForMemoGroup::Pass1); - } - } - - /// The whole-expression top-k heap over the per-job sums implements both - /// aggregates' groups, and reads the sums' input group; the exact top-k - /// reads the sums' group. - #[test] - fn a_fused_candidate_lists_both_groups_it_implements() { - let memo = memo( - &["topk by (job) (10, sum_over_time(m[1m]))"], - AccuracyTarget::max_error(0.01).failure_probability(0.001), - ); - let topk = group_of_kind(&memo, 0, "Aggregate"); - let sums = memo.groups[topk].child_groups[0]; - let sums_input = memo.groups[sums.0 as usize].child_groups.clone(); - let candidates = &memo.candidates[topk]; - assert_eq!(candidates[0].input_groups, vec![sums]); - let fused: Vec<_> = candidates - .iter() - .filter(|c| !c.fused_groups.is_empty()) - .collect(); - assert!(!fused.is_empty()); - for candidate in fused { - assert_eq!(candidate.fused_groups, vec![sums]); - assert_eq!(candidate.input_groups, sums_input); - assert!(algorithms(candidate)[0].contains("Heap")); - } - } - - /// Every candidate reads its child groups through their nodes (the same - /// `Rc`), and outputs its group's schema, so a parent can read it in - /// the group's place. - #[test] - fn candidates_read_child_group_nodes_and_output_the_group_schema() { - for (queries, target) in [ - ( - vec!["quantile_over_time(0.9, m[5m]) / quantile_over_time(0.5, m[5m])"], - AccuracyTarget::max_error(0.01), - ), - ( - vec!["topk by (job) (10, sum_over_time(m[1m]))"], - AccuracyTarget::max_error(0.01).failure_probability(0.001), - ), - (vec!["sum by (job) (rate(m[5m]))"], AccuracyTarget::exact()), - ] { - let memo = memo(&queries, target); - for (g, candidates) in memo.candidates.iter().enumerate() { - for candidate in candidates { - let reached: HashSet<*const OperatorNode> = - OperatorNode::reachable(&candidate.root_node) - .iter() - .map(Rc::as_ptr) - .collect(); - for child in &candidate.input_groups { - let node = &memo.groups[child.0 as usize].logical_node; - assert!(reached.contains(&Rc::as_ptr(node)), "{queries:?} g{g}"); - } - assert_eq!( - candidate.root_node.schema, - memo.groups[g].logical_node.schema - ); - } - } - } - } - - /// The sketch summaries of the first query of `queries` composed with - /// every target at its `algorithm` summary, by [`super::per_computation_candidates::compose_logical_candidate`]: - /// as the exhaustive search sizes them. - fn composed_sketches( - queries: &[&str], - target: &AccuracyTarget, - algorithm: &str, - ) -> Vec { - let roots: Vec<_> = queries - .iter() - .map(|q| QueryRoot::Operator(lower_promql(q, target.clone()))) - .collect(); - let inventory = - generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&roots)) - .unwrap(); - let choice: Vec = inventory - .targets - .iter() - .map(|t| { - t.ways_to_compute - .iter() - .position(|c| { - matches!(c, HowToComputeAggregateMeasure::Summary(a) if format!("{a:?}") == algorithm) - }) - .unwrap() - }) - .collect(); - let composed = compose_logical_candidate(&inventory, &choice).unwrap(); - let QueryRoot::Operator(root) = &composed[0] else { - unreachable!("an operator root") - }; - OperatorNode::reachable(root) - .iter() - .filter_map(|n| match n.asap() { - Some(ASAPOp::SummaryAgg { family, .. }) => Some(format!("{family:?}")), - _ => None, - }) - .collect() - } - - /// The sketch candidates of `memo`'s group `g`: each one's summary and - /// accuracy budget parts. - fn sketch_shares(memo: &Memo, g: usize) -> Vec<(String, usize)> { - memo.candidates[g] - .iter() - .filter(|c| !algorithms(c).is_empty()) - .map(|c| (algorithms(c)[0].clone(), c.accuracy_budget_parts)) - .collect() - } - - /// `q0.9 / q0.5` beside a plain `q0.9`: each of the ratio's quantile - /// groups offers every sketch for the whole budget and for half of it, - /// since its query has two inputs that may be approximate. The plain - /// query's quantile is the numerator's group: equal per-series quantiles - /// return the same rows, so they are one group with both readers. A ratio of quantiles carries no - /// guarantee, so composition does not split its budget: the - /// whole-budget candidate is the exhaustive search's sizing, and the - /// half-budget one, larger, is never the cheaper valid one. - #[test] - fn a_ratio_operand_has_a_candidate_per_accuracy_budget_parts() { - let queries = [ - "quantile_over_time(0.9, m[5m]) / quantile_over_time(0.5, m[5m])", - "quantile_over_time(0.9, m[5m])", - ]; - let target = AccuracyTarget::max_error(0.01); - let memo = memo(&queries, target.clone()); - let ratio = memo.query_roots[0].as_operator().unwrap().0 as usize; - let operand = memo.groups[ratio].child_groups[0].0 as usize; - for operand in &memo.groups[ratio].child_groups { - let shares: Vec = sketch_shares(&memo, operand.0 as usize) - .into_iter() - .map(|(_, share)| share) - .collect(); - assert_eq!(shares, vec![1, 2, 1, 2]); - } - let plain = memo.query_roots[1].as_operator().unwrap().0 as usize; - assert_eq!(plain, operand); - let whole_kll = &sketch_shares(&memo, operand)[0]; - assert_eq!(whole_kll.1, 1); - assert_eq!( - composed_sketches(&queries[..1], &target, "Kll"), - vec![whole_kll.0.clone(); 2] - ); - } - - /// A difference of two Count-Min counts splits its budget: each - /// operand's half-budget candidate is the Count-Min composition sizes - /// for the difference with both counts sketched. - #[test] - fn a_split_budget_matches_its_half_budget_candidate() { - let queries = ["count by (job) (a) - count by (job) (b)"]; - let target = AccuracyTarget::max_error(0.01).failure_probability(0.001); - let memo = memo(&queries, target.clone()); - let difference = memo.query_roots[0].as_operator().unwrap().0 as usize; - let operand = memo.groups[difference].child_groups[0].0 as usize; - let half = sketch_shares(&memo, operand) - .into_iter() - .find(|(family, share)| family.contains("Cms") && *share == 2) - .unwrap(); - assert_eq!(composed_sketches(&queries, &target, "Cms"), vec![half.0; 2]); - } - - /// A group appended after its readers (as a transformation appends a - /// ranking group) is ordered before every group whose candidate reads - /// it. - #[test] - fn topological_order_puts_appended_groups_before_their_readers() { - let mut memo = memo( - &["quantile_over_time(0.9, m[5m])"], - AccuracyTarget::max_error(0.01), - ); - assert_eq!( - memo.topological_order().unwrap(), - (0..memo.groups.len() as u32) - .map(MemoGroupId) - .collect::>() - ); - let quantile = group_of_kind(&memo, 0, "Aggregate"); - let appended = MemoGroupId(memo.groups.len() as u32); - memo.groups.push(memo.groups[1].clone()); - memo.candidates.push(memo.candidates[1].clone()); - let mut reader = memo.candidates[quantile][0].clone(); - reader.input_groups = vec![appended]; - memo.candidates[quantile].push(reader); - let order = memo.topological_order().unwrap(); - let position = |g: MemoGroupId| order.iter().position(|&x| x == g).unwrap(); - assert!(position(appended) < position(MemoGroupId(quantile as u32))); - // A cycle has no order. - memo.candidates[1][0] - .input_groups - .push(MemoGroupId(quantile as u32)); - assert_eq!(memo.topological_order(), None); - } - - /// The workload generator of `joint_search_equivalence`: a seeded - /// xorshift generator. - struct Rng(u64); - - impl Rng { - fn below(&mut self, n: usize) -> usize { - self.0 ^= self.0 << 13; - self.0 ^= self.0 >> 7; - self.0 ^= self.0 << 17; - (self.0 % n as u64) as usize - } - - fn pick<'a, T>(&mut self, items: &'a [T]) -> &'a T { - &items[self.below(items.len())] - } - } - - /// One random PromQL query of `joint_search_equivalence`. - fn random_query(rng: &mut Rng, gauges: &[&str]) -> String { - let m = rng.pick(gauges); - let group = rng.pick(&["job", "job, instance", "instance"]); - let window = rng.pick(&["1m", "5m", "10m"]); - let phi = rng.pick(&["0.5", "0.9", "0.99"]); - let k = rng.pick(&["3", "5"]); - match rng.below(10) { - 0 => format!("sum by ({group}) (sum_over_time({m}[{window}]))"), - 1 => format!("avg by ({group}) (quantile_over_time({phi}, {m}[{window}]))"), - 2 => format!("quantile_over_time({phi}, {m}[{window}])"), - 3 => format!("topk by ({group}) ({k}, sum_over_time({m}[{window}]))"), - 4 => format!("sum by ({group}) (rate(c_total[{window}]))"), - 5 => format!("max by ({group}) (max_over_time({m}[{window}]))"), - 6 => format!("count_over_time({m}[{window}])"), - 7 => format!("sum by ({group}) (increase(c_total[{window}]))"), - 8 => format!("stddev_over_time({m}[{window}])"), - _ => format!("topk({k}, sum by ({group}) (rate(c_total[{window}])))"), - } - } - - /// The roots of `joint_search_equivalence`'s random workload `seed` - /// over `gauges`, lowered as the planner lowers PromQL (with series - /// identity). - fn random_roots(seed: u64, gauges: &[&str]) -> Vec { - use asap_types::ir::properties::AccuracyMetric; - use asap_types::workload::{ - DataArrival, DataWorkload, DurationMs, PlanningWorkload, Predictability, - PrometheusMetricType, Query, QueryLanguage, QueryRequirements, QueryWorkload, - RepeatedDemand, RepeatingEntry, RepetitionInterval, SourceFacts, - }; - let mut rng = Rng(seed.wrapping_mul(0x9E37_79B9_7F4A_7C15) | 1); - let queries = 2 + rng.below(3); - let entries = (0..queries) - .map(|_| RepeatingEntry { - query: Query(random_query(&mut rng, gauges)), - demand: RepeatedDemand::FixedInterval(RepetitionInterval( - *rng.pick(&[60_000u32, 300_000]), - )), - requirements: QueryRequirements { - accuracy: rng - .pick(&[ - AccuracyTarget::exact(), - AccuracyTarget::max_error(0.05), - AccuracyTarget::max_error(0.01).failure_probability(0.01), - AccuracyTarget::max_error(0.05).accuracy_metric(AccuracyMetric::Rank), - ]) - .clone(), - max_latency_ms: None, - }, - predictability: Predictability::Predictable { known_at: None }, - range_evaluation: None, - }) - .collect(); - let source = |metric_type| SourceFacts { - series_count: Some(1_000), - metric_type: Some(metric_type), - sample_interval: Some(DurationMs(15_000)), - ..SourceFacts::default() - }; - let workload = PlanningWorkload { - query_workload: QueryWorkload { - language: QueryLanguage::PromQL, - query_batch: None, - repeating_queries: Some(entries), - }, - data_workload: Some(DataWorkload { - arrival: DataArrival::Ingesting { - rows_per_second: 2_000.0 / 15.0, - }, - sources: gauges - .iter() - .map(|g| (g.to_string(), source(PrometheusMetricType::Gauge))) - .chain([("c_total".to_string(), source(PrometheusMetricType::Counter))]) - .collect(), - }), - }; - asap_frontend_promql::lower_promql_query_workload(&workload) - .unwrap() - .into_iter() - .map(|root| match root { - QueryRoot::Operator(node) => QueryRoot::Operator( - asap_types::ir::schema::schema_support::with_promql_series_identity(&node) - .unwrap(), - ), - scalar => scalar, - }) - .collect() - } - - /// Whether a composed sub-DAG is some combination of the memo's - /// per-group candidates: a candidate's private nodes equal the composed - /// nodes operator by operator (schemas and guarantees aside, which - /// composition derives), and each child group's node is matched by a - /// candidate of that group in turn. - struct Combination<'a> { - memo: &'a Memo, - matched: HashMap<(usize, *const OperatorNode), bool>, - own: HashMap<*const OperatorNode, (Rc, serde_json::Value)>, - } - - impl Combination<'_> { - fn own_fields(&mut self, node: &Rc) -> serde_json::Value { - self.own - .entry(Rc::as_ptr(node)) - .or_insert_with(|| { - let operator = node.operator.map_children(|_| ()); - (Rc::clone(node), serde_json::to_value(&operator).unwrap()) - }) - .1 - .clone() - } - - fn group(&mut self, g: usize, composed: &Rc) -> bool { - let key = (g, Rc::as_ptr(composed)); - if let Some(&done) = self.matched.get(&key) { - return done; - } - self.matched.insert(key, false); - let memo = self.memo; - let found = memo.candidates[g] - .iter() - .any(|c| self.private(&c.root_node, composed, &c.input_groups)); - self.matched.insert(key, found); - found - } - - fn private( - &mut self, - node: &Rc, - composed: &Rc, - children: &[MemoGroupId], - ) -> bool { - let memo = self.memo; - if let Some(child) = children - .iter() - .find(|c| Rc::ptr_eq(&memo.groups[c.0 as usize].logical_node, node)) - { - return self.group(child.0 as usize, composed); - } - let (ours, theirs) = (node.children(), composed.children()); - ours.len() == theirs.len() - && self.own_fields(node) == self.own_fields(composed) - && ours - .iter() - .zip(&theirs) - .all(|(a, b)| self.private(a, b, children)) - } - } - - /// For the random workloads of `joint_search_equivalence` (over one - /// gauge and over two), every whole-workload Pass 1 choice composes, - /// for each query, a plan that is a combination of the memo's per-group - /// candidates: the memo misses no Pass 1 candidate. Every target's - /// group has the target's own and fused candidates, also a target a - /// fused computation absorbs; every candidate of a group a parent reads - /// outputs the group's schema. - #[test] - fn every_composed_pass1_candidate_is_a_combination_of_group_candidates() { - let mut checked = 0; - let mut missing = Vec::new(); - let mut other_schema = Vec::new(); - for gauges in [&["m"][..], &["m", "n"]] { - for seed in 1..=60u64 { - let roots = random_roots(seed, gauges); - let inventory = generate_per_computation_candidates( - &generate_dag_of_logical_memo_groups(&roots), - ) - .unwrap(); - if inventory.combinations() > 400 { - continue; - } - let memo = memo_of(&inventory); - // Each target's own and fused candidates, absorbed or not. - let group_of = |t: usize| { - memo.groups - .iter() - .position(|group| { - *group.logical_node == **inventory.aggregate(&inventory.targets[t]) - }) - .unwrap() - }; - for (t, target) in inventory.targets.iter().enumerate() { - let g = group_of(t); - if inventory.targets.iter().enumerate().any(|(u, other)| { - u != t && group_of(u) == g && other.measure_index != target.measure_index - }) { - continue; - } - let whole_budget = |fused: bool| { - memo.candidates[g] - .iter() - .filter(|c| { - c.accuracy_budget_parts == 1 && !c.fused_groups.is_empty() == fused - }) - .count() - }; - let fused = target - .fused_computation - .as_ref() - .map_or(0, |f| f.ways_to_compute.len()); - if whole_budget(false) != target.ways_to_compute.len() - || whole_budget(true) != fused - { - missing.push(format!("{gauges:?} seed {seed} target {t} candidates")); - } - } - for (g, group) in memo.groups.iter().enumerate() { - for candidate in &memo.candidates[g] { - if !group.parents.is_empty() - && candidate.root_node.schema != group.logical_node.schema - { - other_schema.push(format!("{gauges:?} seed {seed} g{g}")); - } - } - } - let root_group = |q: usize| memo.query_roots[q].as_operator().unwrap().0 as usize; - let mut combination = Combination { - memo: &memo, - matched: HashMap::new(), - own: HashMap::new(), - }; - let mut choice = vec![0; inventory.targets.len()]; - loop { - if !inventory.absorbed_choice_varies(&choice) { - if let Ok(composed) = compose_logical_candidate(&inventory, &choice) { - for (q, root) in composed.iter().enumerate() { - let QueryRoot::Operator(node) = root else { - continue; - }; - checked += 1; - if !combination.group(root_group(q), node) { - missing.push(format!("{gauges:?} seed {seed} q{q} {choice:?}")); - } - } - } - } - let mut advanced = false; - for t in (0..choice.len()).rev() { - choice[t] += 1; - if choice[t] < inventory.targets[t].choice_count() { - advanced = true; - break; - } - choice[t] = 0; - } - if !advanced { - break; - } - } - } - } - assert!(checked > 100, "{checked} plans checked"); - assert!( - missing.is_empty(), - "{} of {checked} missing: {missing:#?}", - missing.len() - ); - } - - /// Two SQL queries over one table, each lowered with its own scan, read - /// one scan group: equal scans are shared, a table's keyless rows - /// included. Their aggregates, grouped differently, stay two groups. - #[test] - fn equal_sql_table_scans_are_one_group() { - use crate::test_support::{agg, scan_from}; - use asap_types::ir::operator::{AggIntent, Source}; - use asap_types::ir::schema::{DataType, Field, Schema}; - let table = || { - scan_from( - Source::Table { - table_ref: "t".into(), - }, - Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("value", DataType::Float64, false), - Field::plain("a", DataType::Utf8, true), - Field::plain("b", DataType::Utf8, true), - ], - 0, - vec![], - ), - ) - }; - let sum = || AggIntent::Sum { col: Some(1) }; - let roots = vec![ - QueryRoot::Operator(agg(vec![2], sum(), table())), - QueryRoot::Operator(agg(vec![3], sum(), table())), - ]; - let memo = memo_of( - &generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&roots)) - .unwrap(), - ); - assert_eq!(memo.groups.len(), 3); - assert_eq!(memo.groups[0].logical_node.operator.kind_name(), "Scan"); - assert_eq!(memo.groups[0].queries, vec![0, 1]); - assert_eq!(memo.groups[0].parents, vec![MemoGroupId(1), MemoGroupId(2)]); - } -} diff --git a/crates/logical-asap-optimization/src/pass1/per_computation_candidates.rs b/crates/logical-asap-optimization/src/pass1/per_computation_candidates.rs index ea2b891ca..96c7f469d 100644 --- a/crates/logical-asap-optimization/src/pass1/per_computation_candidates.rs +++ b/crates/logical-asap-optimization/src/pass1/per_computation_candidates.rs @@ -3,7 +3,7 @@ //! `docs/design_docs/proposals/planner-layering.md`. //! //! [`generate_per_computation_candidates`] walks the memo groups of the -//! workload's hash-consed logical DAG ([`DagOfLogicalMemoGroups`]) and +//! workload's memo groups ([`Memo`]) and //! returns, for every eligible computation, its unranked candidates. A //! computation ([`ComputationTarget`]) is one measure of an `Aggregate` //! group whose intent has a realization besides plain exact execution, once @@ -21,27 +21,23 @@ //! has its own candidates: the whole-expression top-k over a per-item //! `sum` or `count` (`Summary` of a heap family) and the nested sum, a //! grouped `sum` over a `sum` (`ExactAccumulator([Sum])`). -//! - Summary choices here are unsized families. Attaching them to memo -//! groups creates sized candidates, one per split of the accuracy budget -//! (`CandidateForMemoGroup::accuracy_budget_parts`), each sized for ε, its -//! accuracy budget parts and δ over the values its readers' targets cover; Pass 2 -//! inherits sizes or takes the strictest shared reader's, and Stage 3 only -//! checks. +//! - Summary choices here are unsized families. Adding them to memo groups +//! creates one build candidate per listed parameter configuration +//! (`parameter_list`); Stage 3 checks each query's composed accuracy at its +//! root. //! - Candidates are pruned only when provably unable to meet the accuracy //! requirement or to run, or when the exact candidate provably dominates -//! them (a PromQL top-k's heap over its instant vector, one row per item: -//! [`PruneReason::TopKHeapOverOneRowPerItem`]); each pruned candidate -//! keeps its [`PruneReason`]. +//! them (a PromQL top-k's heap over its instant vector, one row per item). //! - A measure filter or `HAVING` prunes nothing. The target records the //! measure filter (applied as `SummaryAgg.filter`) and one [`HavingEntry`] //! per consumer `Filter` directly above the aggregate: a key-only predicate -//! applied by every reader of the aggregate's group, across queries, may go +//! applied by every consumer of the aggregate's group, across queries, may go //! below the aggregate, anything else stays above the output (approximate //! outputs then get guarantee `None`). So every candidate of a group //! computes the group's own rows. //! -//! The input must be a logical DAG: ASAP operators, execution timing or a -//! materialization are rejected. No cost model is used. The accuracy model +//! The input must be a logical DAG: ASAP operators are rejected. A logical +//! DAG has no physical properties (#1200). No cost model is used. The accuracy model //! is used only to prune summaries no query reading them can use //! ([`PerComputationCandidates::without_summaries_no_query_can_use`]); //! ranking and discarding valid candidates belong to Stage 3. @@ -87,11 +83,11 @@ //! `1 − (1 + ε)^{−1/n}`) and `δ/n` for `n` inputs (#669b). Only the inputs //! approximate in this candidate count. This composition sizes for ε and δ //! per output value; attached to the memo groups, each candidate's -//! approximate inputs are sized for the values its readers' targets cover +//! approximate inputs are sized for the values its consumers' targets cover //! (`δ / (n · V)`, -//! [`generate_per_computation_logical_memo_candidates`](crate::pass1::logical_memo_candidates::generate_per_computation_logical_memo_candidates)), +//! [`pass1_memo`](crate::pass1::logical_memo_candidates::pass1_memo)), //! their final sizes. Pass 2 inherits them or takes the strictest shared -//! reader's, and Stage 3 only checks the composed accuracy. +//! consumer's, and Stage 3 only checks the composed accuracy. //! //! Every node of the candidate gets a theoretical guarantee, composed with //! the built-in accuracy model through operators (#669): exact over exact @@ -114,22 +110,20 @@ use asap_types::ir::scalar::{resolve_column_ref, ColumnRef, CompareOpKind, Scala use asap_types::ir::schema::{DataType, Schema}; use asap_types::ir::schema::{ EntityIdentity, ExactKind, FieldDataType, GroupingStrategy, NonNegativeWeightProof, - SketchAlgorithm, SketchKind, StatisticKind, SummaryInputExpr, SummaryUpdate, WeightDomain, + SketchAlgorithm, StatisticKind, SummaryInputExpr, SummaryUpdate, WeightDomain, }; use asap_types::ir::{ ASAPOp, BinaryOperator, ExprSemantics, NonASAPOp, Operator, OperatorNode, OperatorResultKind, Predicate, ProjectItem, QueryRoot, ScalarExpr, SchemaDerivationError, }; -use asap_types::physical::Materialization; use thiserror::Error; use asap_accuracy::composition::parse_raw_data_samples_sign_statistic; use asap_accuracy::estimators::hll; use asap_accuracy::propagation::{ - approximate_inputs, bind_guarantees, raw_data_samples_of_metric, root_guarantee, - with_guarantees, + bind_guarantees, raw_data_samples_of_metric, root_guarantee, with_guarantees, }; -use asap_accuracy::sizing::{accuracy_budget, sized, sizing_target, sketch_statistic}; +use asap_accuracy::sizing::{accuracy_budget, sizing_target, sketch_statistic}; use asap_accuracy::AccuracyModel; use asap_types::ir::properties::AccuracyGuarantee; use asap_types::workload::bounds::{declared_non_negative, series_read, values_per_result}; @@ -138,28 +132,23 @@ use asap_types::workload::DataWorkload; use crate::deployment_capabilities::{summary_supported, SupportedSummary}; use crate::pass1::realization::summary_candidates; use asap_types::stages::{ - ComputationTarget, DagOfLogicalMemoGroups, FusedComputation, HavingEntry, HavingPlacement, - HowToComputeAggregateMeasure, MemoGroupId, PruneReason, + ComputationTarget, FusedComputation, HavingEntry, HavingPlacement, + HowToComputeAggregateMeasure, Memo, MemoGroupId, }; /// Unranked candidates of one measure and the candidates pruned for it. #[derive(Debug, Clone, PartialEq)] pub(super) struct ComputationChoices { pub candidates: Vec, - pub pruned: Vec<(HowToComputeAggregateMeasure, PruneReason)>, + pub pruned: Vec, } impl ComputationChoices { /// Prune matching candidates; a pruned exact accumulator form is replaced by /// plain `Exact`, so the target keeps one exact option. - fn prune( - &mut self, - reason: PruneReason, - applies: impl Fn(&HowToComputeAggregateMeasure) -> bool, - ) { + fn prune(&mut self, applies: impl Fn(&HowToComputeAggregateMeasure) -> bool) { let pruned = prune_own(&mut self.candidates, applies); - self.pruned - .extend(pruned.into_iter().map(|choice| (choice, reason))); + self.pruned.extend(pruned); } } @@ -183,22 +172,10 @@ fn prune_own( pruned } -/// A way to compute a target's measure that Pass 1 pruned, and why. -#[derive(Debug, Clone, PartialEq)] -pub(crate) struct PrunedWayToCompute { - /// The target, by index in [`PerComputationCandidates::targets`]. - pub target: usize, - /// Whether it was a way of the target's fused computation - /// ([`ComputationTarget::fused_computation`]) rather than its own. - pub of_fused_computation: bool, - pub way_to_compute: HowToComputeAggregateMeasure, - pub reason: PruneReason, -} - /// Compact Pass 1 inventory over the memo groups it walked: one target per /// measure of each aggregate group, however many queries read the group. /// Pass 1's working form, not its output: -/// [`generate_per_computation_logical_memo_candidates`](crate::pass1::generate_per_computation_logical_memo_candidates) +/// [`pass1_memo`](crate::pass1::pass1_memo) /// builds, prunes and attaches it to the memo groups. Public only to tests /// (feature `test-support`). #[derive(Debug, Clone)] @@ -212,56 +189,9 @@ pub struct PerComputationCandidates { /// The targets; a whole-workload choice has one digit per target, /// ranging over [`ComputationTarget::choice_count`]. pub targets: Vec, - /// The ways to compute the targets' measures that Pass 1 pruned, in the - /// order it pruned them. - pub(crate) pruned_ways_to_compute: Vec, } impl PerComputationCandidates { - /// Whether this inventory walked `dag`, or a DAG built the same way from - /// the same roots: the same groups (by structural id, in the same order) - /// and each query's root over the same groups. Its targets' group ids - /// then name `dag`'s groups. - pub(crate) fn walked(&self, dag: &DagOfLogicalMemoGroups) -> bool { - use super::memo_population::NodeToStructuralIdCache; - let mut structural_ids = NodeToStructuralIdCache::default(); - let group_of: HashMap<*const OperatorNode, MemoGroupId> = self - .logical_nodes - .iter() - .enumerate() - .map(|(g, node)| (Rc::as_ptr(node), MemoGroupId(g as u32))) - .collect(); - dag.groups.len() == self.logical_nodes.len() - && dag.query_roots.len() == self.roots.len() - && dag - .groups - .iter() - .zip(self.logical_nodes.iter()) - .all(|(group, node)| { - Rc::ptr_eq(&group.logical_node, node) - || structural_ids.structural_id(&group.logical_node) - == structural_ids.structural_id(node) - }) - && dag - .query_roots - .iter() - .zip(&self.roots) - .all(|(groups, root)| { - let nodes = match root { - QueryRoot::Operator(node) => vec![node], - QueryRoot::Scalar(expr) => expr.operator_refs(), - }; - matches!(groups, QueryRoot::Operator(_)) - == matches!(root, QueryRoot::Operator(_)) - && groups.operator_refs().len() == nodes.len() - && groups - .operator_refs() - .into_iter() - .zip(nodes) - .all(|(g, node)| group_of.get(&Rc::as_ptr(node)) == Some(g)) - }) - } - /// The `Aggregate` node of `target`'s memo group. pub fn aggregate(&self, target: &ComputationTarget) -> &Rc { &self.logical_nodes[target.aggregate_group.0 as usize] @@ -323,37 +253,18 @@ impl PerComputationCandidates { }) } - /// The ways to compute target `target`'s measure that Pass 1 pruned, - /// with their reasons, in pruning order: its own, or its fused - /// computation's when `of_fused_computation`. - #[cfg(any(test, feature = "test-support"))] - pub fn pruned_of_target( - &self, - target: usize, - of_fused_computation: bool, - ) -> Vec<(HowToComputeAggregateMeasure, PruneReason)> { - self.pruned_ways_to_compute - .iter() - .filter(|pruned| { - pruned.target == target && pruned.of_fused_computation == of_fused_computation - }) - .map(|pruned| (pruned.way_to_compute.clone(), pruned.reason)) - .collect() - } - /// This inventory without the summary candidates the executor does not /// support (`supported_summaries`, the deployment's /// `DeploymentCapabilities::supported_summaries`; `None` supports - /// every one): a target's own are moved to its pruned candidates with - /// [`PruneReason::UnsupportedByDeployment`], and so are a fused - /// computation's (with all of them pruned, the fusion is not offered). + /// every one): a target's own are removed, and so are a fused + /// computation's (with all of them removed, the fusion is not offered). /// The readout is the statistic the measure's estimate reads (the outer measure's for a /// fused computation). With every summary supported it is unchanged. pub fn without_unsupported_summaries( mut self, supported_summaries: Option<&[SupportedSummary]>, ) -> Self { - for (index, target) in self.targets.iter_mut().enumerate() { + for target in self.targets.iter_mut() { let aggregate = &self.logical_nodes[target.aggregate_group.0 as usize]; let Some(NonASAPOp::Aggregate { measures, .. }) = aggregate.non_asap() else { continue; @@ -366,24 +277,9 @@ impl PerComputationCandidates { matches!(candidate, HowToComputeAggregateMeasure::Summary(family) if !summary_supported(supported_summaries, family, readout)) }; - let pruned = |of_fused_computation| { - move |way_to_compute| PrunedWayToCompute { - target: index, - of_fused_computation, - way_to_compute, - reason: PruneReason::UnsupportedByDeployment, - } - }; - let own = prune_own(&mut target.ways_to_compute, unsupported); - self.pruned_ways_to_compute - .extend(own.into_iter().map(pruned(false))); + prune_own(&mut target.ways_to_compute, unsupported); if let Some(fused) = &mut target.fused_computation { - let (removed, kept): (Vec<_>, _) = std::mem::take(&mut fused.ways_to_compute) - .into_iter() - .partition(unsupported); - fused.ways_to_compute = kept; - self.pruned_ways_to_compute - .extend(removed.into_iter().map(pruned(true))); + fused.ways_to_compute.retain(|way| !unsupported(way)); } } self @@ -391,8 +287,7 @@ impl PerComputationCandidates { /// This inventory without the summary candidates no query can use /// (#1047, accuracy A1 of #1095): a summary of a target (its own, or - /// its fused computation's) is moved to its pruned candidates with - /// [`PruneReason::NoErrorBoundForQueryTarget`] when, for every query + /// its fused computation's) is removed when, for every query /// whose root reaches the target's aggregate, /// `output_can_meet_target` says no: the query's output, composed /// with that summary and every other target exact, has no error bound @@ -413,7 +308,7 @@ impl PerComputationCandidates { /// pruned, and with every fused summary pruned the fusion is not /// offered. A summary whose composition fails is kept. /// - /// Pass 1's entry (`generate_per_computation_logical_memo_candidates`) + /// Pass 1's entry (`pass1_memo`) /// calls it only when the deployment's accuracy model and evidence /// cannot accept an output without its guarantee. pub fn without_summaries_no_query_can_use( @@ -471,9 +366,7 @@ impl PerComputationCandidates { }; let mut choices = vec![0; self.targets.len()]; choices[index] = choice; - let Ok(composed) = - compose_with_accuracy_budget_parts(&self, &roots, &choices, Some(1)) - else { + let Ok(composed) = compose(&self, &roots, &choices) else { continue; }; if !queries @@ -550,24 +443,11 @@ impl PerComputationCandidates { fused.ways_to_compute.get(choice - own).cloned() }) .collect(); - let pruned = |of_fused_computation| { - move |way_to_compute| PrunedWayToCompute { - target: index, - of_fused_computation, - way_to_compute, - reason: PruneReason::NoErrorBoundForQueryTarget, - } - }; - let removed = prune_own(&mut target.ways_to_compute, |way| own_pruned.contains(way)); - self.pruned_ways_to_compute - .extend(removed.into_iter().map(pruned(false))); + prune_own(&mut target.ways_to_compute, |way| own_pruned.contains(way)); if let Some(fused) = &mut target.fused_computation { - let (removed, kept): (Vec<_>, _) = std::mem::take(&mut fused.ways_to_compute) - .into_iter() - .partition(|way| fused_pruned.contains(way)); - fused.ways_to_compute = kept; - self.pruned_ways_to_compute - .extend(removed.into_iter().map(pruned(true))); + fused + .ways_to_compute + .retain(|way| !fused_pruned.contains(way)); } } self @@ -667,8 +547,6 @@ fn absorbed_targets(targets: &[ComputationTarget]) -> impl Iterator HavingPlacement { } } +/// Each memo group's expression as an operator-node tree, by group id: the +/// trees target discovery still walks. `Memo::logical_node_trees` is +/// deleted; discovery reads the groups in #1207 PR 3. +fn logical_node_trees(memo: &Memo) -> Vec> { + let _ = memo; + todo!("#1207 PR 3: discover targets over the memo groups, not node trees") +} + /// Discover one target per measure of each eligible aggregate memo group of /// `dag`, including operator plans read by scalar roots: Pass 1 walks the /// groups, so an aggregate several queries read is one target per measure. /// An aggregate measure is eligible when it has a realization besides exact /// execution, even if every such realization is pruned; the prune reasons /// are reported on the target. Measure filters and `HAVING` keep every -/// candidate; the target records how they are applied. A group's readers +/// candidate; the target records how they are applied. A group's consumers /// are its parent groups and the query roots that read it; a consumer -/// `Filter`'s key-only `HAVING` is pushed below only when every reader of +/// `Filter`'s key-only `HAVING` is pushed below only when every consumer of /// the group applies the same predicate, so every candidate of a shared /// aggregate computes the group's own rows (#1089). pub fn generate_per_computation_candidates( - dag: &DagOfLogicalMemoGroups, + memo: &Memo, ) -> Result { - let logical_nodes: Rc<[Rc]> = dag - .groups - .iter() - .map(|group| Rc::clone(&group.logical_node)) - .collect(); + let logical_nodes: Rc<[Rc]> = logical_node_trees(memo).into(); let group_of: HashMap<*const OperatorNode, MemoGroupId> = logical_nodes .iter() .enumerate() .map(|(g, node)| (Rc::as_ptr(node), MemoGroupId(g as u32))) .collect(); - let roots = dag.roots_over_groups(); + // Each query's root over the groups' logical nodes, in workload order: + // a group several queries read is one node. + let node = |g: &MemoGroupId| Rc::clone(&logical_nodes[g.0 as usize]); + let roots: Vec = memo + .query_roots + .iter() + .map(|root| match root { + QueryRoot::Operator(g) => QueryRoot::Operator(node(g)), + QueryRoot::Scalar(expr) => QueryRoot::Scalar(expr.map_operator_refs(&mut |g| node(g))), + }) + .collect(); let mut seen = HashSet::new(); // Each aggregate group with the first query that reaches it. let mut aggregates = Vec::new(); - // Every reader of each group's node: `Some(parent)`, or `None` for a + // Every consumer of each group's node: `Some(parent)`, or `None` for a // query root or scalar reference. let mut consumers: HashMap<*const OperatorNode, Vec>>> = HashMap::new(); for (query, root) in roots.iter().enumerate() { @@ -835,10 +719,6 @@ pub fn generate_per_computation_candidates( if !seen.insert(Rc::as_ptr(&node)) { continue; } - if node.timing.is_some() || node.materialization != Materialization::NotMaterialized - { - return Err(PerComputationCandidateError::AssignedPhysicalAnnotation); - } for child in node.children() { consumers .entry(Rc::as_ptr(child)) @@ -852,7 +732,6 @@ pub fn generate_per_computation_candidates( } } let mut targets = Vec::new(); - let mut pruned_ways_to_compute = Vec::new(); for (query, node) in aggregates { let Some(NonASAPOp::Aggregate { measures, @@ -888,7 +767,7 @@ pub fn generate_per_computation_candidates( } } for (filter, predicate) in &filters_above { - // Every reader of the group, across queries, applies it. + // Every consumer of the group, across queries, applies it. let shared_by_all = filters_above.len() == parents.len() && filters_above.iter().all(|(_, other)| other == predicate); entries.push(HavingEntry { @@ -914,25 +793,15 @@ pub fn generate_per_computation_candidates( continue; } if matches!(intent, AggIntent::Avg { .. }) && nullable_avg_input(&node, intent) { - choices.prune(PruneReason::NullableAvgInput, |choice| { + choices.prune(|choice| { matches!(choice, HowToComputeAggregateMeasure::ExactAccumulator(_)) }); } if matches!(intent, AggIntent::Count { .. }) && node.children()[0].result_kind == OperatorResultKind::Relation { - choices.prune(PruneReason::NoRelationalCountSketchRealization, |choice| { - matches!(choice, HowToComputeAggregateMeasure::Summary(_)) - }); + choices.prune(|choice| matches!(choice, HowToComputeAggregateMeasure::Summary(_))); } - pruned_ways_to_compute.extend(choices.pruned.into_iter().map( - |(way_to_compute, reason)| PrunedWayToCompute { - target: targets.len(), - of_fused_computation: false, - way_to_compute, - reason, - }, - )); targets.push(ComputationTarget { aggregate_group, measure_index: measure, @@ -944,18 +813,13 @@ pub fn generate_per_computation_candidates( } } add_whole_expression_targets(&mut targets, &logical_nodes); - prune_topk_heaps_over_one_row_per_item( - &mut targets, - &logical_nodes, - &mut pruned_ways_to_compute, - ); + prune_topk_heaps_over_one_row_per_item(&mut targets, &logical_nodes); add_nested_sum_targets(&mut targets, &logical_nodes); close_absorptions(&mut targets); Ok(PerComputationCandidates { roots, logical_nodes, targets, - pruned_ways_to_compute, }) } @@ -1004,7 +868,7 @@ fn close_absorptions(targets: &mut [ComputationTarget]) { /// sum of the inner groups' sums is the sum of their rows), absorbing the /// inner target. Offered whatever else reads the inner sum: it is one more /// candidate of the outer sum's group, and the inner sum's group keeps its -/// own candidates for its other readers. Applied when neither has a filter +/// own candidates for its other consumers. Applied when neither has a filter /// or `HAVING` /// ([`nested_sum_input`]), and when the outer sum is not itself absorbed by /// a merge and has no fused computation yet, and the inner sum has none, so @@ -1130,7 +994,7 @@ fn nested_sum_input(target: &OperatorNode) -> Option<(Rc, AggInten /// counts of raw data samples, the heap reads those samples and absorbs the /// per-series target too. Offered whatever else reads the inner aggregate: /// it is one more candidate of the top-k's group, and the absorbed groups -/// keep their own candidates for their other readers. Applied when neither +/// keep their own candidates for their other consumers. Applied when neither /// has a filter or `HAVING`, and when the keyed-additive rule realizes it /// ([`whole_expression_input`]). fn add_whole_expression_targets( @@ -1199,15 +1063,14 @@ fn add_whole_expression_targets( } } -/// Prunes the own heap candidates of each PromQL top-k -/// ([`PruneReason::TopKHeapOverOneRowPerItem`]); a fused whole-expression +/// Prunes the own heap candidates of each PromQL top-k: the heap reads +/// one row per item, so exact is no costlier; a fused whole-expression /// heap ([`add_whole_expression_targets`]) has already moved out of them. fn prune_topk_heaps_over_one_row_per_item( targets: &mut [ComputationTarget], logical_nodes: &[Rc], - pruned_ways_to_compute: &mut Vec, ) { - for (index, target) in targets.iter_mut().enumerate() { + for target in targets.iter_mut() { let Some(NonASAPOp::Aggregate { measures, child, .. }) = logical_nodes[target.aggregate_group.0 as usize].non_asap() @@ -1225,18 +1088,10 @@ fn prune_topk_heaps_over_one_row_per_item( if !reads_only_time_series(child) { continue; } - let pruned = prune_own(&mut target.ways_to_compute, |candidate| { + prune_own(&mut target.ways_to_compute, |candidate| { matches!(candidate, HowToComputeAggregateMeasure::Summary(family) if family.capability().answers(StatisticKind::TopK)) }); - pruned_ways_to_compute.extend(pruned.into_iter().map(|way_to_compute| { - PrunedWayToCompute { - target: index, - of_fused_computation: false, - way_to_compute, - reason: PruneReason::TopKHeapOverOneRowPerItem, - } - })); } } @@ -1279,14 +1134,7 @@ fn whole_expression_input( // update serves both heap families; plan selection decides whether // Count-Min is admissible, as for every other Count-Min candidate. let family = FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::CountSketchWithHeap, - asap_types::ir::schema::SketchParams::CountSketchWithHeap { - width: 1, - depth: 1, - heap_size: 1, - }, - ), + SketchAlgorithm::CountSketchWithHeap, GroupingStrategy::default(), ); let realized = realize_keyed_additive_summary_input(intent, &family, reduction, child)?; @@ -1342,9 +1190,8 @@ fn with_raw_data_samples_requirement( /// accumulators divided, a fused computation by one state over the absorbed /// target's input, and any other measure by its own single-measure /// aggregate; several measures are joined back on the group keys. A summary -/// is sized here for its measure's accuracy requirement; Pass 1 leaves it -/// unsized; several approximate inputs of one query share its budget -/// equally (`accuracy_budget_parts_per_measure`). A measure filter becomes `SummaryAgg.filter`; +/// is sized here for its measure's accuracy requirement. A measure filter +/// becomes `SummaryAgg.filter`; /// `HAVING` is applied as its target records. A sub-DAG shared between roots /// stays shared. Every node carries the guarantee `asap_accuracy::propagation::composed_guarantee` /// gives it. @@ -1353,38 +1200,15 @@ pub fn compose_logical_candidate( inventory: &PerComputationCandidates, choice: &[usize], ) -> Result, PerComputationCandidateError> { - compose_with_accuracy_budget_parts(inventory, &inventory.roots, choice, None) -} - -/// [`compose_logical_candidate`] with every sketch summary sized for one of -/// `accuracy_budget_parts` equal parts of its measure's budget, whatever the -/// roots compose: the size a summary gets when it is one of -/// `accuracy_budget_parts` approximate inputs of a query -/// ([`accuracy_budget_parts_per_measure`]). `accuracy_budget_parts` 1 sizes -/// for the whole budget. The sub-expression memo composes one aggregate -/// alone this way, once per number of parts its readers may give it. -pub(crate) fn compose_logical_candidate_with_accuracy_budget_parts( - inventory: &PerComputationCandidates, - choice: &[usize], - accuracy_budget_parts: usize, -) -> Result, PerComputationCandidateError> { - compose_with_accuracy_budget_parts( - inventory, - &inventory.roots, - choice, - Some(accuracy_budget_parts), - ) + compose(inventory, &inventory.roots, choice) } /// [`compose_logical_candidate`] of `roots` (some of `inventory.roots`), -/// each summary sized for one of `accuracy_budget_parts` equal parts of its -/// budget, or as -/// [`accuracy_budget_parts_per_measure`] splits it when `None`. -fn compose_with_accuracy_budget_parts( +/// each summary sized for its measure's whole budget. +fn compose( inventory: &PerComputationCandidates, roots: &[QueryRoot], choice: &[usize], - accuracy_budget_parts: Option, ) -> Result, PerComputationCandidateError> { if choice.len() != inventory.targets.len() { return Err(PerComputationCandidateError::InvalidChoice); @@ -1412,29 +1236,7 @@ fn compose_with_accuracy_budget_parts( .filter(|entry| entry.placement == HavingPlacement::BelowAggregate) .filter_map(|entry| entry.filter.as_ref().map(Rc::as_ptr)) .collect(); - // Each summary is first sized for its measure's whole budget. A query - // whose value composes several approximate inputs is then composed - // again with each sized for an equal share ([`accuracy_budget_parts_per_measure`]). - let whole = Sizing::default(); - let composed = compose_roots(roots, &chosen, &pushed_below, &whole)?; - let shares = match accuracy_budget_parts { - None => accuracy_budget_parts_per_measure(&composed, &whole.built.into_inner()), - Some(share) if share > 1 => whole - .built - .into_inner() - .into_iter() - .map(|(_, key)| (key, share)) - .collect(), - Some(_) => HashMap::new(), - }; - if shares.is_empty() { - return Ok(composed); - } - let shared = Sizing { - shares, - ..Sizing::default() - }; - compose_roots(roots, &chosen, &pushed_below, &shared) + compose_roots(roots, &chosen, &pushed_below, &Sizing::default()) } /// One measure of an aggregate: the aggregate and the measure's index. @@ -1443,44 +1245,10 @@ type MeasureKey = (*const OperatorNode, usize); /// How [`compose_logical_candidate`] sizes summaries in one composition. #[derive(Default)] struct Sizing { - /// How many approximate inputs each measure's summary shares its - /// query's budget with; a measure not listed has the whole budget. - shares: HashMap, /// Every sketch summary state built, with the measure it sizes. built: std::cell::RefCell, MeasureKey)>>, } -/// The equal budget split among the approximate inputs of each query -/// (#669b): a query whose value carries a guarantee that is not exact -/// composes every sketch summary state below its root -/// ([`approximate_inputs`]), so with `n ≥ 2` of them each is sized for one -/// of `n` equal shares of the query's target. A state several queries read -/// takes the largest `n`. Only states `built` lists (by the measure they -/// size) are split. Empty when no query composes two approximate inputs. -fn accuracy_budget_parts_per_measure( - roots: &[QueryRoot], - built: &[(Rc, MeasureKey)], -) -> HashMap { - let measure: HashMap<_, _> = built - .iter() - .map(|(state, key)| (Rc::as_ptr(state), *key)) - .collect(); - let mut shares = HashMap::new(); - for root in roots { - let inputs = approximate_inputs(root); - if inputs.len() < 2 { - continue; - } - for state in &inputs { - if let Some(key) = measure.get(&Rc::as_ptr(state)) { - let share = shares.entry(*key).or_insert(1); - *share = inputs.len().max(*share); - } - } - } - shares -} - /// Every root of `roots` with the chosen candidates substituted, each /// summary sized as `sizing` says, and every node with its guarantee. fn compose_roots( @@ -1762,10 +1530,7 @@ fn whole_expression( let state = OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryAgg { child: cache[&Rc::as_ptr(&input)].clone(), input: with_raw_data_samples_requirement(update, raw_data_samples_of), - family: FieldDataType::Sketch( - sized(algorithm, intent, sizing.share(key)), - GroupingStrategy::default(), - ), + family: FieldDataType::Sketch(*algorithm, GroupingStrategy::default()), reduction: partitions, grouping: GroupingStrategy::default(), filter: None, @@ -1785,7 +1550,7 @@ fn whole_expression( /// `topk_k` the estimated Float64 weight, then the item columns. This is /// the exact top-k's physical shape (its ranked rows' value, then the /// columns naming each ranked item: PromQL labels, SQL group keys), so -/// readers of `[keys.., topk_k]` find those columns where they were. +/// consumers of `[keys.., topk_k]` find those columns where they were. fn ranked_as_topk( target: &OperatorNode, estimate: Rc, @@ -1929,10 +1694,7 @@ fn substitute_measure( finalized(&kinds[0], intent, read)? } HowToComputeAggregateMeasure::Summary(algorithm) => { - let family = FieldDataType::Sketch( - sized(algorithm, intent, sizing.share(key)), - GroupingStrategy::default(), - ); + let family = FieldDataType::Sketch(*algorithm, GroupingStrategy::default()); let summary_input = state( family, intent, @@ -2212,20 +1974,12 @@ fn key_items(keys: usize) -> Vec { .collect() } -impl Sizing { - /// How many approximate inputs `key`'s summary shares its query's - /// budget with. - fn share(&self, key: MeasureKey) -> usize { - self.shares.get(&key).copied().unwrap_or(1) - } -} - /// What a measure's state reads from each row of `input`, resolvable there. /// A SQL `count` reads [`counted_column`]; any other measure, a PromQL count /// included, reads its input column (the sample value when it names none). fn measure_input( intent: &AggIntent, - input: &OperatorNode, + input: &Rc, filter: Option<&Predicate>, ) -> Result { let schema = &input.schema; @@ -2313,7 +2067,7 @@ fn summary_update( weight: SummaryInputExpr, ) -> Result { let algorithm = match family { - FieldDataType::Sketch(kind, _) => Some(kind.algorithm()), + FieldDataType::Sketch(kind, _) => Some(kind), _ => None, }; Ok(match (intent, algorithm) { @@ -2432,7 +2186,7 @@ mod tests { AccuracyGuarantee, AccuracyMetric, BoundExpr, ProbabilityExpr, }; use asap_types::ir::properties::AccuracyTarget; - use asap_types::ir::schema::{SketchParams, SketchStatistic}; + use asap_types::ir::schema::SketchStatistic; /// Approximate count keeps one exact option, its accumulator, beside the /// universal summary; plain exact execution is not offered. #[test] @@ -3143,9 +2897,9 @@ mod tests { // as a column; a query over aggregated rows is planned over labels. let root = asap_types::ir::schema::schema_support::with_promql_series_identity(&root) .unwrap_or(root); - generate_per_computation_candidates(&crate::stage0::generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(root), - ])) + generate_per_computation_candidates( + &crate::stage0::memo_of_queries(&[QueryRoot::Operator(root)]).unwrap(), + ) .unwrap() } @@ -3175,7 +2929,7 @@ mod tests { .filter(|node| { matches!(node.asap(), Some(ASAPOp::SummaryAgg { family: FieldDataType::Sketch(kind, _), .. - }) if kind.algorithm().capability().answers(StatisticKind::TopK)) + }) if kind.capability().answers(StatisticKind::TopK)) }) .collect() } @@ -3260,7 +3014,7 @@ mod tests { AccuracyTarget::max_error(0.1), ); let inventory = generate_per_computation_candidates( - &crate::stage0::generate_dag_of_logical_memo_groups(&[QueryRoot::Operator(root)]), + &crate::stage0::memo_of_queries(&[QueryRoot::Operator(root)]).unwrap(), ) .unwrap(); let topk = topk_target(&inventory); @@ -3895,89 +3649,6 @@ mod tests { assert_eq!(sign_statistic(&stats), None); } - /// The sketch parameters of every summary state below `root`. - fn sketch_params(root: &Rc) -> Vec { - OperatorNode::reachable(root) - .into_iter() - .filter_map(|node| match node.asap() { - Some(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), - .. - }) => Some(kind.params().clone()), - _ => None, - }) - .collect() - } - - /// `query` composed with target `t` at the candidate `pick(t, its - /// candidates)` returns the position of. - fn composed_picking( - query: &str, - pick: impl Fn(usize, &[HowToComputeAggregateMeasure]) -> usize, - ) -> Rc { - let inventory = inventory(query); - let choice: Vec = inventory - .targets - .iter() - .enumerate() - .map(|(t, target)| pick(t, &target.ways_to_compute)) - .collect(); - let composed = compose_logical_candidate(&inventory, &choice).unwrap(); - let QueryRoot::Operator(root) = &composed[0] else { - panic!("operator root") - }; - root.clone() - } - - fn position_of( - candidates: &[HowToComputeAggregateMeasure], - candidate: HowToComputeAggregateMeasure, - ) -> usize { - candidates - .iter() - .position(|c| *c == candidate) - .expect("the candidate is offered") - } - - fn cms(eps: f64, delta: f64) -> SketchParams { - SketchParams::Cms { - width: asap_accuracy::test_support::cms_width(eps), - depth: asap_accuracy::test_support::cms_depth(delta), - } - } - - /// A combination of approximate values splits the query's budget - /// equally among them (#669b): the difference of two Count-Min counts - /// sizes each for ε/2 and δ/2 of the query's ε = 0.01, δ = 0.001. - #[test] - fn approximate_inputs_of_a_combination_share_the_budget_equally() { - let both = composed_picking("count by (job) (a) - count by (job) (b)", |_, c| { - position_of( - c, - HowToComputeAggregateMeasure::Summary(SketchAlgorithm::Cms), - ) - }); - assert!(both.guarantee.as_ref().is_some_and(|g| !g.is_exact())); - assert_eq!(sketch_params(&both), vec![cms(0.005, 0.0005); 2]); - } - - /// Only the inputs that are approximate in the combination share the - /// budget: with one count exact, the other gets all of it. - #[test] - fn an_exact_input_takes_no_share_of_the_budget() { - let one = composed_picking("count by (job) (a) - count by (job) (b)", |t, c| { - if t == 0 { - position_of( - c, - HowToComputeAggregateMeasure::Summary(SketchAlgorithm::Cms), - ) - } else { - 0 - } - }); - assert_eq!(sketch_params(&one), vec![cms(0.01, 0.001)]); - } - /// The first composed candidate of `query`. fn inventory_root(query: &str) -> Rc { let inventory = inventory(query); @@ -3993,13 +3664,7 @@ mod tests { /// non-negative: an open requirement (`RawDataSamplesOfMetric`) is not a proof. #[test] fn count_min_estimate_has_no_bound_over_an_open_requirement() { - let family = FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Cms, - asap_types::ir::schema::SketchParams::Cms { width: 8, depth: 2 }, - ), - GroupingStrategy::default(), - ); + let family = FieldDataType::Sketch(SketchAlgorithm::Cms, GroupingStrategy::default()); let query = SketchStatistic::PointCount { key: ColumnRef::SampleValue, value: None, @@ -4033,16 +3698,7 @@ mod tests { /// once bound to 1. #[test] fn count_min_estimate_over_raw_data_samples_is_gated_on_the_metric() { - let family = FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Cms, - asap_types::ir::schema::SketchParams::Cms { - width: 272, - depth: 5, - }, - ), - GroupingStrategy::default(), - ); + let family = FieldDataType::Sketch(SketchAlgorithm::Cms, GroupingStrategy::default()); let query = SketchStatistic::PointCount { key: ColumnRef::SampleValue, value: None, @@ -4076,19 +3732,8 @@ mod tests { /// every group, whose distinct items the source's bound does not bound. #[test] fn classic_hll_confidence_needs_one_instance_over_a_sources_own_rows() { - use asap_types::ir::schema::{HydraKind, HydraParams}; - let hll = |grouping| { - FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Hll, - SketchParams::Hll { - precision: 15, - relative_error: Some(0.05), - }, - ), - grouping, - ) - }; + use asap_types::ir::schema::HydraKind; + let hll = |grouping| FieldDataType::Sketch(SketchAlgorithm::Hll, grouping); let input = |query| crate::test_support::lower_promql(query, AccuracyTarget::exact()); let guarantee = |family: &FieldDataType, query| { source_distinct_guarantee(family, &SketchStatistic::Cardinality, &input(query)) @@ -4106,260 +3751,10 @@ mod tests { } let hydra = hll(GroupingStrategy::SharedMultiSubpopulation { kind: HydraKind::HydraCms, - params: HydraParams::HydraCms { - width: 8, - depth: 2, - shared_rows: 2, - shared_columns: 8, - }, }); assert!(guarantee(&hydra, "a").is_none()); } - /// A deployment whose executor supports only `supported`: each family - /// with its readouts. - fn supporting(supported: &[(SketchAlgorithm, &[StatisticKind])]) -> Vec { - supported - .iter() - .map(|(family, readouts)| SupportedSummary { - family: *family, - readouts: readouts.to_vec(), - }) - .collect() - } - - /// A family the deployment cannot build is pruned from Pass 1 with - /// `UnsupportedByDeployment`; the supported family and the exact option - /// stay. With every summary supported the inventory is unchanged. - #[test] - fn a_family_the_deployment_cannot_build_is_pruned() { - let query = "quantile_over_time(0.99, m[5m])"; - let unrestricted = inventory(query); - let restricted = inventory(query).without_unsupported_summaries(Some(&supporting(&[( - SketchAlgorithm::Kll, - &[StatisticKind::Quantile], - )]))); - let target = &restricted.targets[0]; - assert_eq!( - target.ways_to_compute, - [ - HowToComputeAggregateMeasure::Exact, - HowToComputeAggregateMeasure::Summary(SketchAlgorithm::Kll) - ] - ); - assert_eq!( - restricted.pruned_of_target(0, false), - [( - HowToComputeAggregateMeasure::Summary(SketchAlgorithm::DDSketch), - PruneReason::UnsupportedByDeployment - )] - ); - let default = inventory(query).without_unsupported_summaries(None); - assert_eq!( - default.targets[0].ways_to_compute, - unrestricted.targets[0].ways_to_compute - ); - } - - /// A family the deployment builds but cannot answer the measure's - /// readout from is pruned: UnivMon, supported for distinct counts and - /// L2 only, is no count candidate; Count-Min, supported for the total, - /// stays. - #[test] - fn a_readout_the_deployment_cannot_answer_prunes_its_family() { - let restricted = inventory("count_over_time(m[1m])").without_unsupported_summaries(Some( - &supporting(&[ - (SketchAlgorithm::Cms, &[StatisticKind::Total]), - ( - SketchAlgorithm::UnivMon, - &[StatisticKind::Distinct, StatisticKind::FrequencyL2], - ), - ]), - )); - let target = &restricted.targets[0]; - assert_eq!( - target.ways_to_compute, - [ - HowToComputeAggregateMeasure::ExactAccumulator(vec![ExactKind::Count]), - HowToComputeAggregateMeasure::Summary(SketchAlgorithm::Cms), - ] - ); - assert_eq!( - restricted.pruned_of_target(0, false), - [( - HowToComputeAggregateMeasure::Summary(SketchAlgorithm::UnivMon), - PruneReason::UnsupportedByDeployment - )] - ); - } - - /// A fused whole-expression top-k loses the heap families the - /// deployment cannot answer top-k from, each recorded with its reason, - /// and is not offered once it has none left: the target and the inner - /// sum keep their exact options. - #[test] - fn a_fused_computation_loses_its_unsupported_heaps() { - let query = "topk by (job) (10, sum_over_time(m[1m]))"; - let one_heap = inventory(query).without_unsupported_summaries(Some(&supporting(&[( - SketchAlgorithm::CountSketchWithHeap, - &[StatisticKind::TopK], - )]))); - let topk = topk_target(&one_heap); - assert_eq!( - one_heap.targets[topk] - .fused_computation - .as_ref() - .unwrap() - .ways_to_compute, - [HowToComputeAggregateMeasure::Summary( - SketchAlgorithm::CountSketchWithHeap - )] - ); - assert_eq!( - one_heap.pruned_of_target(topk, true), - [( - HowToComputeAggregateMeasure::Summary(SketchAlgorithm::CmsWithHeap), - PruneReason::UnsupportedByDeployment - )] - ); - assert_eq!(one_heap.combinations(), 2); - // Count-Min with a heap answers item frequencies, not top-k. - let none = inventory(query).without_unsupported_summaries(Some(&supporting(&[( - SketchAlgorithm::CmsWithHeap, - &[StatisticKind::ItemFrequency], - )]))); - let topk = topk_target(&none); - let fused = none.targets[topk].fused_computation.as_ref().unwrap(); - assert!(fused.ways_to_compute.is_empty()); - assert_eq!( - none.pruned_of_target(topk, true), - [ - ( - HowToComputeAggregateMeasure::Summary(SketchAlgorithm::CmsWithHeap), - PruneReason::UnsupportedByDeployment - ), - ( - HowToComputeAggregateMeasure::Summary(SketchAlgorithm::CountSketchWithHeap), - PruneReason::UnsupportedByDeployment - ), - ] - ); - assert_eq!( - none.targets[topk].ways_to_compute, - [HowToComputeAggregateMeasure::Exact] - ); - assert_eq!(none.combinations(), 1); - } - - /// Mergeable intents give exactly their accumulator (`avg`: sum, count); other exact-only intents give `Exact`. - #[test] - fn intent_candidate_matrix() { - use asap_types::ir::operator::agg_intent::{MathFunc, TimeFunc}; - use AggIntent as A; - use ExactKind as E; - let accumulator = |kinds: Vec| HowToComputeAggregateMeasure::ExactAccumulator(kinds); - let exact_count = A::Count { - accuracy: AccuracyTarget::exact(), - }; - let mut matrix: Vec<(A, ComputationChoices)> = [ - (A::Min { col: None }, vec![E::Min]), - (A::Max { col: None }, vec![E::Max]), - (A::IRate, vec![E::IRate]), - (A::Increase, vec![E::Increase]), - (A::Sum { col: None }, vec![E::Sum]), - (A::Rate, vec![E::Rate]), - (A::Avg { col: None }, vec![E::Sum, E::Count]), - ] - .into_iter() - .map(|(intent, kinds)| { - ( - intent, - ComputationChoices { - candidates: vec![accumulator(kinds)], - pruned: vec![], - }, - ) - }) - .collect(); - // An exact count prunes its summaries for requiring an exact answer. - matrix.push(( - exact_count.clone(), - ComputationChoices { - candidates: vec![accumulator(vec![E::Count])], - pruned: summary_candidates(&exact_count) - .iter() - .map(|a| { - ( - HowToComputeAggregateMeasure::Summary(*a), - PruneReason::RequiresExact, - ) - }) - .collect(), - }, - )); - let exact_only = [ - A::PearsonCorr { left: 0, right: 1 }, - A::Extension { - ext_kind: "frequency".into(), - payload: serde_json::json!({ "item": "checkout" }), - }, - A::HistogramQuantile { q: 0.99, le: 0 }, - A::Changes, - A::Delta, - A::IDelta, - A::Deriv, - A::Resets, - A::PredictLinear { seconds: 60.0 }, - A::DoubleExpSmoothing { - smoothing: 0.5, - trend: 0.5, - }, - A::HistogramCount, - A::HistogramSum, - A::HistogramAvg, - A::HistogramStdDev, - A::HistogramStdVar, - A::HistogramFraction { - lower: 0.0, - upper: 1.0, - }, - A::Math(MathFunc::Abs), - A::TimeFn(TimeFunc::Hour), - A::Absent { - labels: Default::default(), - }, - A::AbsentOverTime, - A::PresentOverTime, - A::Group, - A::CountValues { label: "v".into() }, - A::LastOverTime, - A::FirstOverTime, - A::MadOverTime, - A::TsOfMinOverTime, - A::TsOfMaxOverTime, - A::TsOfFirstOverTime, - A::TsOfLastOverTime, - ]; - matrix.extend(exact_only.into_iter().map(|intent| { - ( - intent, - ComputationChoices { - candidates: vec![HowToComputeAggregateMeasure::Exact], - pruned: vec![], - }, - ) - })); - for (intent, expected) in &matrix { - assert_eq!( - &computation_candidates_for_intent(intent).unwrap(), - expected, - "{intent:?}" - ); - } - // Correlation never gets a single-input summary. - assert!(summary_candidates(&A::PearsonCorr { left: 0, right: 1 }).is_empty()); - } - /// `avg_over_time` divides with a finite-division guard; a grouped `avg` does not. #[test] fn temporal_average_divides_with_a_finite_division_guard() { @@ -4390,9 +3785,9 @@ mod tests { /// (closed) rows a grouped top-k gets no whole-expression heap. fn inventory_at(query: &str, accuracy: AccuracyTarget) -> PerComputationCandidates { let root = crate::test_support::lower_promql(query, accuracy); - generate_per_computation_candidates(&crate::stage0::generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(root), - ])) + generate_per_computation_candidates( + &crate::stage0::memo_of_queries(&[QueryRoot::Operator(root)]).unwrap(), + ) .unwrap() } @@ -4581,342 +3976,4 @@ mod tests { Some(SummaryInputExpr::Tuple(_)) )); } - - /// A KLL quantile over an exact sum accumulator composes per node, the sum adding no error. - #[test] - fn nested_quantile_over_an_exact_sum_composes_per_node() { - let sum = HowToComputeAggregateMeasure::ExactAccumulator(vec![ExactKind::Sum]); - let kll = HowToComputeAggregateMeasure::Summary(SketchAlgorithm::Kll); - let inventory = inventory_at( - "quantile(0.9, sum by (job) (m))", - AccuracyTarget::max_error(0.01).failure_probability(0.01), - ); - let choice: Vec = inventory - .targets - .iter() - .map(|target| { - let candidates = &target.ways_to_compute; - if candidates.contains(&kll) { - position_of(candidates, kll.clone()) - } else { - position_of(candidates, sum.clone()) - } - }) - .collect(); - let composed = compose_logical_candidate(&inventory, &choice).unwrap(); - let QueryRoot::Operator(root) = &composed[0] else { - panic!("operator root") - }; - let estimate = OperatorNode::reachable(root) - .into_iter() - .find(|node| matches!(node.asap(), Some(ASAPOp::SummaryEstimate { .. }))) - .expect("a quantile estimate"); - let Some(ASAPOp::SummaryEstimate { summary_input, .. }) = estimate.asap() else { - unreachable!() - }; - let Some(ASAPOp::SummaryAgg { child, family, .. }) = summary_input.asap() else { - panic!("a KLL state: {summary_input:#?}") - }; - let FieldDataType::Sketch(kind, _) = family else { - panic!("a sketch: {family:?}") - }; - assert_eq!(kind.algorithm(), &SketchAlgorithm::Kll); - assert_eq!( - kind.params(), - &SketchParams::Kll { - k: 269, - failure_probability: None - } - ); - let Some(ASAPOp::FinalizeExactAccumulator { child: sum_state }) = child.asap() else { - panic!("the finalized sum: {child:#?}") - }; - let Some(ASAPOp::SummaryAgg { - family: sum_family, - child: leaf, - .. - }) = sum_state.asap() - else { - panic!("the sum's accumulator: {sum_state:#?}") - }; - assert_eq!( - sum_family, - &FieldDataType::ExactAggregate( - ExactKind::Sum, - asap_types::ir::schema::ExactParams::Sum - ) - ); - assert!(!leaf.contains_asap()); - assert!(child - .guarantee - .as_ref() - .is_some_and(AccuracyGuarantee::is_exact)); - let guarantee = root.guarantee.as_ref().expect("a rank bound"); - assert_eq!(guarantee.spec.accuracy_metric, AccuracyMetric::Rank); - assert_eq!( - guarantee.spec.error.evaluate(), - Some(asap_accuracy::test_support::kll_rank_error_99(269)) - ); - } - - /// Pass 1's inventory of `queries`, each lowered with its target, less - /// the summaries no query can use over `data`. - fn usable_inventory( - queries: &[(&str, AccuracyTarget)], - data: &DataWorkload, - ) -> PerComputationCandidates { - let roots: Vec = queries - .iter() - .map(|(query, target)| { - let root = crate::test_support::lower_promql(query, target.clone()); - QueryRoot::Operator(root) - }) - .collect(); - let targets: Vec = - queries.iter().map(|(_, target)| target.clone()).collect(); - generate_per_computation_candidates(&crate::stage0::generate_dag_of_logical_memo_groups( - &roots, - )) - .unwrap() - .without_summaries_no_query_can_use( - &targets, - data, - &asap_accuracy::BuiltinAccuracyModel, - ) - } - - /// A per-value target at ε 0.05, δ 0.01 in `metric`. - fn target_in(metric: AccuracyMetric) -> AccuracyTarget { - AccuracyTarget::max_error(0.05) - .failure_probability(0.01) - .accuracy_metric(metric) - .accuracy_scope(asap_types::ir::properties::AccuracyScope::PerValue) - } - - /// Metric `x`, its samples declared non-negative or not. - fn x_data(non_negative: bool) -> DataWorkload { - let mut data = - DataWorkload::sampled_every(["x".to_string()], asap_types::workload::DurationMs(1_000)); - data.sources.get_mut("x").unwrap().values_non_negative = non_negative; - data - } - - /// The summary families pruned for `reason`, and those still offered. - fn pruned_and_offered( - inventory: &PerComputationCandidates, - reason: PruneReason, - ) -> (Vec, Vec) { - let summary = |way: &HowToComputeAggregateMeasure| match way { - HowToComputeAggregateMeasure::Summary(algorithm) => Some(*algorithm), - _ => None, - }; - let pruned = inventory - .pruned_ways_to_compute - .iter() - .filter(|pruned| pruned.reason == reason) - .filter_map(|pruned| summary(&pruned.way_to_compute)) - .collect(); - let offered = inventory - .targets - .iter() - .flat_map(|target| &target.ways_to_compute) - .filter_map(summary) - .collect(); - (pruned, offered) - } - - /// An average of per-series p99s at a relative target: a relative error - /// passes through `avg` only over samples declared non-negative, and a - /// rank error never meets a relative target. Over an undeclared gauge - /// every quantile sketch is pruned with - /// [`PruneReason::NoErrorBoundForQueryTarget`]; declared non-negative, - /// DDSketch is kept and KLL still pruned. At a rank target, a rank - /// error has no rule through `avg`, so KLL is pruned too. - #[test] - fn a_relative_bound_through_avg_is_pruned_without_a_sign_declaration() { - let query = "avg by (g) (quantile_over_time(0.99, x[100s]))"; - let reason = PruneReason::NoErrorBoundForQueryTarget; - let relative = target_in(AccuracyMetric::Relative); - let undeclared = usable_inventory(&[(query, relative.clone())], &x_data(false)); - let (pruned, offered) = pruned_and_offered(&undeclared, reason); - assert!(pruned.contains(&SketchAlgorithm::DDSketch), "{pruned:?}"); - assert!(pruned.contains(&SketchAlgorithm::Kll), "{pruned:?}"); - assert!(offered.is_empty(), "{offered:?}"); - let declared = usable_inventory(&[(query, relative)], &x_data(true)); - let (pruned, offered) = pruned_and_offered(&declared, reason); - assert_eq!(offered, vec![SketchAlgorithm::DDSketch], "{pruned:?}"); - assert!(pruned.contains(&SketchAlgorithm::Kll), "{pruned:?}"); - let rank = usable_inventory(&[(query, target_in(AccuracyMetric::Rank))], &x_data(true)); - let (pruned, offered) = pruned_and_offered(&rank, reason); - assert!(pruned.contains(&SketchAlgorithm::Kll), "{pruned:?}"); - assert!(offered.is_empty(), "{offered:?}"); - } - - /// A summary one of the queries reading its group can use is kept: the - /// per-series p99s are both a query's result, which a KLL rank bound - /// meets, and the input of an average, which no rank bound reaches. - #[test] - fn a_summary_one_of_two_queries_can_use_is_kept() { - let rank = target_in(AccuracyMetric::Rank); - let alone = usable_inventory( - &[("avg(quantile_over_time(0.99, x[5m]))", rank.clone())], - &x_data(false), - ); - let (pruned, _) = pruned_and_offered(&alone, PruneReason::NoErrorBoundForQueryTarget); - assert!(pruned.contains(&SketchAlgorithm::Kll), "{pruned:?}"); - let shared = usable_inventory( - &[ - ("quantile_over_time(0.99, x[5m])", rank.clone()), - ("avg(quantile_over_time(0.99, x[5m]))", rank), - ], - &x_data(false), - ); - let (pruned, offered) = - pruned_and_offered(&shared, PruneReason::NoErrorBoundForQueryTarget); - assert!(offered.contains(&SketchAlgorithm::Kll), "{offered:?}"); - assert!(!pruned.contains(&SketchAlgorithm::Kll), "{pruned:?}"); - } - - /// Exact candidates are never pruned: every target keeps its exact - /// option first, and only summaries are pruned for the target. - #[test] - fn exact_candidates_are_never_pruned_for_the_target() { - let rank = target_in(AccuracyMetric::Rank); - for query in [ - "avg by (g) (quantile_over_time(0.99, x[100s]))", - "avg(quantile_over_time(0.99, x[5m])) > 3", - "sum(count_over_time(x[5m])) / avg(quantile_over_time(0.5, x[5m]))", - ] { - let inventory = usable_inventory(&[(query, rank.clone())], &x_data(false)); - assert!(!inventory.targets.is_empty(), "{query}"); - for target in &inventory.targets { - assert!( - matches!( - target.ways_to_compute.first(), - Some( - HowToComputeAggregateMeasure::Exact - | HowToComputeAggregateMeasure::ExactAccumulator(_) - ) - ), - "{query}: {:?}", - target.ways_to_compute - ); - } - assert!( - inventory - .pruned_ways_to_compute - .iter() - .filter(|pruned| pruned.reason == PruneReason::NoErrorBoundForQueryTarget) - .all(|pruned| matches!( - pruned.way_to_compute, - HowToComputeAggregateMeasure::Summary(_) - )), - "{query}" - ); - } - } - - /// A summary another aggregate over the same source also offers is - /// kept, though no query reading its own group can use it: Pass 2 may - /// build the other group's state from it (a rollup or window rollup), - /// which Pass 1 does not see. - #[test] - fn a_summary_pass2_may_reuse_for_another_aggregate_is_kept() { - let rank = target_in(AccuracyMetric::Rank); - let reason = PruneReason::NoErrorBoundForQueryTarget; - let with_other = usable_inventory( - &[ - ("avg(quantile_over_time(0.99, x[5m]))", rank.clone()), - ("quantile_over_time(0.99, x[10m])", rank), - ], - &x_data(false), - ); - let (pruned, _) = pruned_and_offered(&with_other, reason); - assert!(!pruned.contains(&SketchAlgorithm::Kll), "{pruned:?}"); - } - - /// A whole-result target with a failure probability over a number of - /// values the data does not bound cannot be met by a per-value - /// guarantee that may fail, whatever its size: KLL is pruned. With the - /// series count declared, it is kept. DDSketch's relative bound never - /// fails, so it is not pruned for the scope (only for the metric under - /// a rank target). - #[test] - fn a_whole_result_target_over_unbound_values_prunes_a_failing_guarantee() { - let whole = target_in(AccuracyMetric::Rank) - .accuracy_scope(asap_types::ir::properties::AccuracyScope::WholeResult); - let query = "quantile_over_time(0.99, x[5m])"; - let reason = PruneReason::NoErrorBoundForQueryTarget; - let unbound = usable_inventory(&[(query, whole.clone())], &x_data(false)); - let (pruned, _) = pruned_and_offered(&unbound, reason); - assert!(pruned.contains(&SketchAlgorithm::Kll), "{pruned:?}"); - let bound = usable_inventory(&[(query, whole)], &x_data(false).with_series_count(10_000)); - let (pruned, offered) = pruned_and_offered(&bound, reason); - assert!(offered.contains(&SketchAlgorithm::Kll), "{pruned:?}"); - } - - /// The reuse escape keeps a summary only for another target whose own - /// queries can use the family: two averages of p99s over different - /// windows of one metric at a rank target both offer KLL, and neither - /// can use it (a rank error has no rule through `avg`), so it is - /// pruned for both. Before, each kept it for the other. - #[test] - fn a_summary_no_reusing_target_can_use_is_pruned() { - let rank = target_in(AccuracyMetric::Rank); - let inventory = usable_inventory( - &[ - ("avg(quantile_over_time(0.99, x[5m]))", rank.clone()), - ("avg(quantile_over_time(0.99, x[10m]))", rank), - ], - &x_data(false), - ); - let (pruned, offered) = - pruned_and_offered(&inventory, PruneReason::NoErrorBoundForQueryTarget); - assert_eq!( - pruned - .iter() - .filter(|&&family| family == SketchAlgorithm::Kll) - .count(), - 2, - "{pruned:?}" - ); - assert!(!offered.contains(&SketchAlgorithm::Kll), "{offered:?}"); - } - - /// Classic HLL is pruned when its source's declared distinct bound is - /// outside the contract at every precision (above 2^17 distinct items): - /// its failure probability is 1 at any size. Under the bound it is kept, - /// and with no declared bound too (plan selection may bind one). Two - /// queries over the same metric do not keep it for each other. - #[test] - fn hll_over_a_distinct_bound_outside_its_contract_is_pruned() { - let relative = target_in(AccuracyMetric::Relative); - let queries = [ - ("distinct_over_time(x[5m])", relative.clone()), - ("distinct_over_time(x[10m])", relative), - ]; - let with_bound = |bound: Option| { - let mut data = x_data(true); - data.sources - .get_mut("x") - .unwrap() - .max_distinct_per_evaluation = bound; - data - }; - let reason = PruneReason::NoErrorBoundForQueryTarget; - let (_, offered) = - pruned_and_offered(&usable_inventory(&queries, &with_bound(None)), reason); - assert!(offered.contains(&SketchAlgorithm::Hll), "{offered:?}"); - let (_, offered) = pruned_and_offered( - &usable_inventory(&queries, &with_bound(Some(1_000))), - reason, - ); - assert!(offered.contains(&SketchAlgorithm::Hll), "{offered:?}"); - let (pruned, offered) = pruned_and_offered( - &usable_inventory(&queries, &with_bound(Some(1_000_000))), - reason, - ); - assert!(pruned.contains(&SketchAlgorithm::Hll), "{pruned:?}"); - assert!(!offered.contains(&SketchAlgorithm::Hll), "{offered:?}"); - } } diff --git a/crates/logical-asap-optimization/src/pass1/per_computation_candidates_tests.rs b/crates/logical-asap-optimization/src/pass1/per_computation_candidates_tests.rs index 87189af51..bcc2a625d 100644 --- a/crates/logical-asap-optimization/src/pass1/per_computation_candidates_tests.rs +++ b/crates/logical-asap-optimization/src/pass1/per_computation_candidates_tests.rs @@ -3,19 +3,19 @@ use crate::pass1::per_computation_candidates::{ computation_candidates_for_intent, generate_per_computation_candidates, PerComputationCandidateError, }; -use crate::stage0::generate_dag_of_logical_memo_groups; +use crate::stage0::memo_of_queries; use asap_types::ir::operator::operator_properties::{Reduction, Source}; use asap_types::ir::operator::AggIntent; use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::scalar::{ColumnRef, CompareOpKind, ScalarValue}; use asap_types::ir::schema::{ DataType, ExactKind, Field, FieldDataType, GroupingStrategy, Schema, SketchAlgorithm, - SketchKind, SketchParams, SummaryUpdate, + SummaryUpdate, }; use asap_types::ir::{ ASAPOp, ExprSemantics, NonASAPOp, Operator, OperatorNode, Predicate, QueryRoot, ScalarExpr, }; -use asap_types::stages::PruneReason; + use asap_types::stages::{HavingEntry, HavingPlacement, HowToComputeAggregateMeasure}; use std::rc::Rc; @@ -117,41 +117,6 @@ fn frequency_statistics_keep_universal_choices() { } } -/// An exact request cannot acquire an approximate sketch merely because one is -/// available; each pruned family records why. -#[test] -fn exact_quantile_stays_exact_and_approximate_keeps_both_families() { - let choices = candidates_for(AggIntent::Quantile { - col: Some(0), - q: 0.99, - accuracy: approximate(), - }); - assert_eq!( - algorithms(&choices), - vec![SketchAlgorithm::Kll, SketchAlgorithm::DDSketch] - ); - let exact = computation_candidates_for_intent(&AggIntent::Quantile { - col: Some(0), - q: 0.99, - accuracy: AccuracyTarget::exact(), - }) - .unwrap(); - assert_eq!(exact.candidates, vec![HowToComputeAggregateMeasure::Exact]); - assert_eq!( - exact.pruned, - vec![ - ( - HowToComputeAggregateMeasure::Summary(SketchAlgorithm::Kll), - PruneReason::RequiresExact - ), - ( - HowToComputeAggregateMeasure::Summary(SketchAlgorithm::DDSketch), - PruneReason::RequiresExact - ), - ] - ); -} - /// Scalar roots expose their producer targets; repeated references retain one target identity. #[test] fn scalar_root_producers_are_discovered_once() { @@ -164,14 +129,14 @@ fn scalar_root_producers_are_discovered_once() { QueryRoot::Operator(producer.clone()), ]; let candidates = - generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&roots)).unwrap(); + generate_per_computation_candidates(&memo_of_queries(&roots).unwrap()).unwrap(); assert_eq!(candidates.roots.len(), 2); for root in &candidates.roots { root.validate_structure().unwrap(); } assert_eq!(candidates.targets.len(), 1); assert_eq!(**candidates.aggregate(&candidates.targets[0]), *producer); - assert!(producer.timing.is_none()); + assert!(producer.guarantee.is_none()); producer.validate_structure().unwrap(); } @@ -212,7 +177,7 @@ fn exact_promql(query: &str) -> Vec { fn example_1_exact_q1_keeps_exact_accumulators_only() { let roots = exact_promql("sum by (job) (rate(http_requests_total[1m]))"); let candidates = - generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&roots)).unwrap(); + generate_per_computation_candidates(&memo_of_queries(&roots).unwrap()).unwrap(); let found: Vec<_> = candidates .targets .iter() @@ -229,61 +194,15 @@ fn example_1_exact_q1_keeps_exact_accumulators_only() { ])], ] ); - assert!(candidates - .targets - .iter() - .all(|target| candidates.aggregate(target).timing.is_none())); } -/// Physical annotations, ASAP operators and invalid probability requirements -/// fail at the stage boundary. +/// ASAP operators and invalid probability requirements fail at the stage +/// boundary. #[test] -fn assigned_physical_annotation_asap_input_and_invalid_accuracy_are_rejected() { - let mut producer = (*aggregate(AggIntent::Count { - accuracy: approximate(), - })) - .clone(); - producer.timing = Some(asap_types::ir::properties::ExecutionTiming::QueryTime); - assert!(matches!( - generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(Rc::new(producer.clone())) - ])), - Err(PerComputationCandidateError::AssignedPhysicalAnnotation) - )); - let mut stored = (*aggregate(AggIntent::Count { - accuracy: approximate(), - })) - .clone(); - stored.materialization = - asap_types::physical::Materialization::MatAtQuery(asap_types::physical::Medium::Memory); - assert!(matches!( - generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(Rc::new(stored.clone())) - ])), - Err(PerComputationCandidateError::AssignedPhysicalAnnotation) - )); - // An annotated node below an unannotated root is rejected too. - for annotated in [producer, stored] { - let root = having_filter(Rc::new(annotated), always()); - assert!(matches!( - generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(root) - ])), - Err(PerComputationCandidateError::AssignedPhysicalAnnotation) - )); - } +fn asap_input_and_invalid_accuracy_are_rejected() { let summary = OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryAgg { child: scan(false), - family: FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: 200, - failure_probability: None, - }, - ), - Default::default(), - ), + family: FieldDataType::Sketch(SketchAlgorithm::Kll, Default::default()), input: SummaryUpdate::column(ColumnRef::SampleValue), reduction: Reduction::by(vec![]), grouping: GroupingStrategy::default(), @@ -291,9 +210,9 @@ fn assigned_physical_annotation_asap_input_and_invalid_accuracy_are_rejected() { })) .unwrap(); assert!(matches!( - generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(summary) - ])), + generate_per_computation_candidates( + &memo_of_queries(&[QueryRoot::Operator(summary)]).unwrap() + ), Err(PerComputationCandidateError::ContainsAsap) )); let epsilon_delta = @@ -333,9 +252,9 @@ fn assigned_physical_annotation_asap_input_and_invalid_accuracy_are_rejected() { vec![], None, ); - let error = generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(invalid), - ])) + let error = generate_per_computation_candidates( + &memo_of_queries(&[QueryRoot::Operator(invalid)]).unwrap(), + ) .unwrap_err(); assert!( matches!( @@ -394,9 +313,9 @@ fn multi_measure_aggregate_has_one_target_per_measure() { vec![], None, ); - let inventory = generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(node), - ])) + let inventory = generate_per_computation_candidates( + &memo_of_queries(&[QueryRoot::Operator(node)]).unwrap(), + ) .unwrap(); let candidates = &inventory.targets; assert_eq!( @@ -422,14 +341,12 @@ fn aggregate_without_realization_is_not_a_target() { col: Some(1), population: true, }); - assert!( - generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(node) - ])) - .unwrap() - .targets - .is_empty() - ); + assert!(generate_per_computation_candidates( + &memo_of_queries(&[QueryRoot::Operator(node)]).unwrap() + ) + .unwrap() + .targets + .is_empty()); } /// A filtered SQL count keeps its executable accumulator and records its @@ -445,9 +362,9 @@ fn filtered_measure_keeps_candidates_and_records_its_filter() { vec![Some(always()), None], None, ); - let inventory = generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(filtered), - ])) + let inventory = generate_per_computation_candidates( + &memo_of_queries(&[QueryRoot::Operator(filtered)]).unwrap(), + ) .unwrap(); let targets = &inventory.targets; assert_eq!(targets.len(), 2); @@ -459,36 +376,21 @@ fn filtered_measure_keeps_candidates_and_records_its_filter() { assert_eq!(targets[1].measure_filter, None); } -/// Target `target`'s own ways to compute and the ones Pass 1 pruned. +/// Target `target`'s own ways to compute. fn choices_of( inventory: &crate::pass1::per_computation_candidates::PerComputationCandidates, target: usize, -) -> crate::pass1::per_computation_candidates::ComputationChoices { - crate::pass1::per_computation_candidates::ComputationChoices { - candidates: inventory.targets[target].ways_to_compute.clone(), - pruned: inventory.pruned_of_target(target, false), - } +) -> Vec { + inventory.targets[target].ways_to_compute.clone() } -fn assert_sql_count_choices( - choices: &crate::pass1::per_computation_candidates::ComputationChoices, -) { +fn assert_sql_count_choices(choices: &[HowToComputeAggregateMeasure]) { assert_eq!( - choices.candidates, + choices, vec![HowToComputeAggregateMeasure::ExactAccumulator(vec![ ExactKind::Count ])] ); - assert_eq!( - choices.pruned, - [SketchAlgorithm::Cms, SketchAlgorithm::UnivMon] - .into_iter() - .map(|algorithm| ( - HowToComputeAggregateMeasure::Summary(algorithm), - PruneReason::NoRelationalCountSketchRealization, - )) - .collect::>() - ); } fn quantile() -> AggIntent { @@ -545,9 +447,9 @@ fn equals(column: usize, value: ScalarValue) -> Predicate { fn key_only_having_is_pushed_below_and_keeps_candidates() { let node = grouped_count(None); let root = having_filter(node.clone(), equals(0, ScalarValue::Utf8("a".into()))); - let inventory = generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(root.clone()), - ])) + let inventory = generate_per_computation_candidates( + &memo_of_queries(&[QueryRoot::Operator(root.clone())]).unwrap(), + ) .unwrap(); let targets = &inventory.targets; assert_eq!(targets.len(), 1); @@ -568,10 +470,13 @@ fn key_only_having_is_pushed_below_and_keeps_candidates() { #[test] fn value_having_stays_above_output_and_keeps_candidates() { let value = || equals(1, ScalarValue::Int64(10)); - let inventory = generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(having_filter(grouped_count(None), value())), - QueryRoot::Operator(grouped_count(Some(value()))), - ])) + let inventory = generate_per_computation_candidates( + &memo_of_queries(&[ + QueryRoot::Operator(having_filter(grouped_count(None), value())), + QueryRoot::Operator(grouped_count(Some(value()))), + ]) + .unwrap(), + ) .unwrap(); let targets = &inventory.targets; assert_eq!(targets.len(), 2); @@ -586,37 +491,6 @@ fn value_having_stays_above_output_and_keeps_candidates() { } } -/// `avg`'s one exact option is sum and count states when its input is never -/// null, and plain `Exact` when that rewrite is pruned. -#[test] -fn avg_rewrites_to_sum_and_count_states() { - let avg = || vec![AggIntent::Avg { col: None }]; - let inventory = generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(aggregate_over(scan(false), avg(), vec![], None)), - QueryRoot::Operator(aggregate_over(scan(true), avg(), vec![], None)), - ])) - .unwrap(); - let targets = &inventory.targets; - assert_eq!( - targets[0].ways_to_compute, - vec![HowToComputeAggregateMeasure::ExactAccumulator(vec![ - ExactKind::Sum, - ExactKind::Count - ])] - ); - assert_eq!( - targets[1].ways_to_compute, - vec![HowToComputeAggregateMeasure::Exact] - ); - assert_eq!( - inventory.pruned_of_target(1, false), - vec![( - HowToComputeAggregateMeasure::ExactAccumulator(vec![ExactKind::Sum, ExactKind::Count]), - PruneReason::NullableAvgInput - )] - ); -} - /// A key-only `HAVING` above a shared aggregate stays above the output when /// another consumer reads the aggregate without it. #[test] @@ -624,10 +498,13 @@ fn having_on_one_consumer_of_a_shared_aggregate_stays_above() { let node = grouped_count(None); let key = equals(0, ScalarValue::Utf8("a".into())); let filtered = having_filter(node.clone(), key.clone()); - let targets = generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(filtered.clone()), - QueryRoot::Operator(node.clone()), - ])) + let targets = generate_per_computation_candidates( + &memo_of_queries(&[ + QueryRoot::Operator(filtered.clone()), + QueryRoot::Operator(node.clone()), + ]) + .unwrap(), + ) .unwrap() .targets; assert_eq!(targets.len(), 1); @@ -643,10 +520,9 @@ fn having_on_one_consumer_of_a_shared_aggregate_stays_above() { // Two consumers applying the same key-only predicate push it below. let first = having_filter(node.clone(), key.clone()); let second = having_filter(node, key); - let targets = generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(first), - QueryRoot::Operator(second), - ])) + let targets = generate_per_computation_candidates( + &memo_of_queries(&[QueryRoot::Operator(first), QueryRoot::Operator(second)]).unwrap(), + ) .unwrap() .targets; assert!(targets[0] @@ -672,10 +548,13 @@ fn a_scalar_reference_consumer_keeps_a_key_only_having_above() { negated: false, }), ); - let inventory = generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(filtered.clone()), - QueryRoot::Operator(exists), - ])) + let inventory = generate_per_computation_candidates( + &memo_of_queries(&[ + QueryRoot::Operator(filtered.clone()), + QueryRoot::Operator(exists), + ]) + .unwrap(), + ) .unwrap(); let targets = &inventory.targets; let target = targets @@ -700,9 +579,9 @@ fn only_the_adjacent_of_stacked_filters_is_having() { let inner_key = equals(0, ScalarValue::Utf8("a".into())); let inner = having_filter(node.clone(), inner_key.clone()); let outer = having_filter(inner.clone(), equals(0, ScalarValue::Utf8("b".into()))); - let targets = generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(outer), - ])) + let targets = generate_per_computation_candidates( + &memo_of_queries(&[QueryRoot::Operator(outer)]).unwrap(), + ) .unwrap() .targets; assert_eq!(targets.len(), 1); @@ -726,13 +605,16 @@ fn global_aggregate_having_stays_above() { }] }; let never = Predicate(ScalarExpr::Literal(ScalarValue::Boolean(false))); - let targets = generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(having_filter( - aggregate_over(scan(false), count(), vec![], None), - never.clone(), - )), - QueryRoot::Operator(aggregate_over(scan(false), count(), vec![], Some(never))), - ])) + let targets = generate_per_computation_candidates( + &memo_of_queries(&[ + QueryRoot::Operator(having_filter( + aggregate_over(scan(false), count(), vec![], None), + never.clone(), + )), + QueryRoot::Operator(aggregate_over(scan(false), count(), vec![], Some(never))), + ]) + .unwrap(), + ) .unwrap() .targets; assert_eq!(targets.len(), 2); @@ -746,45 +628,6 @@ fn global_aggregate_having_stays_above() { } } -/// #509 Example 1 Q2 under the default exact target: the top-k's one option is plain -/// `Exact`, its heap summaries pruned `RequiresExact`; `sum_over_time` gets -/// its exact Sum accumulator. -#[test] -fn example_1_exact_q2_prunes_topk_heaps() { - let roots = exact_promql("topk by (job) (10, sum_over_time(http_requests_total[1m]))"); - let candidates = - generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&roots)).unwrap(); - let found: Vec<_> = candidates - .targets - .iter() - .enumerate() - .map(|(index, _)| choices_of(&candidates, index)) - .collect(); - let topk = found - .iter() - .find(|choices| choices.candidates == [HowToComputeAggregateMeasure::Exact]) - .expect("top-k target"); - assert_eq!( - topk.pruned, - vec![ - ( - HowToComputeAggregateMeasure::Summary(SketchAlgorithm::CmsWithHeap), - PruneReason::RequiresExact - ), - ( - HowToComputeAggregateMeasure::Summary(SketchAlgorithm::CountSketchWithHeap), - PruneReason::RequiresExact - ), - ] - ); - assert!(found.iter().any(|choices| choices.candidates - == [HowToComputeAggregateMeasure::ExactAccumulator(vec![ - ExactKind::Sum - ])] - && choices.pruned.is_empty())); - assert_eq!(found.len(), 2, "{found:#?}"); -} - /// The `HAVING` entries of the target whose measure satisfies `is`. fn having_of( inventory: &crate::pass1::per_computation_candidates::PerComputationCandidates, @@ -822,9 +665,9 @@ fn scalar_reference_filter_is_not_having() { }; let agg = aggregate_over(scan(false), count(), vec![], None); let root = having_filter(scan(false), above_scalar(1, agg.clone())); - let inventory = generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(root), - ])) + let inventory = generate_per_computation_candidates( + &memo_of_queries(&[QueryRoot::Operator(root)]).unwrap(), + ) .unwrap(); let targets = &inventory.targets; assert_eq!(targets.len(), 1); @@ -837,9 +680,9 @@ fn scalar_reference_filter_is_not_having() { let agg = aggregate_over(scan(false), count(), vec![], None); let root = having_filter(agg.clone(), above_scalar(0, agg.clone())); - let targets = generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(root.clone()), - ])) + let targets = generate_per_computation_candidates( + &memo_of_queries(&[QueryRoot::Operator(root.clone())]).unwrap(), + ) .unwrap() .targets; assert_eq!(targets.len(), 1); @@ -857,7 +700,7 @@ fn scalar_reference_filter_is_not_having() { #[test] fn promql_scalar_operand_is_not_having() { let roots = exact_promql("http_requests_total > scalar(sum(http_requests_total))"); - let targets = generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&roots)) + let targets = generate_per_computation_candidates(&memo_of_queries(&roots).unwrap()) .unwrap() .targets; assert!(!targets.is_empty()); @@ -870,8 +713,7 @@ fn promql_scalar_operand_is_not_having() { } let roots = exact_promql("sum by (job) (x) > scalar(count(x))"); - let inventory = - generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&roots)).unwrap(); + let inventory = generate_per_computation_candidates(&memo_of_queries(&roots).unwrap()).unwrap(); assert!(having_of(&inventory, |intent| matches!( intent, AggIntent::Count { .. } @@ -897,10 +739,9 @@ fn having_shared_with_a_project_consumer_stays_above() { child: node, })) .unwrap(); - let targets = generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(filtered), - QueryRoot::Operator(project), - ])) + let targets = generate_per_computation_candidates( + &memo_of_queries(&[QueryRoot::Operator(filtered), QueryRoot::Operator(project)]).unwrap(), + ) .unwrap() .targets; assert_eq!(targets.len(), 1); @@ -934,9 +775,9 @@ fn without_and_per_entity_having_stays_above() { })) .unwrap(); let root = having_filter(node, equals(0, ScalarValue::Utf8("a".into()))); - let targets = generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(root), - ])) + let targets = generate_per_computation_candidates( + &memo_of_queries(&[QueryRoot::Operator(root)]).unwrap(), + ) .unwrap() .targets; assert_eq!(targets.len(), 1, "{reduction:?}"); @@ -963,9 +804,9 @@ fn composed_candidate_replaces_chosen_target_with_summary_evaluation() { cols: vec![0], accuracy: approximate(), }); - let inventory = generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(producer.clone()), - ])) + let inventory = generate_per_computation_candidates( + &memo_of_queries(&[QueryRoot::Operator(producer.clone())]).unwrap(), + ) .unwrap(); let exact = compose_logical_candidate(&inventory, &[0]).unwrap(); let QueryRoot::Operator(kept) = &exact[0] else { @@ -979,20 +820,10 @@ fn composed_candidate_replaces_chosen_target_with_summary_evaluation() { let QueryRoot::Operator(estimate) = &hll[0] else { panic!("operator root expected") }; - let Some(ASAPOp::SummaryEstimate { summary_input, .. }) = estimate.asap() else { - panic!("summary evaluation expected") - }; - // The state takes every row of its source: one unrestricted selection box. - let coverage = summary_input.coverage().unwrap(); - assert!(OperatorNode::reachable(coverage.definition()).iter().any(|node| matches!( - node.non_asap(), - Some(NonASAPOp::Scan { source: Source::Table { table_ref }, .. }) if table_ref == "flows" - ))); - assert_eq!(coverage.selection().len(), 1); - assert!( - coverage.selection()[0].columns().is_empty() - && coverage.selection()[0].relative_time().is_none() - ); + assert!(matches!( + estimate.asap(), + Some(ASAPOp::SummaryEstimate { .. }) + )); hll[0].validate_structure().unwrap(); assert!(compose_logical_candidate(&inventory, &[99]).is_err()); } @@ -1000,12 +831,15 @@ fn composed_candidate_replaces_chosen_target_with_summary_evaluation() { /// The one root of the workload candidate composed from `roots` and `choice`. fn composed(roots: Vec>, choice: &[usize]) -> Vec> { use crate::pass1::per_computation_candidates::compose_logical_candidate; - let inventory = generate_per_computation_candidates(&generate_dag_of_logical_memo_groups( - &roots - .into_iter() - .map(QueryRoot::Operator) - .collect::>(), - )) + let inventory = generate_per_computation_candidates( + &memo_of_queries( + &roots + .into_iter() + .map(QueryRoot::Operator) + .collect::>(), + ) + .unwrap(), + ) .unwrap(); compose_logical_candidate(&inventory, choice) .unwrap() @@ -1049,27 +883,6 @@ fn summary_agg(node: &OperatorNode) -> &ASAPOp { } } -/// A summary is sized when the workload candidate is composed, for its -/// measure's accuracy requirement. -#[test] -fn composed_summary_is_sized_for_its_measure() { - use asap_accuracy::test_support::default_size_params; - let intent = quantile(); - let [root] = <[_; 1]>::try_from(composed(vec![aggregate(intent.clone())], &[1])).unwrap(); - let ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), - .. - } = summary_agg(&root) - else { - panic!("sketch state expected") - }; - assert_eq!(*kind.algorithm(), SketchAlgorithm::Kll); - assert_eq!( - kind.params(), - &default_size_params(SketchAlgorithm::Kll, &intent, 0.05, 0.01) - ); -} - /// A measure filter becomes the state's filter; a key-only `HAVING` applied /// by every consumer moves below the state; a value `HAVING` stays a `Filter` /// above the estimate. @@ -1268,9 +1081,9 @@ fn own_having_subquery_is_substituted() { use crate::pass1::per_computation_candidates::compose_logical_candidate; let inner = aggregate(quantile()); let outer = grouped_quantile(Some(above_scalar(1, inner.clone()))); - let inventory = generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(outer.clone()), - ])) + let inventory = generate_per_computation_candidates( + &memo_of_queries(&[QueryRoot::Operator(outer.clone())]).unwrap(), + ) .unwrap(); assert_eq!(inventory.targets.len(), 2); let kll = 1; @@ -1379,7 +1192,7 @@ fn memo_cached_rules_preserve_occurrence_specific_having() { QueryRoot::Operator(shared), QueryRoot::Operator(having_filter(other, key)), ]; - let memo = generate_dag_of_logical_memo_groups(&roots); + let memo = memo_of_queries(&roots).unwrap(); let cached = generate_per_computation_candidates(&memo).unwrap(); assert_eq!(cached.targets.len(), 1); // Roots 0 and 2 apply the same `Filter`, one memo group: one entry. diff --git a/crates/logical-asap-optimization/src/pass1/realization.rs b/crates/logical-asap-optimization/src/pass1/realization.rs index d5dc1f09c..f510bdf65 100644 --- a/crates/logical-asap-optimization/src/pass1/realization.rs +++ b/crates/logical-asap-optimization/src/pass1/realization.rs @@ -83,7 +83,9 @@ pub(crate) fn summarised_column(intent: &AggIntent, child_schema: &Schema) -> Co /// [`ranked_column`] as a reference: the sample value when it is that /// column, else the column by name. -pub(crate) fn ranked_value(child: &OperatorNode) -> Result<(ColumnId, ColumnRef), ResolveError> { +pub(crate) fn ranked_value( + child: &Rc, +) -> Result<(ColumnId, ColumnRef), ResolveError> { let ranked = ranked_column(child)?; let column = match resolve_column_ref(&ColumnRef::SampleValue, &child.schema) { Ok(value) if value == ranked => ColumnRef::SampleValue, @@ -146,7 +148,7 @@ pub(crate) fn realize_keyed_additive_summary_input( let FieldDataType::Sketch(kind, _) = family else { return None; }; - let heap_algorithm = kind.algorithm(); + let heap_algorithm = kind; if !matches!( heap_algorithm, SketchAlgorithm::CmsWithHeap | SketchAlgorithm::CountSketchWithHeap @@ -353,7 +355,7 @@ mod tests { use crate::test_support::metric_scan; use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::scalar::{CompareOpKind, ScalarValue}; - use asap_types::ir::schema::{GroupingStrategy, SketchKind, SketchParams}; + use asap_types::ir::schema::GroupingStrategy; use asap_types::ir::{ExprSemantics, Predicate}; /// A heap sketch consumes every raw row, so the keyed input rule refuses @@ -377,17 +379,8 @@ mod tests { k: 5, accuracy: AccuracyTarget::max_error(0.01), }; - let family = FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::CmsWithHeap, - SketchParams::CmsWithHeap { - width: 64, - depth: 4, - heap_size: 5, - }, - ), - GroupingStrategy::default(), - ); + let family = + FieldDataType::Sketch(SketchAlgorithm::CmsWithHeap, GroupingStrategy::default()); let output = Reduction::Reduce(asap_types::ir::operator::operator_properties::GroupKeys::none()); assert!(realize_keyed_additive_summary_input( diff --git a/crates/logical-asap-optimization/src/pass2.rs b/crates/logical-asap-optimization/src/pass2.rs index 2607e1edf..dbdc6f273 100644 --- a/crates/logical-asap-optimization/src/pass2.rs +++ b/crates/logical-asap-optimization/src/pass2.rs @@ -1,11 +1,6 @@ -//! Stage 1, Pass 2: ASAP-aware sharing over the Pass 1 candidates, through the -//! logical memo and its rules. +//! Stage 1, Pass 2: the rules over groups and the logical edges. -/// Stage 1, Pass 2: ASAP-aware common-subexpression elimination (the -/// identical-expression rule). -pub(crate) mod asap_aware_cse; -pub(crate) mod finer_aggregate; /// Stage 1, Pass 2's entry, producing [`asap_types::stages::LogicalASAPMemoCandidates`]. pub(crate) mod logical_memo_candidates; -/// The logical memo: candidates, rules and enumeration over Stage 0's groups. +/// The rules over groups. pub(crate) mod memo; diff --git a/crates/logical-asap-optimization/src/pass2/asap_aware_cse.rs b/crates/logical-asap-optimization/src/pass2/asap_aware_cse.rs deleted file mode 100644 index 991e96f40..000000000 --- a/crates/logical-asap-optimization/src/pass2/asap_aware_cse.rs +++ /dev/null @@ -1,3557 +0,0 @@ -//! Stage 1 (logical ASAP-aware optimization), Pass 2: ASAP-aware -//! common-subexpression elimination, as defined in -//! `docs/design_docs/proposals/planner-layering.md` (Algorithm 2). -//! -//! The memo rules (`pass2::memo::rules`) are built on these helpers: the -//! window-composition rule for tumbling windows, and two reuse rules (#680), -//! rollup and exact top-k reuse. Identical expressions are shared by memo -//! group identity, and summary builds of one family at different sizes by -//! the memo's shared-producer rule (`memo::rules::shared_producer`); both -//! rules' whole-workload forms are gone. -//! -//! Scans of one metric that project different labels read the same rows. -//! [`with_union_scan_labels`] widens them to the union of their labels, so -//! interning finds them identical (`memo::rules::scan_labels`). -//! -//! # Window-composition rule -//! -//! A query repeating every `E` over a window `W` can build its summary as -//! back-to-back tumbling windows of width `w = gcd(W, E)` (Algorithm 2's -//! `Tumbling(L)` for one query), merged before each read. A *tumbling window -//! candidate* ([`tumbling_window_candidates`]) is a `SummaryAgg` over a range -//! `TimeRange(W)` whose -//! family merges in the family-capability table ([`FamilyCapability`]): -//! exact accumulators included, and also families whose merged readouts -//! are only approximate (heap top-k), whose readers then carry no -//! guarantee and are left to plan selection's evidence. A family without a -//! merge (IRate) has no tumbling-window candidate. -//! [`window_composition_pass2_candidates`] returns one Pass 2 candidate per -//! non-empty subset of the tumbling-window candidates: the build of each in the subset -//! becomes a `SummaryMerge` over one [`ASAPOp::SummaryTumblingWindowChain`] of `W / w` tumbling windows, -//! a compact node whatever the tumbling-window count, with the build's family and -//! size (a mergeable family's merged state has one build's guarantee), and -//! every guarantee above is composed again. Nothing is materialized here: -//! the tumbling windows are rebuilt at each evaluation until Stage 2 keeps them. -//! -//! Nested windows share one chain: when a subset holds builds of one family -//! (at any size) over the same input below their ranges, read the same way, -//! with the same output column names, whose tumbling windows differ in width or count, -//! the subset also comes as a Pass 2 candidate in which each such group -//! reads one chain ([`SharedTumblingWindowChain`]) over the longest window, sized for -//! every member ([`size_for_all_readers`]). Two grids are offered: the -//! finest member's own tumbling windows, for the members whose widths it divides -//! ([`nested_window_groups`]), and, when that differs, the gcd of all the -//! members' widths, even finer than every member's own tumbling windows -//! ([`gcd_window_groups`]); Stage 3's pricing weighs the extra states and -//! merges of finer tumbling windows. Queries repeating at different intervals read the -//! same tumbling windows, each at its own cadence. Every window ends at the evaluation -//! time, so each shorter one is the chain's newest tumbling windows: its -//! `SummaryMerge` merges only those (`newest_tumbling_windows`), and has the guarantee -//! of one build over its own window. A build grouped by fewer keys also -//! reads the chain of a finer grouping over a window at least its own -//! ([`rollup_window_groups`]): it merges its newest tumbling windows, then rolls the -//! groups up into its own, as the rollup rule does. Stage 2 never keeps a -//! chain read at different intervals across evaluations from query time -//! (each reader would need one new tumbling window per evaluation). Long-window segments, sliding -//! windows and Exponential Histograms are not implemented. -//! -//! # Rollup rule -//! -//! A coarse grouped computation can be answered from a finer one the -//! candidate already computes over the same rows: `sum by (job)` from -//! `sum by (job, instance)`. The rule never introduces a finer grouping -//! nobody asked for. A *rollup candidate* ([`rollup_candidates`]) is a -//! coarse build and a finer build of the same family over the same rows (a -//! PromQL scan may project other labels), whose group keys include the -//! coarse ones: -//! - An exact accumulator whose family merges across disjoint groups with -//! an exact merged value ([`rollup_kind`]: sums, counts, minima and -//! maxima; counter states merge only in time). The coarse build becomes -//! an exact accumulator over its coarsest finer build's finalized values, -//! grouped by the coarse keys (a count sums the finer counts), so the -//! value stays exact. -//! - A per-group sketch (KMV, HLL, KLL, DDSketch, Count-Min, Count Sketch, -//! UnivMon, at any size) read only by estimates whose merged readouts -//! keep a guarantee (not a heap's top-k or UnivMon's entropy). The coarse build becomes a rollup (a regrouping -//! `SummaryMerge`) of its finest finer build's states by the coarse -//! keys. The groups of one build partition its rows, so the rolled-up -//! state keeps the family's merged guarantee: the direct coarse build's -//! for KMV, HLL, DDSketch and the linear sketches, the same rank bound -//! for KLL. -//! -//! A sketch build over fewer rows than a finer one, whose extra conjuncts -//! read only the finer keys (`… WHERE city IN ('NYC', 'SF')` from a build -//! by `city`), is answered by first keeping the finer states that pass -//! those conjuncts, a compensating `Filter` over the states -//! ([`filtered_finer`], [`match_finer`](crate::pass2::finer_aggregate)), then -//! merging them. A SQL aggregate's finer *result* is read the same way by -//! the memo rule `memo::rules::finer_aggregate`. -//! -//! [`rollup_pass2_candidates`] returns one Pass 2 candidate per non-empty -//! subset of the rollup *choices* ([`rollup_choices`]): each exact rollup -//! alone, and every sketch rollup of one finest build together, since they -//! share its size. A chosen sketch choice sizes the finest build for every -//! reader, its own and each coarse one's (the strictest consumer, as the -//! shared-producer rule sizes a shared build), and composes every -//! guarantee above it again. So 15 groupings of one window are one choice, -//! not 2^14 subsets. -//! -//! # Exact top-k reuse rule -//! -//! Exact top-k computations of one input and partition with different `k` -//! read one ranking: the rows of the largest top-k, `Limit(k_max)` over -//! `Sort`, which hold every smaller top-k ([`topk_reuse_candidates`], -//! `memo::rules::topk`). Only exact computations over exact input are -//! reused; a sketch's heap is not, as its membership has no guarantee yet. - -use std::collections::HashMap; -use std::collections::HashSet; -use std::rc::Rc; - -use asap_types::ir::operator::operator_properties::{Reduction, Source}; -use asap_types::ir::operator::AggIntent; -use asap_types::ir::scalar::resolve_column_ref; -use asap_types::ir::schema::{ - ExactKind, FamilyCapability, FieldDataType, GroupingStrategy, MergeInputRule, MergedError, - Schema, SketchKind, SummaryInputExpr, SummaryUpdate, WeightDomain, PROMQL_SERIES_IDENTITY, -}; -use asap_types::ir::ScalarExpr; -use asap_types::ir::SortKey; -use asap_types::ir::{ - ASAPOp, NonASAPOp, Operator, OperatorNode, OperatorResultKind, Predicate, QueryRoot, - TimeRangeKind, -}; - -use asap_accuracy::estimators::size_for_all_readers; -use asap_accuracy::propagation::recompose_guarantees; - -use asap_types::ir::SchemaDerivationError; - -/// Why a sharing rewrite builds no candidate. The rules report it as text -/// ([`Pass2RuleError`](crate::pass2::memo::state_groups::Pass2RuleError)) or -/// drop the candidate. -#[derive(Debug, thiserror::Error)] -pub enum CseError { - #[error(transparent)] - Structure(#[from] SchemaDerivationError), - #[error("unsupported substitution in logical ASAP-aware optimization: {0}")] - Unsupported(&'static str), - /// A rule found more than `max` candidates of one kind, whose 2^n subsets - /// it would enumerate: the workload is too large to plan this way, not - /// bad input. - #[error("found {found} {counted}, more than {max}")] - TooManyCandidates { - /// What was counted, e.g. "rollup choices". - counted: &'static str, - /// How many were found, more than `max`. - found: usize, - /// The most the rule enumerates. - max: usize, - }, -} - -/// `roots` with the time-series scans of one metric made identical when -/// they project different labels, so that interning finds them identical: the PromQL frontend projects only the labels each query uses -/// (`ts, value, job, …` for `quantile by (job) (0.5, data)`), and -/// structural CSE shares only nodes with equal schemas. Every scan of a -/// metric without predicates projects the union of the labels its scans -/// project, in the order the walk meets them (before the series identity, -/// with the unique keys moved along). -/// The nodes that pass the labels on (`passes_labels_on`) keep the wider -/// rows above a widened scan; any other consumer reads them through a -/// `Project` that keeps the columns it read, by name. `None` when every metric's scans already -/// agree, when two of them give one label different types or one is closed -/// and the other not, when a node would change arity, or when a consumer -/// would have to narrow a summary state or a grouped or filtered tumbling-window chain -/// would read a `Project`. -pub fn with_union_scan_labels(roots: &[QueryRoot]) -> Result>, CseError> { - let mut nodes = Vec::new(); - let mut seen = HashSet::new(); - for node in roots.iter().flat_map(QueryRoot::operator_refs) { - collect_children_first(node, &mut seen, &mut nodes); - } - // Each metric's union schema, `None` once two of its scans conflict. - let mut unions: Vec<(&Source, Option)> = Vec::new(); - for node in &nodes { - let Some((source, schema)) = unfiltered_series_scan(node) else { - continue; - }; - let Some(i) = unions.iter().position(|(s, _)| *s == source) else { - unions.push((source, Some(schema.clone()))); - continue; - }; - let Some(union) = unions[i].1.as_mut() else { - continue; - }; - // A scan known to hold every label and one that is not cannot share. - if union.closed != schema.closed { - unions[i].1 = None; - continue; - } - for field in &schema.fields { - match union.fields.iter().find(|f| f.name == field.name) { - Some(known) if known == field => {} - Some(_) => { - unions[i].1 = None; - break; - } - None => { - let at = union - .fields - .iter() - .position(|f| f.name == PROMQL_SERIES_IDENTITY) - .unwrap_or(union.fields.len()); - union.fields.insert(at, field.clone()); - // Unique keys (the series identity and ts) name columns - // by position. - for column in union.unique_keys.iter_mut().flatten() { - if *column >= at { - *column += 1; - } - } - } - } - } - } - let union_of = |node: &OperatorNode| { - let (source, schema) = unfiltered_series_scan(node)?; - let (_, union) = unions.iter().find(|(s, _)| *s == source)?; - union.as_ref().filter(|union| *union != schema) - }; - if !nodes.iter().any(|node| union_of(node).is_some()) { - return Ok(None); - } - let mut widened: HashMap<*const OperatorNode, Rc> = HashMap::new(); - for node in &nodes { - let out = match (union_of(node), node.non_asap()) { - (Some(union), Some(NonASAPOp::Scan { source, .. })) => { - let mut schema = union.clone(); - schema.time_index = schema.fields.iter().position(|f| f.name == "ts"); - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Scan { - source: source.clone(), - predicates: Vec::new(), - schema, - }))? - } - _ if node - .children() - .iter() - .all(|c| Rc::ptr_eq(c, &widened[&Rc::as_ptr(c)])) => - { - Rc::clone(node) - } - // A node whose renamed outputs would change arity is not widened. - _ if passes_labels_on(node) => { - match node.with_new_children(|c| Rc::clone(&widened[&Rc::as_ptr(c)])) { - Ok(rebuilt) => Rc::new(rebuilt), - Err(_) => return Ok(None), - } - } - // A grouped or filtered tumbling-window chain over a `Project` would hide - // its window from plan selection. - _ if matches!(node.asap(), Some(ASAPOp::SummaryTumblingWindowChain { .. })) => { - return Ok(None); - } - _ => { - let mut children = HashMap::new(); - for child in node.children() { - let wide = &widened[&Rc::as_ptr(child)]; - let Some(narrow) = narrowed(wide, &child.schema)? else { - return Ok(None); - }; - children.insert(Rc::as_ptr(child), narrow); - } - Rc::new(node.with_new_children(|c| Rc::clone(&children[&Rc::as_ptr(c)]))?) - } - }; - widened.insert(Rc::as_ptr(node), out); - } - let mut out = Vec::with_capacity(roots.len()); - for query_root in roots { - let mut mapped = HashMap::new(); - for node in query_root.operator_refs() { - let Some(narrow) = narrowed(&widened[&Rc::as_ptr(node)], &node.schema)? else { - return Ok(None); - }; - mapped.insert(Rc::as_ptr(node), narrow); - } - out.push(match query_root { - QueryRoot::Operator(node) => QueryRoot::Operator(Rc::clone(&mapped[&Rc::as_ptr(node)])), - QueryRoot::Scalar(expr) => QueryRoot::Scalar( - expr.map_operator_refs(&mut |node| Rc::clone(&mapped[&Rc::as_ptr(node)])), - ), - }); - } - Ok(Some(recompose_guarantees(&out)?)) -} - -/// The source and schema of a time-series `Scan` without predicates. -fn unfiltered_series_scan(node: &OperatorNode) -> Option<(&Source, &Schema)> { - match node.non_asap() { - Some(NonASAPOp::Scan { - source: source @ Source::TimeSeries { .. }, - predicates, - schema, - }) if predicates.is_empty() => Some((source, schema)), - _ => None, - } -} - -/// Whether `node`, over a child with more labels, keeps them in its own -/// output: a time range or shift of the rows, and a per-series build or -/// tumbling-window chain without a filter (keyed by the series identity, which -/// determines every label) with its merges, estimates and finalizations. -/// The executor reads a per-series build's raw rows from its `TimeRange` -/// directly, and plan selection reads a tumbling-window chain's window from the rows -/// below it, so nothing may sit between them. -fn passes_labels_on(node: &OperatorNode) -> bool { - matches!( - node.non_asap(), - Some(NonASAPOp::TimeRange { .. } | NonASAPOp::TimeShift { .. }) - ) || matches!( - node.asap(), - Some( - ASAPOp::SummaryAgg { - reduction: Reduction::PerEntity, - filter: None, - .. - } | ASAPOp::SummaryTumblingWindowChain { - reduction: Reduction::PerEntity, - filter: None, - .. - } | ASAPOp::SummaryMerge { .. } - | ASAPOp::SummaryEstimate { .. } - | ASAPOp::FinalizeExactAccumulator { .. } - ) - ) -} - -/// `wide`, which holds every column of `schema` and perhaps more, with -/// only `schema`'s columns, by name: `wide` itself when it has exactly -/// them; `None` when `wide` is a summary state, which has no columns to -/// project. -fn narrowed( - wide: &Rc, - schema: &Schema, -) -> Result>, CseError> { - use asap_types::ir::ProjectItem; - if wide.schema.fields == schema.fields { - return Ok(Some(Rc::clone(wide))); - } - if wide.asap().is_some_and(|op| op.produced_state().is_some()) { - return Ok(None); - } - let cols = schema - .fields - .iter() - .map(|field| { - let column = wide - .schema - .fields - .iter() - .position(|f| f.name == field.name && f.table == field.table) - .expect("a widened node keeps every column it had"); - ProjectItem { - alias: Some(field.name.clone()), - expr: ScalarExpr::Column(column), - } - }) - .collect(); - Ok(Some(OperatorNode::new_shared(Operator::NonASAP( - NonASAPOp::Project { - cols, - qualifier: None, - child: Rc::clone(wide), - }, - ))?)) -} - -fn collect_children_first( - node: &Rc, - seen: &mut std::collections::HashSet<*const OperatorNode>, - out: &mut Vec>, -) { - if !seen.insert(Rc::as_ptr(node)) { - return; - } - for child in node.children() { - collect_children_first(child, seen, out); - } - out.push(Rc::clone(node)); -} - -/// Whether builds of families `a` and `b` can be one build: one sketch -/// algorithm and grouping at any sizes, or one exact accumulator. -fn same_build_family(a: &FieldDataType, b: &FieldDataType) -> bool { - match (a, b) { - (FieldDataType::Sketch(a, grouping), FieldDataType::Sketch(b, grouping_b)) => { - a.algorithm() == b.algorithm() && grouping == grouping_b - } - _ => a == b, - } -} - -/// One build serving every member: the first member with the members' -/// shared size ([`size_for_all_readers`]). -fn sized_for_all(members: &[&Rc]) -> Result>, CseError> { - // An exact accumulator has one size: the first member serves them all. - if matches!( - members[0].asap(), - Some(ASAPOp::SummaryAgg { - family: FieldDataType::ExactAggregate(..), - .. - }) - ) { - return Ok(Some(Rc::clone(members[0]))); - } - let kinds: Vec<&SketchKind> = members - .iter() - .filter_map(|member| match member.asap() { - Some(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), - .. - }) => Some(kind), - _ => None, - }) - .collect(); - let params: Vec<_> = kinds.iter().map(|kind| kind.params()).collect(); - let Some(params) = size_for_all_readers(¶ms) else { - return Ok(None); - }; - let first = members[0]; - let Some(ASAPOp::SummaryAgg { - child, - family: FieldDataType::Sketch(kind, grouping_strategy), - input, - reduction, - grouping, - filter, - }) = first.asap() - else { - return Ok(None); - }; - let operator = Operator::ASAP(ASAPOp::SummaryAgg { - child: Rc::clone(child), - family: FieldDataType::Sketch( - SketchKind::new(*kind.algorithm(), params), - grouping_strategy.clone(), - ), - input: input.clone(), - reduction: reduction.clone(), - grouping: grouping.clone(), - filter: filter.clone(), - }); - let mut build = OperatorNode::new(operator)?; - // Keep the member's explicit output names. - for (field, retained) in build.schema.fields.iter_mut().zip(&first.schema.fields) { - field.name = retained.name.clone(); - field.table = retained.table.clone(); - } - Ok(Some(Rc::new(build))) -} - -/// One summary build the window-composition rule can build as tumbling -/// tumbling windows: a `SummaryAgg` over a range `TimeRange(W)` whose family merges -/// (the family-capability table: exact accumulators included, and families -/// whose merged readouts are only approximate), read only by queries that -/// repeat at a fixed interval. Its tumbling windows are `tumbling_window_count = W / w` back-to-back -/// windows of `w = gcd(W, E)` over the queries' intervals `E` (and, for a -/// query with a range evaluation, its step), at least two. -#[derive(Debug, Clone)] -pub struct TumblingWindowCandidate { - /// The `SummaryAgg` over `TimeRange(W)` built as tumbling windows. - pub build: Rc, - /// The width `w` of one tumbling window. - pub tumbling_window_width_ms: u64, - /// The number of tumbling windows, `W / w`, at least two. - pub tumbling_window_count: u32, -} - -impl TumblingWindowCandidate { - /// The window `W` its tumbling windows cover, `tumbling_window_width_ms · tumbling_window_count`. - pub fn window_ms(&self) -> u64 { - self.tumbling_window_width_ms * u64::from(self.tumbling_window_count) - } -} - -/// A Pass 2 candidate of the window-composition rule: the candidate's roots -/// with the build of each of `tumbling_window_candidates` built as its tumbling windows, -/// merged before every read, and every guarantee composed again; the error -/// when that fails, which leaves the other Pass 2 candidates unaffected. -#[derive(Debug)] -pub struct WindowCompositionPass2Candidate { - /// The builds built as tumbling windows, in [`tumbling_window_candidates`] order. - /// Only tests read it. - #[cfg_attr(not(any(test, feature = "test-support")), allow(dead_code))] - pub tumbling_window_candidates: Vec, - /// The chains each shared by several of `tumbling_window_candidates` - /// ([`nested_window_groups`] or [`gcd_window_groups`]); empty when each - /// build has its own chain. - pub shared_chains: Vec, - /// The candidate's roots with those builds in tumbling windows. - pub roots: Result, CseError>, -} - -/// One tumbling-window chain several tumbling-window candidates read (window-composition rule): -/// tumbling windows of `tumbling_window_width_ms`, a width that divides every member's own tumbling window -/// width, and `tumbling_window_count` of them, covering the longest member's window. -/// Each member merges the newest `W / tumbling_window_width_ms` tumbling windows of its window `W`. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct SharedTumblingWindowChain { - /// Positions in the Pass 2 candidate's `tumbling_window_candidates`, two or more. - pub members: Vec, - /// The width of one tumbling window of the shared chain. - pub tumbling_window_width_ms: u64, - /// The number of tumbling windows of the shared chain. - pub tumbling_window_count: u32, -} - -/// The most tumbling-window candidates one candidate may have: every non-empty subset -/// is a Pass 2 candidate, and with shared grids up to three, about three -/// million at 20. -pub(crate) const MAX_TUMBLING_WINDOW_CANDIDATES: usize = 20; - -/// The tumbling-window candidates of the candidate with query roots `roots`, children -/// first. `evaluation_grid_ms[q]` is the grid query `q`'s evaluations fall -/// on: how often it repeats, or with a range evaluation the greatest common -/// divisor of that and its step, and of its phase if declared -/// (`QueryWorkloadEntry::evaluation_grid_ms`); `None` when it does not -/// repeat at a fixed interval. Each tumbling-window width divides every reader's grid, -/// so with tumbling windows anchored at the epoch every evaluation, every range step -/// included, ends on a tumbling-window boundary. -pub fn tumbling_window_candidates( - roots: &[QueryRoot], - evaluation_grid_ms: &[Option], -) -> Vec { - let mut nodes = Vec::new(); - let mut seen = std::collections::HashSet::new(); - for node in roots.iter().flat_map(QueryRoot::operator_refs) { - collect_children_first(node, &mut seen, &mut nodes); - } - let reach: Vec> = roots - .iter() - .map(|root| { - root.operator_refs() - .into_iter() - .flat_map(OperatorNode::reachable) - .map(|node| Rc::as_ptr(&node)) - .collect() - }) - .collect(); - nodes - .into_iter() - .filter_map(|build| { - let window_ms = splittable_window_ms(&build)?; - let readers: Vec = (0..roots.len()) - .filter(|&q| reach[q].contains(&Rc::as_ptr(&build))) - .collect(); - let mut width = window_ms; - for &q in &readers { - width = asap_types::workload::greatest_common_divisor( - width, - evaluation_grid_ms.get(q).copied().flatten()?, - ); - } - let tumbling_window_count = u32::try_from(window_ms / width).ok()?; - (!readers.is_empty() && tumbling_window_count >= 1).then_some(TumblingWindowCandidate { - build, - tumbling_window_width_ms: width, - tumbling_window_count, - }) - }) - .collect() -} - -/// The window `W` of a build the rule can split: a `SummaryAgg` whose -/// family merges, over a range `TimeRange(W)` over an instant vector or a -/// SQL table's rows, read at the query's time or a fixed offset from it: no `@` anchor on any of -/// the time shifts stacked below the range. -fn splittable_window_ms(build: &OperatorNode) -> Option { - let Some(ASAPOp::SummaryAgg { child, family, .. }) = build.asap() else { - return None; - }; - FamilyCapability::of(family).merge_inputs?; - let Some(NonASAPOp::TimeRange { - range, - kind: TimeRangeKind::Range, - child: input, - }) = child.non_asap() - else { - return None; - }; - // An instant vector (PromQL `m[w]`) or a SQL table's rows (a trailing - // window); not a range vector, whose rows are already windows. - if !matches!( - input.result_kind, - OperatorResultKind::InstantVector | OperatorResultKind::Relation - ) { - return None; - } - // Refuse an `@` anchor on any of the shifts stacked below the range. - let mut below = input; - while let Some(NonASAPOp::TimeShift { shift, child }) = below.non_asap() { - if shift.at.is_some() { - return None; - } - below = child; - } - u64::try_from(range.as_millis()).ok().filter(|&ms| ms > 0) -} - -/// Every Pass 2 candidate of the window-composition rule for the candidate -/// with query roots `roots` and tumbling-window candidates `candidates` -/// ([`tumbling_window_candidates`]): one per non-empty subset of them, in -/// binary-counter order (the first candidate the lowest bit). Each build in -/// the subset becomes a `SummaryMerge` over one [`ASAPOp::SummaryTumblingWindowChain`] of -/// its tumbling windows; every other build is untouched. A subset whose builds can -/// share chains is followed by one Pass 2 candidate per shared grid, in -/// which each group reads one shared chain, each member merging its newest -/// tumbling windows: the finest member's tumbling windows (`nested_window_groups`), then the -/// gcd of the members' widths when that groups them differently -/// (`gcd_window_groups`); and by one error candidate per group whose -/// sizing fails. Each grid groups every build of a subset at once: a -/// shared gcd grid over only some of the builds that could share it is not -/// listed. An error only for more than -/// `MAX_TUMBLING_WINDOW_CANDIDATES` tumbling-window candidates; a Pass 2 candidate whose -/// composition fails carries its own error. -pub fn window_composition_pass2_candidates( - roots: &[QueryRoot], - candidates: &[TumblingWindowCandidate], -) -> Result, CseError> { - if candidates.len() > MAX_TUMBLING_WINDOW_CANDIDATES { - return Err(CseError::TooManyCandidates { - counted: "summary builds that could be built in tumbling windows", - found: candidates.len(), - max: MAX_TUMBLING_WINDOW_CANDIDATES, - }); - } - let rollable = rollable_tumbling_window_candidates(roots, candidates); - let mut pass2_candidates = Vec::new(); - for mask in 1u64..(1u64 << candidates.len()) { - let chosen: Vec = (0..candidates.len()) - .filter(|&i| mask >> i & 1 == 1) - .map(|i| candidates[i].clone()) - .collect(); - pass2_candidates.push(WindowCompositionPass2Candidate { - roots: in_tumbling_windows(roots, &chosen, &[]), - tumbling_window_candidates: chosen.clone(), - shared_chains: Vec::new(), - }); - let chosen_rollable: Vec = (0..candidates.len()) - .filter(|&i| mask >> i & 1 == 1) - .map(|i| rollable[i]) - .collect(); - push_every_shared_chain(roots, &chosen, &chosen_rollable, &mut pass2_candidates); - } - Ok(pass2_candidates) -} - -/// The Pass 2 candidates of [`window_composition_pass2_candidates`] in -/// which every one of `candidates` is in tumbling windows and reads a shared chain: -/// that function's candidates of the whole set with shared chains, in its -/// order, without building the other subsets'. -pub fn window_composition_shared_chain_candidates( - roots: &[QueryRoot], - candidates: &[TumblingWindowCandidate], -) -> Vec { - let rollable = rollable_tumbling_window_candidates(roots, candidates); - let mut pass2_candidates = Vec::new(); - push_every_shared_chain(roots, candidates, &rollable, &mut pass2_candidates); - pass2_candidates -} - -/// Push the shared-chain Pass 2 candidates of the subset `chosen` (each -/// rollable or not, `chosen_rollable`) of [`window_composition_pass2_candidates`]. -fn push_every_shared_chain( - roots: &[QueryRoot], - chosen: &[TumblingWindowCandidate], - chosen_rollable: &[bool], - pass2_candidates: &mut Vec, -) { - // Shared chains on the members' own finest tumbling windows, then, when it - // groups them differently, on the gcd of their widths. - let on_member_tumbling_windows = nested_window_groups(chosen); - let on_gcd_tumbling_windows = gcd_window_groups(chosen); - push_shared_chains(roots, chosen, &on_member_tumbling_windows, pass2_candidates); - if on_gcd_tumbling_windows != on_member_tumbling_windows { - push_shared_chains(roots, chosen, &on_gcd_tumbling_windows, pass2_candidates); - } - // Coarser groupings read a finer grouping's chain, rolled up. - push_shared_chains( - roots, - chosen, - &rollup_window_groups(chosen, chosen_rollable), - pass2_candidates, - ); -} - -/// Push the Pass 2 candidate in which each of `groups` reads its one shared -/// chain: the groups whose members have a shared size, and the error of -/// each group whose sizing fails. Nothing when no group has a shared size. -fn push_shared_chains( - roots: &[QueryRoot], - chosen: &[TumblingWindowCandidate], - groups: &[SharedTumblingWindowChain], - pass2_candidates: &mut Vec, -) { - // Each group with a size that serves every member, and that build. - let mut shared = Vec::new(); - for group in groups { - match shared_chain_build(chosen, &group.members) { - Ok(Some(build)) => shared.push((group.clone(), build)), - Ok(None) => {} - Err(error) => pass2_candidates.push(WindowCompositionPass2Candidate { - roots: Err(error), - tumbling_window_candidates: chosen.to_vec(), - shared_chains: vec![group.clone()], - }), - } - } - if !shared.is_empty() { - pass2_candidates.push(WindowCompositionPass2Candidate { - roots: in_tumbling_windows(roots, chosen, &shared), - tumbling_window_candidates: chosen.to_vec(), - shared_chains: shared.into_iter().map(|(group, _)| group).collect(), - }); - } -} - -/// The tumbling-window candidates (positions in `candidates`) that can read one -/// shared chain at the gcd of their tumbling-window widths, each group with its -/// chain: every build over one input ([`same_build_over_one_input`]) in one -/// group, whatever its tumbling-window width, on tumbling windows of the gcd of the members' -/// widths over the longest window. Finer tumbling windows than any member builds mean -/// more states and more merges; Stage 3's pricing decides whether sharing -/// pays. Groups of members with the same tumbling windows and window are left to the -/// other rules. -fn gcd_window_groups(candidates: &[TumblingWindowCandidate]) -> Vec { - let mut groups: Vec> = Vec::new(); - for (i, candidate) in candidates.iter().enumerate() { - match groups - .iter_mut() - .find(|group| same_build_over_one_input(&candidates[group[0]], candidate)) - { - Some(group) => group.push(i), - None => groups.push(vec![i]), - } - } - groups - .into_iter() - .filter_map(|members| { - let tumbling_windows = |i: usize| { - ( - candidates[i].tumbling_window_width_ms, - candidates[i].tumbling_window_count, - ) - }; - if members - .iter() - .all(|&i| tumbling_windows(i) == tumbling_windows(members[0])) - { - return None; - } - let tumbling_window_width_ms = members.iter().fold(0, |width, &i| { - asap_types::workload::greatest_common_divisor( - width, - candidates[i].tumbling_window_width_ms, - ) - }); - let longest = longest(candidates, &members); - let tumbling_window_count = - u32::try_from(candidates[longest].window_ms() / tumbling_window_width_ms).ok()?; - Some(SharedTumblingWindowChain { - members, - tumbling_window_width_ms, - tumbling_window_count, - }) - }) - .collect() -} - -/// The tumbling-window candidates (positions in `candidates`) that can read one -/// shared chain, each group with its chain: builds of one family -/// ([`same_family`], at any size) that read the same input below their -/// ranges the same way (item and weight, grouping and filter), with the -/// same output column names (each reader reads the shared chain's columns -/// by position). Their windows end at the same time, so each is the newest -/// tumbling windows of the longest. The shared tumbling windows are the group's finest member's -/// own tumbling windows: a build joins a group only when that width divides its own -/// tumbling-window width (the gcd of the widths, never finer than a chain some member -/// builds anyway), and otherwise starts its own group. Members at different -/// repeat intervals read the same tumbling windows, each at its own cadence. One chain -/// over the longest window, sized for every reader -/// ([`size_for_all_readers`]), serves them all. Groups of members with the -/// same tumbling windows and window are left to the other rules. -fn nested_window_groups(candidates: &[TumblingWindowCandidate]) -> Vec { - // Finest tumbling windows first: each build joins the first group over its input - // whose (finest) tumbling-window width divides its own, so a build whose tumbling windows no - // group's divide starts its own group instead of breaking another. - let mut by_width: Vec = (0..candidates.len()).collect(); - by_width.sort_by_key(|&i| candidates[i].tumbling_window_width_ms); - let mut groups: Vec> = Vec::new(); - for i in by_width { - let candidate = &candidates[i]; - match groups.iter_mut().find(|group| { - let finest = &candidates[group[0]]; - candidate - .tumbling_window_width_ms - .is_multiple_of(finest.tumbling_window_width_ms) - && same_build_over_one_input(finest, candidate) - }) { - Some(group) => group.push(i), - None => groups.push(vec![i]), - } - } - let mut shared = groups - .into_iter() - .filter_map(|mut members| { - let tumbling_windows = |i: usize| { - ( - candidates[i].tumbling_window_width_ms, - candidates[i].tumbling_window_count, - ) - }; - if members - .iter() - .all(|&i| tumbling_windows(i) == tumbling_windows(members[0])) - { - return None; - } - let tumbling_window_width_ms = candidates[members[0]].tumbling_window_width_ms; - let longest = longest(candidates, &members); - let tumbling_window_count = - u32::try_from(candidates[longest].window_ms() / tumbling_window_width_ms).ok()?; - members.sort_unstable(); - Some(SharedTumblingWindowChain { - members, - tumbling_window_width_ms, - tumbling_window_count, - }) - }) - .collect::>(); - // In the order of their first members, as [`gcd_window_groups`] lists - // its groups, so equal groupings compare equal. - shared.sort_by_key(|group| group.members[0]); - shared -} - -/// Whether each of `candidates` may be read rolled up from a finer -/// grouping's chain: a sketch build over a SQL relation's rows (a PromQL -/// rollup over newest tumbling windows is not offered yet) whose every reader in -/// `roots` is an estimate whose merged readout keeps a guarantee, as for -/// the rollup rule. -fn rollable_tumbling_window_candidates( - roots: &[QueryRoot], - candidates: &[TumblingWindowCandidate], -) -> Vec { - let mut nodes = Vec::new(); - let mut seen = HashSet::new(); - for node in roots.iter().flat_map(QueryRoot::operator_refs) { - collect_children_first(node, &mut seen, &mut nodes); - } - let mut readers: HashMap<*const OperatorNode, Vec<&Rc>> = HashMap::new(); - for node in &nodes { - for child in node.children() { - readers.entry(Rc::as_ptr(child)).or_default().push(node); - } - } - candidates - .iter() - .map(|candidate| match candidate.build.asap() { - Some(ASAPOp::SummaryAgg { - child: range, - family: family @ FieldDataType::Sketch(..), - .. - }) => { - range.children()[0].result_kind == OperatorResultKind::Relation - && readers - .get(&Rc::as_ptr(&candidate.build)) - .is_some_and(|readers| read_with_merged_guarantee(family, readers)) - } - _ => false, - }) - .collect() -} - -/// The tumbling-window candidates (positions in `candidates`) that can read one -/// chain of a finer grouping's tumbling windows, each group with its chain: a host, -/// and every candidate whose group keys are the host's or fewer -/// ([`rolls_up_into`]), whose window is at most the host's and whose tumbling window -/// width the host's divides. Each member merges the newest tumbling windows of its -/// window, and a coarser member then rolls the host's groups up into its -/// own (a regrouping `SummaryMerge`): `p90 per city over 20 minutes` in -/// 10 s tumbling windows serves `p90 over 5 minutes` as the newest 30 tumbling windows merged -/// across cities. A coarser member is one `rollable[i]` allows -/// ([`rollable_tumbling_window_candidates`]). Hosts are tried by most group keys, -/// then longest window, and each candidate joins one group. Only groups -/// with a coarser member are listed; the others are -/// [`nested_window_groups`]'s. -fn rollup_window_groups( - candidates: &[TumblingWindowCandidate], - rollable: &[bool], -) -> Vec { - let mut hosts: Vec = (0..candidates.len()).collect(); - hosts.sort_by_key(|&i| { - ( - std::cmp::Reverse(group_key_count(&candidates[i].build)), - std::cmp::Reverse(candidates[i].window_ms()), - i, - ) - }); - let mut grouped = vec![false; candidates.len()]; - let mut groups = Vec::new(); - for host in hosts { - if grouped[host] { - continue; - } - let coarser = |i: usize| { - group_key_count(&candidates[i].build) < group_key_count(&candidates[host].build) - }; - let reads_host = |i: usize| { - let (member, chain) = (&candidates[i], &candidates[host]); - i == host - || (!grouped[i] - && member.window_ms() <= chain.window_ms() - && member - .tumbling_window_width_ms - .is_multiple_of(chain.tumbling_window_width_ms) - && rolls_up_into(member, chain) - && (!coarser(i) || rollable[i])) - }; - let members: Vec = (0..candidates.len()).filter(|&i| reads_host(i)).collect(); - if !members.iter().any(|&i| coarser(i)) { - continue; - } - for &i in &members { - grouped[i] = true; - } - groups.push(SharedTumblingWindowChain { - members, - tumbling_window_width_ms: candidates[host].tumbling_window_width_ms, - tumbling_window_count: candidates[host].tumbling_window_count, - }); - } - groups.sort_by_key(|group| group.members[0]); - groups -} - -/// The number of group keys of a build grouped by listed keys; 0 for any -/// other build. -fn group_key_count(build: &OperatorNode) -> usize { - match build.asap() { - Some(ASAPOp::SummaryAgg { - reduction: Reduction::Reduce(by), - .. - }) if !by.is_without() => by.len(), - _ => 0, - } -} - -/// Whether `member`'s build is `host`'s ([`same_build_over_one_input`]), -/// or the same but grouped by fewer of `host`'s keys, of a family whose -/// states merge across groups ([`FamilyCapability::merges_groups`]), so a -/// rollup of `host`'s states answers it. -fn rolls_up_into(member: &TumblingWindowCandidate, host: &TumblingWindowCandidate) -> bool { - if same_build_over_one_input(member, host) { - return true; - } - let ( - Some(ASAPOp::SummaryAgg { - child: range, - family, - input, - reduction: Reduction::Reduce(by), - grouping, - filter, - }), - Some(ASAPOp::SummaryAgg { - child: host_range, - family: host_family, - input: host_input, - reduction: Reduction::Reduce(host_by), - grouping: host_grouping, - filter: host_filter, - }), - ) = (member.build.asap(), host.build.asap()) - else { - return false; - }; - let (below, host_below) = (range.children()[0], host_range.children()[0]); - // Both ranges read the same rows below, so a key names one column. - !by.is_without() - && !host_by.is_without() - && by.len() < host_by.len() - && by.keys().iter().all(|key| host_by.keys().contains(key)) - && FamilyCapability::merges_groups(family) - && same_build_family(family, host_family) - && (Rc::ptr_eq(below, host_below) || below == host_below) - && input == host_input - && grouping == host_grouping - && filter == host_filter -} - -/// Whether two tumbling-window candidates build one family (at any size) over the -/// same input below their ranges, read the same way, with the same output -/// column names. -fn same_build_over_one_input(a: &TumblingWindowCandidate, b: &TumblingWindowCandidate) -> bool { - let ( - Some(ASAPOp::SummaryAgg { - child: range, - family, - input, - reduction, - grouping, - filter, - }), - Some(ASAPOp::SummaryAgg { - child: range_b, - family: family_b, - input: input_b, - reduction: reduction_b, - grouping: grouping_b, - filter: filter_b, - }), - ) = (a.build.asap(), b.build.asap()) - else { - return false; - }; - let (below, below_b) = (range.children()[0], range_b.children()[0]); - let names = |build: &OperatorNode| -> Vec<(String, Option)> { - build - .schema - .fields - .iter() - .map(|f| (f.name.clone(), f.table.clone())) - .collect() - }; - names(&a.build) == names(&b.build) - && same_build_family(family, family_b) - && (Rc::ptr_eq(below, below_b) || below == below_b) - && input == input_b - && reduction == reduction_b - && grouping == grouping_b - && filter == filter_b -} - -/// The member of `group` (positions in `candidates`) with the longest -/// window. -fn longest(candidates: &[TumblingWindowCandidate], group: &[usize]) -> usize { - *group - .iter() - .max_by_key(|&&i| candidates[i].window_ms()) - .expect("a group has members") -} - -/// The build one chain shared by `group` (positions in `candidates`) is -/// made of: the longest member's, sized for every member -/// ([`sized_for_all`]); `None` when their sizes have no shared size. -fn shared_chain_build( - candidates: &[TumblingWindowCandidate], - group: &[usize], -) -> Result>, CseError> { - // Of the longest windows, the finest grouping: a rollup group's host. - let host = *group - .iter() - .max_by_key(|&&i| { - ( - candidates[i].window_ms(), - group_key_count(&candidates[i].build), - ) - }) - .expect("a group has members"); - let members: Vec<&Rc> = std::iter::once(host) - .chain(group.iter().copied().filter(|&i| i != host)) - .map(|i| &candidates[i].build) - .collect(); - sized_for_all(&members) -} - -/// `roots` with the build of each of `candidates` built as its tumbling -/// tumbling windows, and every guarantee composed again. The members of each -/// [`SharedTumblingWindowChain`] (with its [`shared_chain_build`]) read that one -/// chain: each merges the newest tumbling windows covering its own window. -fn in_tumbling_windows( - roots: &[QueryRoot], - candidates: &[TumblingWindowCandidate], - shared_chains: &[(SharedTumblingWindowChain, Rc)], -) -> Result, CseError> { - let mut replaced: HashMap<*const OperatorNode, Rc> = HashMap::new(); - for (shared, build) in shared_chains { - let chain = tumbling_window_chain(&TumblingWindowCandidate { - build: Rc::clone(build), - tumbling_window_width_ms: shared.tumbling_window_width_ms, - tumbling_window_count: shared.tumbling_window_count, - })?; - for &i in &shared.members { - let newest = u32::try_from(candidates[i].window_ms() / shared.tumbling_window_width_ms) - .expect("a member's window is at most the chain's"); - let merge = Rc::new(OperatorNode::new(Operator::ASAP(ASAPOp::SummaryMerge { - children: vec![Rc::clone(&chain)], - newest_tumbling_windows: (newest < shared.tumbling_window_count).then_some(newest), - group_by: None, - }))?); - // A coarser member rolls the chain's groups up into its own. - let member = &candidates[i].build; - let read = if group_key_count(member) == group_key_count(build) { - merge - } else { - regrouped(member, &merge).ok_or(CseError::Unsupported( - "a coarser tumbling-window build whose keys the shared chain does not hold", - ))? - }; - replaced.insert(Rc::as_ptr(member), read); - } - } - for candidate in candidates { - if let std::collections::hash_map::Entry::Vacant(entry) = - replaced.entry(Rc::as_ptr(&candidate.build)) - { - entry.insert(Rc::new(OperatorNode::new(Operator::ASAP( - ASAPOp::SummaryMerge { - children: vec![tumbling_window_chain(candidate)?], - newest_tumbling_windows: None, - group_by: None, - }, - ))?)); - } - } - let mut rebuilt: HashMap<*const OperatorNode, Rc> = HashMap::new(); - let mut out = Vec::new(); - for root in roots { - out.push(match root { - QueryRoot::Operator(node) => { - QueryRoot::Operator(rebuild(node, &replaced, &mut rebuilt)?) - } - QueryRoot::Scalar(expr) => { - let mut nodes = HashMap::new(); - for node in expr.operator_refs() { - nodes.insert(Rc::as_ptr(node), rebuild(node, &replaced, &mut rebuilt)?); - } - QueryRoot::Scalar( - expr.map_operator_refs(&mut |node| nodes[&Rc::as_ptr(node)].clone()), - ) - } - }); - } - Ok(recompose_guarantees(&out)?) -} - -/// `roots` with the `SummaryMerge` `merge` (over one [`ASAPOp::SummaryTumblingWindowChain`] -/// of `roots`) reading the newest tumbling windows of another chain over the same -/// input: `tumbling_window_count` tumbling windows of `tumbling_window_width_ms` of family `family` (a size -/// that serves every reader of the chain). The merge reads the newest -/// tumbling windows covering its own window (its newest tumbling windows, or its whole chain), -/// which `tumbling_window_width_ms` divides and the new chain covers; every guarantee -/// is composed again. -pub(crate) fn with_tumbling_window_chain( - roots: &[QueryRoot], - merge: &Rc, - family: FieldDataType, - tumbling_window_width_ms: u64, - tumbling_window_count: u32, -) -> Result, CseError> { - let Some(ASAPOp::SummaryMerge { - children, - newest_tumbling_windows: own_newest, - .. - }) = merge.asap() - else { - unreachable!("a merge over a tumbling-window chain"); - }; - let own = &children[0]; - let Some(ASAPOp::SummaryTumblingWindowChain { - child, - input, - reduction, - grouping, - filter, - tumbling_window_width_ms: own_width, - tumbling_window_count: own_count, - .. - }) = own.asap() - else { - unreachable!("a merge over a tumbling-window chain"); - }; - // The merge's window: its newest tumbling windows, or the whole chain. - let window_ms = own_width * u64::from(own_newest.unwrap_or(*own_count)); - let newest = u32::try_from(window_ms / tumbling_window_width_ms) - .expect("a reader's window is at most the chain's"); - let mut chain = OperatorNode::new(Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { - child: Rc::clone(child), - family, - input: input.clone(), - reduction: reduction.clone(), - grouping: grouping.clone(), - filter: filter.clone(), - tumbling_window_width_ms, - tumbling_window_count, - }))?; - for (field, kept) in chain.schema.fields.iter_mut().zip(&own.schema.fields) { - field.name = kept.name.clone(); - field.table = kept.table.clone(); - } - let read = OperatorNode::new(Operator::ASAP(ASAPOp::SummaryMerge { - children: vec![Rc::new(chain)], - newest_tumbling_windows: (newest < tumbling_window_count).then_some(newest), - group_by: None, - }))?; - let replaced = HashMap::from([(Rc::as_ptr(merge), Rc::new(read))]); - let mut rebuilt = HashMap::new(); - let mut out = Vec::new(); - for root in roots { - out.push(match root { - QueryRoot::Operator(node) => { - QueryRoot::Operator(rebuild(node, &replaced, &mut rebuilt)?) - } - QueryRoot::Scalar(expr) => { - let mut nodes = HashMap::new(); - for node in expr.operator_refs() { - nodes.insert(Rc::as_ptr(node), rebuild(node, &replaced, &mut rebuilt)?); - } - QueryRoot::Scalar( - expr.map_operator_refs(&mut |node| nodes[&Rc::as_ptr(node)].clone()), - ) - } - }); - } - Ok(recompose_guarantees(&out)?) -} - -/// `roots` with the summary build `build` (a `SummaryAgg` of `roots`) -/// built at family `family` instead (the same algorithm at another size), -/// its output names kept; every guarantee is composed again. -pub(crate) fn with_resized_build( - roots: &[QueryRoot], - build: &Rc, - family: FieldDataType, -) -> Result, CseError> { - let Some(ASAPOp::SummaryAgg { - child, - input, - reduction, - grouping, - filter, - .. - }) = build.asap() - else { - unreachable!("a summary build"); - }; - let mut resized = OperatorNode::new(Operator::ASAP(ASAPOp::SummaryAgg { - child: Rc::clone(child), - family, - input: input.clone(), - reduction: reduction.clone(), - grouping: grouping.clone(), - filter: filter.clone(), - }))?; - for (field, kept) in resized.schema.fields.iter_mut().zip(&build.schema.fields) { - field.name = kept.name.clone(); - field.table = kept.table.clone(); - } - let replaced = HashMap::from([(Rc::as_ptr(build), Rc::new(resized))]); - let mut rebuilt = HashMap::new(); - let mut out = Vec::new(); - for root in roots { - out.push(match root { - QueryRoot::Operator(node) => { - QueryRoot::Operator(rebuild(node, &replaced, &mut rebuilt)?) - } - QueryRoot::Scalar(expr) => { - let mut nodes = HashMap::new(); - for node in expr.operator_refs() { - nodes.insert(Rc::as_ptr(node), rebuild(node, &replaced, &mut rebuilt)?); - } - QueryRoot::Scalar( - expr.map_operator_refs(&mut |node| nodes[&Rc::as_ptr(node)].clone()), - ) - } - }); - } - Ok(recompose_guarantees(&out)?) -} - -/// The [`ASAPOp::SummaryTumblingWindowChain`] of `candidate`'s tumbling windows, with the build's -/// output names. -fn tumbling_window_chain( - candidate: &TumblingWindowCandidate, -) -> Result, CseError> { - let Some(ASAPOp::SummaryAgg { - child: range, - family, - input, - reduction, - grouping, - filter, - }) = candidate.build.asap() - else { - unreachable!("a tumbling-window candidate's build is a SummaryAgg"); - }; - let mut chain = OperatorNode::new(Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { - child: Rc::clone(range.children()[0]), - family: family.clone(), - input: input.clone(), - reduction: reduction.clone(), - grouping: grouping.clone(), - filter: filter.clone(), - tumbling_window_width_ms: candidate.tumbling_window_width_ms, - tumbling_window_count: candidate.tumbling_window_count, - }))?; - for (field, kept) in chain - .schema - .fields - .iter_mut() - .zip(&candidate.build.schema.fields) - { - field.name = kept.name.clone(); - field.table = kept.table.clone(); - } - Ok(Rc::new(chain)) -} - -/// `node` with every replaced build below it swapped for its shared build; -/// the same `Rc` when nothing below changed. -fn rebuild( - node: &Rc, - replaced: &HashMap<*const OperatorNode, Rc>, - rebuilt: &mut HashMap<*const OperatorNode, Rc>, -) -> Result, CseError> { - if let Some(done) = rebuilt.get(&Rc::as_ptr(node)) { - return Ok(Rc::clone(done)); - } - let out = match replaced.get(&Rc::as_ptr(node)) { - Some(build) => Rc::clone(build), - None => { - let mut changed = false; - let mut children = HashMap::new(); - for child in node.children() { - let new = rebuild(child, replaced, rebuilt)?; - changed |= !Rc::ptr_eq(&new, child); - children.insert(Rc::as_ptr(child), new); - } - if changed { - Rc::new(node.with_new_children(|child| children[&Rc::as_ptr(child)].clone())?) - } else { - Rc::clone(node) - } - } - }; - rebuilt.insert(Rc::as_ptr(node), Rc::clone(&out)); - Ok(out) -} - -/// One coarse build the rollup rule can answer from a finer one: the -/// coarse `SummaryAgg`, the finer node it reads, and the node that replaces -/// the coarse value or state. -#[derive(Debug, Clone)] -pub struct RollupCandidate { - /// The coarse build (`SummaryAgg`). An exact accumulator's every - /// reader is a `FinalizeExactAccumulator`, which the rollup replaces; a - /// sketch's every reader is an estimate its family answers, and the - /// rollup replaces the build itself. - pub coarse: Rc, - /// For an exact accumulator, the finer build's - /// `FinalizeExactAccumulator`, whose values the rollup reads; for a - /// sketch, the finest build of the same algorithm over the same rows - /// (a `SummaryAgg`), whose states the rollup merges. Both are already - /// in the candidate. - pub finer: Rc, - /// For an exact accumulator, the finalized exact accumulator over - /// `finer`'s values, grouped by the coarse keys, with the coarse value's - /// column names; for a sketch, the regrouping `SummaryMerge` of - /// `finer`'s states by the coarse keys, at `finer`'s size (a Pass 2 - /// candidate resizes `finer` for every reader). - pub rollup: Rc, -} - -/// A Pass 2 candidate of the rollup rule: the candidate's roots with the -/// coarse build of each chosen rollup candidate answered from its finer -/// build, and every guarantee composed again; the error when that fails. -#[derive(Debug)] -pub struct RollupPass2Candidate { - /// The candidate's roots with those builds rolled up and every guarantee - /// composed again; `Err` when the rebuilt DAG does not derive or - /// compose, which drops only this candidate. - pub roots: Result, CseError>, -} - -/// The most rollup or top-k reuse candidates one candidate may have: every -/// non-empty subset is a Pass 2 candidate, about a million at 20. -pub(crate) const MAX_REUSE_CANDIDATES: usize = 20; - -/// The exact accumulator that merges finalized values of `family` across -/// disjoint groups, by the family-capability table: a family whose states -/// merge over disjoint (or overlapping) selections into an exact state. -/// `None` for a family without such a merge: counter states merge only -/// consecutive windows, and sketch states have no regrouping merge in the IR. -pub(crate) fn rollup_kind(family: &FieldDataType) -> Option { - let FieldDataType::ExactAggregate(kind, _) = family else { - return None; - }; - // An exact accumulator's merged state is exact; a counter state merges - // only consecutive windows. - if FamilyCapability::of(family).merge_inputs? == MergeInputRule::DisjointInTime { - return None; - } - match kind { - // Counts add up. - ExactKind::Sum | ExactKind::Count => Some(ExactKind::Sum), - ExactKind::Min => Some(ExactKind::Min), - ExactKind::Max => Some(ExactKind::Max), - ExactKind::Rate | ExactKind::IRate | ExactKind::Increase => None, - } -} - -/// The rollup candidates of the candidate with query roots `roots`, children -/// first: each coarse exact accumulator build read only through its -/// finalized value, with its coarsest finer build of the same family over -/// the same rows that the candidate finalizes; then each coarse sketch build -/// read only by estimates, with the finest build of its algorithm over the -/// same rows whose keys include its own (`sketch_rollup_candidates`). -#[cfg(any(test, feature = "test-support"))] -pub fn rollup_candidates(roots: &[QueryRoot]) -> Vec { - rollup_candidates_of(roots, None) -} - -/// The rollup candidate of coarse build `coarse` among -/// [`rollup_candidates`]`(roots)`, the first when several, computed for that -/// build alone (each coarse build's candidate depends only on `roots`, not -/// on the other coarse builds). -pub(crate) fn rollup_candidate_for( - roots: &[QueryRoot], - coarse: *const OperatorNode, -) -> Option { - rollup_candidates_of(roots, Some(coarse)).into_iter().next() -} - -/// [`rollup_candidates`], over the coarse builds that are `only` when set. -fn rollup_candidates_of( - roots: &[QueryRoot], - only: Option<*const OperatorNode>, -) -> Vec { - let mut nodes = Vec::new(); - let mut seen = std::collections::HashSet::new(); - for node in roots.iter().flat_map(QueryRoot::operator_refs) { - collect_children_first(node, &mut seen, &mut nodes); - } - let mut readers: HashMap<*const OperatorNode, Vec<&Rc>> = HashMap::new(); - for node in &nodes { - for child in node.children() { - readers.entry(Rc::as_ptr(child)).or_default().push(node); - } - } - let finalized = |build: &Rc| -> Vec<&Rc> { - readers - .get(&Rc::as_ptr(build)) - .map_or_else(Vec::new, |readers| { - readers - .iter() - .filter(|r| matches!(r.asap(), Some(ASAPOp::FinalizeExactAccumulator { .. }))) - .copied() - .collect() - }) - }; - let builds: Vec<&Rc> = nodes.iter().filter(|n| grouped_exact(n)).collect(); - let wanted = |coarse: &Rc| only.is_none_or(|only| Rc::as_ptr(coarse) == only); - let mut candidates = Vec::new(); - for &coarse in builds.iter().filter(|coarse| wanted(coarse)) { - let finals = finalized(coarse); - // Every reader of the coarse state reads its finalized value. - if finals.is_empty() || finals.len() != readers[&Rc::as_ptr(coarse)].len() { - continue; - } - let mut best: Option<(usize, RollupCandidate)> = None; - for &fine in &builds { - let Some(keys) = finer_keys(coarse, fine) else { - continue; - }; - if best.as_ref().is_some_and(|(k, _)| *k <= keys) { - continue; - } - let Some(&finer) = finalized(fine).first() else { - continue; - }; - if let Some(rollup) = rolled_up(coarse, finer, finals[0]) { - best = Some(( - keys, - RollupCandidate { - coarse: Rc::clone(coarse), - finer: Rc::clone(finer), - rollup, - }, - )); - } - } - candidates.extend(best.map(|(_, candidate)| candidate)); - } - candidates.extend(sketch_rollup_candidates(&nodes, &readers, only)); - candidates -} - -/// The sketch rollup candidates among `nodes` (children first), whose -/// readers are `readers`: each per-group sketch build grouped by listed keys -/// and read only by estimates its family answers, answered from the finest -/// build ([`finer_keys`] with the most keys, and no build finer than it) of -/// the same algorithm (at any size), update and filter over the same rows, -/// whose family's states merge into a state with a guarantee -/// ([`regroupable`]). Every coarse build of one finest build is one -/// choice in [`rollup_pass2_candidates`], so the choices stay few however -/// many groupings a workload asks for. -fn sketch_rollup_candidates( - nodes: &[Rc], - readers: &HashMap<*const OperatorNode, Vec<&Rc>>, - only: Option<*const OperatorNode>, -) -> Vec { - let builds: Vec<&Rc> = nodes.iter().filter(|n| regroupable(n)).collect(); - // The finer build with the most keys; among several (groupings that - // are not nested), the one whose sorted key names come first, so the - // choice does not depend on query order. - let key_names = |build: &OperatorNode| -> Vec { - let mut names: Vec = build - .schema - .fields - .iter() - .filter(|field| field.is_plain()) - .map(column_name) - .collect(); - names.sort(); - names - }; - let finest = |coarse: &OperatorNode| { - builds - .iter() - .filter_map(|&fine| Some((finer_keys(coarse, fine)?, fine))) - .min_by(|(keys, fine), (other_keys, other)| { - other_keys - .cmp(keys) - .then_with(|| key_names(fine).cmp(&key_names(other))) - }) - .map(|(_, fine)| fine) - }; - let mut candidates = Vec::new(); - for &coarse in builds - .iter() - .filter(|coarse| only.is_none_or(|only| Rc::as_ptr(coarse) == only)) - { - let Some(ASAPOp::SummaryAgg { family, .. }) = coarse.asap() else { - continue; - }; - if !readers - .get(&Rc::as_ptr(coarse)) - .is_some_and(|readers| read_with_merged_guarantee(family, readers)) - { - continue; - } - // A build finer than the one with the most keys would have more. - // Over more rows, the finer build's states are first filtered by - // the coarse build's extra conjuncts (a compensating filter). - let rollup = match finest(coarse) { - Some(fine) => regrouped(coarse, fine).map(|rollup| (fine, rollup)), - None => builds - .iter() - .filter_map(|&fine| Some((filtered_finer(coarse, fine)?, fine))) - // The same choice as `finest`'s, whatever the query order. - .min_by(|((keys, _), fine), ((other_keys, _), other)| { - other_keys - .cmp(keys) - .then_with(|| key_names(fine).cmp(&key_names(other))) - }) - .and_then(|((_, pred), fine)| { - let kept = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Filter { - pred, - child: Rc::clone(fine), - })) - .ok()?; - Some((fine, regrouped(coarse, &kept)?)) - }), - }; - if let Some((fine, rollup)) = rollup { - candidates.push(RollupCandidate { - coarse: Rc::clone(coarse), - finer: Rc::clone(fine), - rollup, - }); - } - } - candidates -} - -/// Whether every one of `readers`, the readers of a build of `family`, is -/// an estimate whose merged readout keeps a guarantee: not a heap's top-k -/// or a UnivMon's entropy. -fn read_with_merged_guarantee(family: &FieldDataType, readers: &[&Rc]) -> bool { - let capability = FamilyCapability::of(family); - readers.iter().all(|reader| match reader.asap() { - Some(ASAPOp::SummaryEstimate { query, .. }) => { - capability.answers(query) - && matches!( - capability.merged_error(query), - Some(MergedError::Exact | MergedError::AsIfBuiltOverUnion) - ) - } - _ => false, - }) -} - -/// Whether `node` is a per-group sketch build grouped by listed keys whose -/// states a rollup may merge across groups -/// ([`FamilyCapability::merges_groups`]). -fn regroupable(node: &OperatorNode) -> bool { - matches!( - node.asap(), - Some(ASAPOp::SummaryAgg { - family: family @ FieldDataType::Sketch(..), - reduction: Reduction::Reduce(by), - .. - }) if !by.is_without() && FamilyCapability::merges_groups(family) - ) -} - -/// The regrouping `SummaryMerge` of `fine`'s states by `coarse`'s keys, in -/// `coarse`'s order and with its column names; `None` when a key is not in -/// `fine`'s state. -fn regrouped(coarse: &OperatorNode, fine: &Rc) -> Option> { - let Some(ASAPOp::SummaryAgg { - reduction: Reduction::Reduce(by), - .. - }) = coarse.asap() - else { - return None; - }; - let keys = (0..by.keys().len()) - .map(|key| { - let name = column_name(&coarse.schema.fields[key]); - fine.schema - .fields - .iter() - .position(|field| field.is_plain() && column_name(field) == name) - }) - .collect::>>()?; - regroup_merge(fine, keys, coarse).ok() -} - -/// `SummaryMerge { [state], group_by: keys }`, its columns named as -/// `named`'s (the coarse build it stands for). -fn regroup_merge( - state: &Rc, - keys: Vec, - named: &OperatorNode, -) -> Result, asap_types::ir::SchemaDerivationError> { - let merge = OperatorNode::new(Operator::ASAP(ASAPOp::SummaryMerge { - children: vec![Rc::clone(state)], - group_by: Some(keys), - newest_tumbling_windows: None, - }))?; - let mut schema = merge.schema.clone(); - for (field, coarse) in schema.fields.iter_mut().zip(&named.schema.fields) { - field.name = coarse.name.clone(); - field.table = coarse.table.clone(); - } - Ok(Rc::new(OperatorNode::with_schema(merge.operator, schema))) -} - -/// Whether `node` is an exact accumulator build grouped by listed keys whose -/// family merges across groups ([`rollup_kind`]). -fn grouped_exact(node: &OperatorNode) -> bool { - matches!( - node.asap(), - Some(ASAPOp::SummaryAgg { - family, - reduction: Reduction::Reduce(by), - grouping: GroupingStrategy::PerSubpopulationInstance, - .. - }) if !by.is_without() && rollup_kind(family).is_some() - ) -} - -/// The number of group keys of `fine` when it can answer `coarse`: the same -/// family (a sketch's algorithm at any size), update and filter over the -/// same rows, and group keys (by column) that strictly include the coarse -/// ones. -fn finer_keys(coarse: &OperatorNode, fine: &OperatorNode) -> Option { - let ( - Some(ASAPOp::SummaryAgg { - child, - family, - input, - reduction: Reduction::Reduce(by), - filter, - .. - }), - Some(ASAPOp::SummaryAgg { - child: fine_child, - family: fine_family, - input: fine_input, - reduction: Reduction::Reduce(fine_by), - filter: fine_filter, - .. - }), - ) = (coarse.asap(), fine.asap()) - else { - return None; - }; - let names = |node: &OperatorNode, keys: &[usize]| -> Option> { - keys.iter() - .map(|&k| node.schema.fields.get(k).map(column_name)) - .collect() - }; - let coarse_keys = names(child, by.keys())?; - let fine_keys = names(fine_child, fine_by.keys())?; - let same_filter = match (filter, fine_filter) { - (None, None) => true, - // A filter indexes its input's columns: equal only over equal rows. - (a, b) => a == b && (Rc::ptr_eq(child, fine_child) || child == fine_child), - }; - (same_build_family(family, fine_family) - && input == fine_input - && same_filter - && fine_keys.len() > coarse_keys.len() - && coarse_keys.iter().all(|key| fine_keys.contains(key)) - && same_rows(child, fine_child)) - .then_some(fine_keys.len()) -} - -/// The number of group keys of `fine` and the compensating filter over its -/// states when `fine` answers `coarse` from more rows: the same family, -/// update and no build filter, over rows that hold `coarse`'s, whose extra -/// conjuncts read only `fine`'s keys ([`match_finer`]), with group keys that -/// include the coarse ones. `None` over the same rows ([`finer_keys`]). -fn filtered_finer(coarse: &OperatorNode, fine: &OperatorNode) -> Option<(usize, Predicate)> { - let ( - Some(ASAPOp::SummaryAgg { - child, - family, - input, - reduction: Reduction::Reduce(by), - filter: None, - .. - }), - Some(ASAPOp::SummaryAgg { - child: fine_child, - family: fine_family, - input: fine_input, - reduction: Reduction::Reduce(fine_by), - filter: None, - .. - }), - ) = (coarse.asap(), fine.asap()) - else { - return None; - }; - if by.is_without() - || fine_by.is_without() - || !same_build_family(family, fine_family) - || input != fine_input - { - return None; - } - let matched = - crate::pass2::finer_aggregate::match_finer(child, by.keys(), fine_child, fine_by.keys())?; - // The filter reads the finer keys as the state's columns of those names. - let state_columns = fine_by - .keys() - .iter() - .map(|&key| { - // A state's key column keeps the input column's name, not - // always its table qualifier. - let name = &fine_child.schema.fields.get(key)?.name; - fine.schema - .fields - .iter() - .position(|field| field.is_plain() && field.name == *name) - }) - .collect::>>()?; - let pred = matched.compensating_filter?; - Some(( - fine_by.keys().len(), - Predicate(pred.0.map_columns(&mut |key| state_columns[key])), - )) -} - -type ColumnName = (Option, String); - -fn column_name(field: &asap_types::ir::schema::Field) -> ColumnName { - (field.table.clone(), field.name.clone()) -} - -/// Whether `a` and `b` output the same rows: the same or equal nodes, or -/// time-series scans of one metric without predicates (projecting perhaps -/// other labels) under the same time ranges and shifts. -fn same_rows(a: &Rc, b: &Rc) -> bool { - if Rc::ptr_eq(a, b) || a == b { - return true; - } - match (a.non_asap(), b.non_asap()) { - ( - Some(NonASAPOp::Scan { - source: source @ Source::TimeSeries { .. }, - predicates, - .. - }), - Some(NonASAPOp::Scan { - source: other, - predicates: other_predicates, - .. - }), - ) => source == other && predicates.is_empty() && other_predicates.is_empty(), - ( - Some(NonASAPOp::TimeRange { range, kind, child }), - Some(NonASAPOp::TimeRange { - range: other_range, - kind: other_kind, - child: other, - }), - ) => range == other_range && kind == other_kind && same_rows(child, other), - ( - Some(NonASAPOp::TimeShift { shift, child }), - Some(NonASAPOp::TimeShift { - shift: other_shift, - child: other, - }), - ) => shift == other_shift && same_rows(child, other), - ( - Some(NonASAPOp::Aggregate { - reduction: Reduction::PerEntity, - measures, - filters, - having: None, - child, - .. - }), - Some(NonASAPOp::Aggregate { - reduction: Reduction::PerEntity, - measures: other_measures, - filters: other_filters, - having: None, - child: other, - .. - }), - ) => { - // These reductions preserve series identity and read the implicit - // sample value. Extra projected labels do not change their values. - // Explicit column indices and filtered measures need remapping and - // are deliberately excluded. - measures == other_measures - && matches!( - measures.as_slice(), - [AggIntent::Avg { col: None } - | AggIntent::Sum { col: None } - | AggIntent::Min { col: None } - | AggIntent::Max { col: None }] - ) - && filters.iter().all(Option::is_none) - && other_filters.iter().all(Option::is_none) - && same_rows(child, other) - } - _ => false, - } -} - -/// The finalized value of `coarse` (`coarse_value`) as an exact -/// accumulator over `finer` (the finer build's finalized values), grouped by -/// the coarse keys, finalized, with `coarse_value`'s column names; `None` -/// when it cannot be built, a non-count finer value is not Float64 (the -/// supported accumulator update type), or it differs from -/// `coarse_value` in type. -fn rolled_up( - coarse: &OperatorNode, - finer: &Rc, - coarse_value: &OperatorNode, -) -> Option> { - let Some(ASAPOp::SummaryAgg { - child, - family, - reduction, - .. - }) = coarse.asap() - else { - return None; - }; - let Some(ASAPOp::FinalizeExactAccumulator { child: fine }) = finer.asap() else { - return None; - }; - let Some(ASAPOp::SummaryAgg { - reduction: Reduction::Reduce(fine_by), - .. - }) = fine.asap() - else { - return None; - }; - // A grouped accumulator outputs its keys, then its value. - let value = fine_by.len(); - // Merge count states directly: converting finalized Int64 counts through - // a Float64 sum would lose integer precision above 2^53. - if matches!(family, FieldDataType::ExactAggregate(ExactKind::Count, _)) { - let merged = regrouped(coarse, fine)?; - let finalized = - OperatorNode::new_shared(Operator::ASAP(ASAPOp::FinalizeExactAccumulator { - child: merged, - })) - .ok()?; - if finalized.schema.fields.len() != coarse_value.schema.fields.len() - || finalized - .schema - .fields - .iter() - .zip(&coarse_value.schema.fields) - .any(|(a, b)| a.dtype != b.dtype || a.nullable != b.nullable) - || finalized.schema.unique_keys != coarse_value.schema.unique_keys - { - return None; - } - // Finalization preserves the count's type; only the reader's SQL alias - // differs from the canonical state readout name. - return Some(Rc::new(OperatorNode::with_schema( - finalized.operator.clone(), - coarse_value.schema.clone(), - ))); - } - if finer.schema.fields.get(value).map(|f| &f.dtype) - != Some(&FieldDataType::Plain( - asap_types::ir::schema::DataType::Float64, - )) - { - return None; - } - let regrouped = reduction.partitions_over(&child.schema, &finer.schema)?; - if regrouped - .group_keys() - .is_some_and(|keys| keys.iter().any(|&k| k >= value)) - { - return None; - } - let kind = rollup_kind(family)?; - // An exact accumulator over the finer value column: what Pass 1 builds - // for a Sum, Min or Max of that column (no item, signed weights). - let weight = finer.schema.fields[value].column_ref(); - resolve_column_ref(&weight, &finer.schema).ok()?; - let state = OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryAgg { - child: Rc::clone(finer), - input: SummaryUpdate { - item: None, - weight: SummaryInputExpr::Column(weight), - weight_domain: WeightDomain::UnknownOrSigned, - }, - family: FieldDataType::exact(kind), - reduction: regrouped, - grouping: GroupingStrategy::default(), - filter: None, - })) - .ok()?; - let mut finalized = - OperatorNode::new_shared(Operator::ASAP(ASAPOp::FinalizeExactAccumulator { - child: state, - })) - .ok()?; - let types = |node: &OperatorNode| -> Vec { - node.schema.fields.iter().map(|f| f.dtype.clone()).collect() - }; - if types(&finalized) != types(coarse_value) { - return None; - } - // A count summed is named after the sum: rename to the coarse columns. - if finalized.schema.fields != coarse_value.schema.fields { - let cols = coarse_value - .schema - .fields - .iter() - .enumerate() - .map(|(column, field)| asap_types::ir::ProjectItem { - alias: Some(field.name.clone()), - expr: ScalarExpr::Column(column), - }) - .collect(); - finalized = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Project { - cols, - qualifier: None, - child: finalized, - })) - .ok()?; - } - Some(finalized) -} - -/// The independent choices among rollup candidates `candidates` -/// ([`rollup_candidates`]), in order: each exact accumulator's alone, and -/// every sketch candidate of one finest build together, since they share -/// that build's size. -pub(crate) fn rollup_choices(candidates: &[RollupCandidate]) -> Vec> { - let mut choices: Vec> = Vec::new(); - for candidate in candidates { - let sketch = is_sketch_build(&candidate.finer); - match choices.iter_mut().find(|choice| { - sketch - && is_sketch_build(&choice[0].finer) - && Rc::ptr_eq(&choice[0].finer, &candidate.finer) - }) { - Some(choice) => choice.push(candidate.clone()), - None => choices.push(vec![candidate.clone()]), - } - } - choices -} - -fn is_sketch_build(node: &OperatorNode) -> bool { - matches!( - node.asap(), - Some(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(..), - .. - }) - ) -} - -/// Every Pass 2 candidate of the rollup rule for the candidate with query -/// roots `roots` and rollup candidates `candidates` ([`rollup_candidates`]): -/// one per non-empty subset of their choices (`rollup_choices`), in -/// binary-counter order (the first the lowest bit). A chosen sketch choice -/// resizes its finest build for every reader, its own and each coarse -/// build's (`sized_for_all`: the strictest consumer's size), and each -/// coarse build becomes the regrouping merge of the resized build's -/// states. An error only for more than `MAX_REUSE_CANDIDATES` choices; a -/// Pass 2 candidate whose composition fails carries its own error. -pub fn rollup_pass2_candidates( - roots: &[QueryRoot], - candidates: &[RollupCandidate], -) -> Result, CseError> { - Ok(subsets(&rollup_choices(candidates), "rollup")? - .into_iter() - .map(|choices| { - let values: HashMap<*const OperatorNode, &RollupCandidate> = choices - .iter() - .flatten() - .filter(|candidate| !is_sketch_build(&candidate.finer)) - .map(|candidate| (Rc::as_ptr(&candidate.coarse), candidate)) - .collect(); - // Every finalized value of a chosen coarse exact build is - // replaced. - let mut replaced: HashMap<*const OperatorNode, Rc> = roots - .iter() - .flat_map(QueryRoot::operator_refs) - .flat_map(OperatorNode::reachable) - .filter_map(|node| match node.asap() { - Some(ASAPOp::FinalizeExactAccumulator { child }) => { - let candidate = values.get(&Rc::as_ptr(child))?; - Some((Rc::as_ptr(&node), Rc::clone(&candidate.rollup))) - } - _ => None, - }) - .collect(); - let roots = sketch_rollups(&choices).and_then(|sketches| { - replaced.extend(sketches); - replace_all(roots, &replaced) - }); - RollupPass2Candidate { roots } - }) - .collect()) -} - -/// The replacements of the sketch rollup choices among `choices` -/// ([`rollup_choices`]): each finest build by the build sized for it and -/// every coarse build it serves, and each coarse build by the regrouping -/// merge of that build's states. An error when a finest build and its -/// coarse builds have no shared size, or a node does not derive. -fn sketch_rollups( - choices: &[Vec], -) -> Result)>, CseError> { - let unsupported = || { - CseError::Unsupported("a sketch rollup whose finest and coarse builds have no shared size") - }; - // A finest build can also be the coarse build of another choice, read - // through a compensating filter. Each build is replaced once: as a - // finest build, sized for its readers. - let finest: HashSet<*const OperatorNode> = choices - .iter() - .filter(|choice| is_sketch_build(&choice[0].finer)) - .map(|choice| Rc::as_ptr(&choice[0].finer)) - .collect(); - let mut replaced = Vec::new(); - for choice in choices { - let fine = &choice[0].finer; - if !is_sketch_build(fine) { - continue; - } - let mut members = vec![fine]; - members.extend(choice.iter().map(|candidate| &candidate.coarse)); - let sized = sized_for_all(&members)?.ok_or_else(unsupported)?; - replaced.push((Rc::as_ptr(fine), Rc::clone(&sized))); - for candidate in choice { - if finest.contains(&Rc::as_ptr(&candidate.coarse)) { - continue; - } - let Some(ASAPOp::SummaryMerge { - group_by: Some(keys), - .. - }) = candidate.rollup.asap() - else { - unreachable!("a sketch rollup candidate's rollup is a regrouping merge"); - }; - // A compensating filter is kept over the resized states. - let states = match candidate.rollup.children()[0].non_asap() { - Some(NonASAPOp::Filter { pred, .. }) => { - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Filter { - pred: pred.clone(), - child: Rc::clone(&sized), - }))? - } - _ => Rc::clone(&sized), - }; - let merge = regroup_merge(&states, keys.clone(), &candidate.coarse)?; - replaced.push((Rc::as_ptr(&candidate.coarse), merge)); - } - } - Ok(replaced) -} - -/// Every non-empty subset of `candidates`, in binary-counter order (the -/// first candidate the lowest bit); an error past [`MAX_REUSE_CANDIDATES`]. -fn subsets(candidates: &[T], rule: &'static str) -> Result>, CseError> { - let n = candidates.len(); - if n > MAX_REUSE_CANDIDATES { - return Err(CseError::TooManyCandidates { - counted: match rule { - "rollup" => "rollup choices", - _ => "top-k reuse candidates", - }, - found: n, - max: MAX_REUSE_CANDIDATES, - }); - } - Ok((1u64..(1u64 << n)) - .map(|mask| { - (0..n) - .filter(|&i| mask >> i & 1 == 1) - .map(|i| candidates[i].clone()) - .collect() - }) - .collect()) -} - -/// `roots` with every node in `replaced` swapped for its replacement, and -/// the nodes a replacement reads rebuilt too (a rollup may read a build -/// another rollup replaces); every guarantee composed again. -fn replace_all( - roots: &[QueryRoot], - replaced: &HashMap<*const OperatorNode, Rc>, -) -> Result, CseError> { - fn walk( - node: &Rc, - replaced: &HashMap<*const OperatorNode, Rc>, - rebuilt: &mut HashMap<*const OperatorNode, Rc>, - ) -> Result, CseError> { - if let Some(done) = rebuilt.get(&Rc::as_ptr(node)) { - return Ok(Rc::clone(done)); - } - let source = replaced.get(&Rc::as_ptr(node)).unwrap_or(node); - let mut changed = !Rc::ptr_eq(source, node); - let mut children = HashMap::new(); - for child in source.children() { - let new = walk(child, replaced, rebuilt)?; - changed |= !Rc::ptr_eq(&new, child); - children.insert(Rc::as_ptr(child), new); - } - let out = if !changed { - Rc::clone(node) - } else if children - .iter() - .all(|(old, new)| std::ptr::eq(*old, Rc::as_ptr(new))) - { - Rc::clone(source) - } else { - Rc::new(source.with_new_children(|child| children[&Rc::as_ptr(child)].clone())?) - }; - rebuilt.insert(Rc::as_ptr(node), Rc::clone(&out)); - Ok(out) - } - let mut rebuilt = HashMap::new(); - let mut out = Vec::new(); - for root in roots { - out.push(match root { - QueryRoot::Operator(node) => QueryRoot::Operator(walk(node, replaced, &mut rebuilt)?), - QueryRoot::Scalar(expr) => { - let mut nodes = HashMap::new(); - for node in expr.operator_refs() { - nodes.insert(Rc::as_ptr(node), walk(node, replaced, &mut rebuilt)?); - } - QueryRoot::Scalar( - expr.map_operator_refs(&mut |node| nodes[&Rc::as_ptr(node)].clone()), - ) - } - }); - } - Ok(recompose_guarantees(&out)?) -} - -/// Exact top-k computations of one input and partition with at least two -/// different `k`, in the order a children-first walk meets them, and the -/// ranking they all read in a reuse candidate: `Limit(k_max)` over `Sort` -/// of the input, partitioned as they are. -#[derive(Debug, Clone)] -pub(crate) struct TopKReuseCandidate { - /// The plain `Aggregate` top-k nodes of one input and partition, in - /// children-first order; at least two different `k`. - pub members: Vec>, - /// `Limit(k_max)` over a descending `Sort` of their input, partitioned - /// as they are: what every member reads in a reuse candidate. - pub ranking: Rc, -} - -/// The exact top-k reuse candidates of the candidate with query roots `roots`: -/// plain (exact) `Aggregate` top-k nodes over an exact input, unfiltered and -/// grouped by listed keys, grouped by equal input and partition. -pub(crate) fn topk_reuse_candidates(roots: &[QueryRoot]) -> Vec { - let mut nodes = Vec::new(); - let mut seen = std::collections::HashSet::new(); - for node in roots.iter().flat_map(QueryRoot::operator_refs) { - collect_children_first(node, &mut seen, &mut nodes); - } - let mut groups: Vec>> = Vec::new(); - for node in nodes.into_iter().filter(|n| exact_topk(n).is_some()) { - let same = |member: &Rc| { - let (Some((child, by, _)), Some((other, other_by, _))) = - (exact_topk(member), exact_topk(&node)) - else { - return false; - }; - by == other_by && (Rc::ptr_eq(child, other) || child == other) - }; - match groups.iter_mut().find(|g| same(&g[0])) { - Some(group) => group.push(node), - None => groups.push(vec![node]), - } - } - groups - .into_iter() - .filter_map(|members| { - let ks: Vec = members - .iter() - .filter_map(|m| exact_topk(m)) - .map(|t| t.2) - .collect(); - let largest = *ks.iter().max()?; - if ks.iter().all(|&k| k == largest) { - return None; - } - let (child, by, _) = exact_topk(&members[0])?; - let ranking = ranking(child, by, largest)?; - Some(TopKReuseCandidate { members, ranking }) - }) - .collect() -} - -/// The input, partition and `k` of an exact top-k over an exact input. -fn exact_topk(node: &OperatorNode) -> Option<(&Rc, &Reduction, usize)> { - let Some(NonASAPOp::Aggregate { - child, - reduction: reduction @ Reduction::Reduce(by), - measures, - filters, - having: None, - .. - }) = node.non_asap() - else { - return None; - }; - let [AggIntent::TopK { k, .. }] = measures.as_slice() else { - return None; - }; - let exact = child - .guarantee - .as_ref() - .is_some_and(asap_types::ir::properties::AccuracyGuarantee::is_exact); - (exact && !by.is_without() && filters.iter().all(Option::is_none)) - .then_some((child, reduction, *k)) -} - -/// The `k` highest-ranked rows of `child` per partition: `Limit` over a -/// descending `Sort` by its ranked column, as exact top-k runs. -fn ranking(child: &Rc, by: &Reduction, k: usize) -> Option> { - let Reduction::Reduce(keys) = by else { - return None; - }; - let value = asap_types::ir::schema::topk::ranked_column(child).ok()?; - let sorted = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Sort { - keys: vec![SortKey { - expr: ScalarExpr::Column(value), - ascending: false, - nulls_first: false, - }], - partition_by: keys.clone(), - child: Rc::clone(child), - })) - .ok()?; - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Limit { - n: Some(k), - offset: 0, - partition_by: keys.clone(), - child: sorted, - })) - .ok() -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::test_support::lower_promql; - use asap_types::ir::properties::AccuracyTarget; - use asap_types::ir::schema::schema_support::with_promql_series_identity; - use asap_types::ir::schema::SketchAlgorithm; - - fn roots(queries: &[&str]) -> Vec { - queries - .iter() - .map(|q| { - let root = lower_promql(q, AccuracyTarget::exact()); - QueryRoot::Operator(with_promql_series_identity(&root).unwrap()) - }) - .collect() - } - - /// The positions in `all` of the items of `chosen`, matched by the node - /// `node` gives for each. - fn positions(all: &[T], chosen: &[T], node: impl Fn(&T) -> &Rc) -> Vec { - chosen - .iter() - .map(|item| { - all.iter() - .position(|other| Rc::ptr_eq(node(other), node(item))) - .expect("a chosen item is listed") - }) - .collect() - } - - /// `sum by (job) (m)` and `max by (host, job) (m)` scan `m` with other - /// labels; widened to `job, host`, both scans project the same - /// columns, and each root keeps its columns. - #[test] - fn scans_of_one_metric_with_other_labels_share_once_widened() { - let roots = roots(&["sum by (job) (m)", "max by (host, job) (m)"]); - let widened = with_union_scan_labels(&roots).unwrap().unwrap(); - let scan_labels = |root: &QueryRoot| { - let QueryRoot::Operator(node) = root else { - unreachable!() - }; - OperatorNode::reachable(node) - .iter() - .find_map(|n| match n.non_asap() { - Some(NonASAPOp::Scan { schema, .. }) => Some( - schema - .fields - .iter() - .map(|f| f.name.clone()) - .collect::>(), - ), - _ => None, - }) - .unwrap() - }; - assert_eq!( - scan_labels(&widened[0]), - ["ts", "value", "job", "host", PROMQL_SERIES_IDENTITY] - ); - assert_eq!(scan_labels(&widened[0]), scan_labels(&widened[1])); - // The unique key still names the series identity and ts. - let QueryRoot::Operator(node) = &widened[0] else { - unreachable!() - }; - let scan = OperatorNode::reachable(node) - .into_iter() - .find(|n| matches!(n.non_asap(), Some(NonASAPOp::Scan { .. }))) - .unwrap(); - assert_eq!(scan.schema.unique_keys, [vec![4, 0]]); - for (root, wide) in roots.iter().zip(&widened) { - let (QueryRoot::Operator(root), QueryRoot::Operator(wide)) = (root, wide) else { - unreachable!() - }; - assert_eq!(root.schema, wide.schema); - } - // Scans that already agree are left alone. - assert!(with_union_scan_labels(&widened).unwrap().is_none()); - } - - /// A count over a window, read from a Count-Min built in tumbling - /// tumbling windows, keeps the guarantee of one build over the window: Pass 1 - /// offers Count-Min for counts, and its merged counters add. - #[test] - fn a_count_from_count_min_tumbling_windows_keeps_its_guarantee() { - use crate::pass1::per_computation_candidates::{ - compose_logical_candidate, generate_per_computation_candidates, - }; - let root = lower_promql( - "count_over_time(m[5m])", - asap_types::ir::properties::AccuracyTarget::max_error(0.01).failure_probability(0.001), - ); - let inventory = generate_per_computation_candidates( - &crate::stage0::generate_dag_of_logical_memo_groups(&[QueryRoot::Operator( - with_promql_series_identity(&root).unwrap(), - )]), - ) - .unwrap(); - let choice: Vec = inventory - .targets - .iter() - .map(|t| { - t.ways_to_compute - .iter() - .position(|c| { - *c == asap_types::stages::HowToComputeAggregateMeasure::Summary( - SketchAlgorithm::Cms, - ) - }) - .expect("Count-Min is offered for a count") - }) - .collect(); - let roots: Vec = compose_logical_candidate(&inventory, &choice).unwrap(); - let candidates = tumbling_window_candidates(&roots, &[Some(60_000)]); - let pass2_candidates = window_composition_pass2_candidates(&roots, &candidates).unwrap(); - let [pass2_candidate] = pass2_candidates.as_slice() else { - panic!("one Pass 2 candidate: {pass2_candidates:?}") - }; - let (QueryRoot::Operator(whole), QueryRoot::Operator(windowed)) = - (&roots[0], &pass2_candidate.roots.as_ref().unwrap()[0]) - else { - panic!("operator roots") - }; - assert!(whole.guarantee.is_some()); - assert_eq!(windowed.guarantee, whole.guarantee); - } - - /// The composed candidate of `queries` (each at its own ε, δ = 0.01) - /// in which every target chose `family`. - fn chose(queries: &[(&str, f64)], family: SketchAlgorithm) -> Vec { - use asap_types::stages::HowToComputeAggregateMeasure; - chose_where(queries, |c| { - *c == HowToComputeAggregateMeasure::Summary(family) - }) - } - - /// The composed candidate of `queries` (each at its own ε, δ = 0.01) - /// in which every target chose the first candidate `pick` accepts. - fn chose_where( - queries: &[(&str, f64)], - pick: impl Fn(&asap_types::stages::HowToComputeAggregateMeasure) -> bool, - ) -> Vec { - use crate::pass1::per_computation_candidates::{ - compose_logical_candidate, generate_per_computation_candidates, - }; - let roots: Vec<_> = queries - .iter() - .map(|(q, epsilon)| { - let root = lower_promql( - q, - asap_types::ir::properties::AccuracyTarget::max_error(*epsilon) - .failure_probability(0.01) - .accuracy_metric(asap_types::ir::properties::AccuracyMetric::Rank), - ); - QueryRoot::Operator(with_promql_series_identity(&root).unwrap()) - }) - .collect(); - let inventory = generate_per_computation_candidates( - &crate::stage0::generate_dag_of_logical_memo_groups(&roots), - ) - .unwrap(); - let choice: Vec = inventory - .targets - .iter() - .map(|t| { - t.ways_to_compute - .iter() - .position(&pick) - .expect("the family is a candidate") - }) - .collect(); - compose_logical_candidate(&inventory, &choice).unwrap() - } - - /// The summary builds `roots` read, by address, with each one's KLL `k`. - fn kll_builds(roots: &[QueryRoot]) -> Vec<(*const OperatorNode, u32)> { - let mut out: Vec<(*const OperatorNode, u32)> = Vec::new(); - for node in roots - .iter() - .flat_map(QueryRoot::operator_refs) - .flat_map(OperatorNode::reachable) - { - if let Some(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), - .. - }) = node.asap() - { - let asap_types::ir::schema::SketchParams::Kll { k, .. } = kind.params() else { - panic!("{kind:?}") - }; - if !out.iter().any(|(p, _)| *p == Rc::as_ptr(&node)) { - out.push((Rc::as_ptr(&node), *k)); - } - } - } - out - } - - fn root_bound(root: &QueryRoot) -> f64 { - let QueryRoot::Operator(node) = root else { - panic!("operator root") - }; - node.guarantee - .as_ref() - .and_then(|g| g.spec.error.evaluate()) - .expect("a constant bound") - } - - /// The tumbling-window candidates of `roots` as (tumbling-window width, tumbling-window count). - fn tumbling_windows(roots: &[QueryRoot], interval_ms: &[Option]) -> Vec<(u64, u32)> { - tumbling_window_candidates(roots, interval_ms) - .iter() - .map(|rule_candidate| { - ( - rule_candidate.tumbling_window_width_ms, - rule_candidate.tumbling_window_count, - ) - }) - .collect() - } - - /// A 5-min window evaluated every 2 min, 90 s or 1 min is built in tumbling windows - /// of gcd(window, interval). A once-per-window reader has one tumbling window, - /// which can reuse the newest tumbling window of a longer chain. - #[test] - fn tumbling_window_width_is_the_gcd_of_the_window_and_the_interval() { - let roots = chose( - &[("quantile_over_time(0.99, m[5m])", 0.01)], - SketchAlgorithm::Kll, - ); - assert_eq!(tumbling_windows(&roots, &[Some(120_000)]), [(60_000, 5)]); - assert_eq!(tumbling_windows(&roots, &[Some(90_000)]), [(30_000, 10)]); - assert_eq!(tumbling_windows(&roots, &[Some(60_000)]), [(60_000, 5)]); - assert_eq!(tumbling_windows(&roots, &[Some(300_000)]), [(300_000, 1)]); - assert!(tumbling_windows(&roots, &[None]).is_empty()); - } - - /// Past their caps, the tumbling-window rule and the reuse subsets name how many - /// candidates they found. - #[test] - fn too_many_tumbling_window_or_reuse_candidates_name_the_count_found() { - let roots = chose( - &[("quantile_over_time(0.99, m[5m])", 0.01)], - SketchAlgorithm::Kll, - ); - let tumbling_windows: Vec<_> = tumbling_window_candidates(&roots, &[Some(60_000)]) - .into_iter() - .cycle() - .take(MAX_TUMBLING_WINDOW_CANDIDATES + 1) - .collect(); - let error = window_composition_pass2_candidates(&roots, &tumbling_windows).unwrap_err(); - assert_eq!( - error.to_string(), - format!( - "found {} summary builds that could be built in tumbling windows, more than \ - {MAX_TUMBLING_WINDOW_CANDIDATES}", - MAX_TUMBLING_WINDOW_CANDIDATES + 1 - ) - ); - let error = subsets(&[0u8; MAX_REUSE_CANDIDATES + 1], "rollup").unwrap_err(); - assert_eq!( - error.to_string(), - format!( - "found {} rollup choices, more than {MAX_REUSE_CANDIDATES}", - MAX_REUSE_CANDIDATES + 1 - ) - ); - } - - /// A range read at an `@` anchor has no tumbling windows, however many shifts - /// stack above the anchored one: `m[5m] @ 100` with an extra - /// `offset 1m` shift between the range and the anchor is still anchored. - #[test] - fn an_anchor_below_stacked_shifts_has_no_tumbling_windows() { - use asap_types::ir::operator::TimeShift; - let roots = chose( - &[("quantile_over_time(0.99, m[5m] @ 100)", 0.01)], - SketchAlgorithm::Kll, - ); - let QueryRoot::Operator(root) = &roots[0] else { - panic!("operator root") - }; - let build = OperatorNode::reachable(root) - .into_iter() - .find(|n| matches!(n.asap(), Some(ASAPOp::SummaryAgg { .. }))) - .expect("a build"); - assert_eq!(splittable_window_ms(&build), None, "one anchored shift"); - // Put an `offset 1m` shift between the range and its anchored shift. - let restacked = build - .with_new_children(|range| { - Rc::new( - range - .with_new_children(|anchored| { - assert!(matches!( - anchored.non_asap(), - Some(NonASAPOp::TimeShift { shift, .. }) if shift.at.is_some() - )); - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeShift { - shift: TimeShift { - offset_ms: 60_000, - at: None, - }, - child: anchored.clone(), - })) - .unwrap() - }) - .unwrap(), - ) - }) - .unwrap(); - assert_eq!( - splittable_window_ms(&restacked), - None, - "anchor below an offset" - ); - } - - /// Exact accumulators whose states merge get tumbling windows; IRate, which reads - /// the last two samples of the window, does not. - #[test] - fn exact_accumulators_that_merge_get_tumbling_windows() { - use asap_types::stages::HowToComputeAggregateMeasure; - let accumulator = |c: &HowToComputeAggregateMeasure| { - matches!(c, HowToComputeAggregateMeasure::ExactAccumulator(_)) - }; - for query in [ - "rate(m[5m])", - "increase(m[5m])", - "sum_over_time(m[5m])", - "max_over_time(m[5m])", - ] { - let roots = chose_where(&[(query, 0.01)], accumulator); - assert_eq!( - tumbling_windows(&roots, &[Some(60_000)]), - [(60_000, 5)], - "{query}" - ); - } - let roots = chose_where(&[("irate(m[5m])", 0.01)], accumulator); - assert!(tumbling_windows(&roots, &[Some(60_000)]).is_empty()); - } - - /// The rule's candidate reads a `SummaryMerge` over one `SummaryTumblingWindowChain` of - /// the build's family and size over the range's input, keeps the - /// build's output names, and keeps the whole build's guarantee (a KLL - /// merges into the state of one build over the union). - #[test] - fn tumbling_windows_replace_the_build_with_a_merged_chain() { - let roots = chose( - &[("quantile_over_time(0.99, m[5m])", 0.01)], - SketchAlgorithm::Kll, - ); - let rule_candidates = tumbling_window_candidates(&roots, &[Some(60_000)]); - let pass2_candidates = - window_composition_pass2_candidates(&roots, &rule_candidates).unwrap(); - let [pass2_candidate] = pass2_candidates.as_slice() else { - panic!("one Pass 2 candidate: {pass2_candidates:?}") - }; - assert_eq!( - positions( - &rule_candidates, - &pass2_candidate.tumbling_window_candidates, - |c| &c.build - ), - [0] - ); - let tumbling_windows = pass2_candidate.roots.as_ref().unwrap(); - let QueryRoot::Operator(root) = &tumbling_windows[0] else { - panic!("operator root") - }; - root.validate_structure().unwrap(); - let QueryRoot::Operator(whole) = &roots[0] else { - panic!("operator root") - }; - assert_eq!(root.guarantee, whole.guarantee); - assert!(root.guarantee.is_some()); - let merge = OperatorNode::reachable(root) - .into_iter() - .find(|n| matches!(n.asap(), Some(ASAPOp::SummaryMerge { .. }))) - .expect("a merge"); - let Some(ASAPOp::SummaryMerge { children, .. }) = merge.asap() else { - unreachable!() - }; - let [chain] = children.as_slice() else { - panic!("one chain") - }; - let Some(ASAPOp::SummaryTumblingWindowChain { - child, - family, - tumbling_window_width_ms: 60_000, - tumbling_window_count: 5, - .. - }) = chain.asap() - else { - panic!("{:?}", chain.operator) - }; - let Some(ASAPOp::SummaryAgg { - child: range, - family: whole_family, - .. - }) = rule_candidates[0].build.asap() - else { - unreachable!() - }; - assert_eq!(family, whole_family); - assert_eq!(child.operator, range.children()[0].operator); - assert_eq!(chain.schema, rule_candidates[0].build.schema); - assert_eq!(merge.coverage(), rule_candidates[0].build.coverage()); - } - - /// Two builds that could be split give three candidates: each alone and - /// both, in binary-counter order. Their windows (5 and 2 min) nest on - /// 1-min tumbling windows, so both together are also offered as one shared chain. - #[test] - fn every_non_empty_subset_of_the_tumbling_window_candidates_is_a_pass2_candidate() { - let roots = chose( - &[ - ("quantile_over_time(0.99, m[5m])", 0.01), - ("quantile_over_time(0.5, m[2m])", 0.01), - ], - SketchAlgorithm::Kll, - ); - let rule_candidates = tumbling_window_candidates(&roots, &[Some(60_000), Some(60_000)]); - assert_eq!(rule_candidates.len(), 2); - let subsets: Vec<_> = window_composition_pass2_candidates(&roots, &rule_candidates) - .unwrap() - .iter() - .map(|v| { - assert!(v.roots.is_ok()); - ( - positions(&rule_candidates, &v.tumbling_window_candidates, |c| { - &c.build - }), - v.shared_chains.clone(), - ) - }) - .collect(); - assert_eq!( - subsets, - [ - (vec![0], vec![]), - (vec![1], vec![]), - (vec![0, 1], vec![]), - ( - vec![0, 1], - vec![SharedTumblingWindowChain { - members: vec![0, 1], - tumbling_window_width_ms: 60_000, - tumbling_window_count: 5 - }] - ) - ] - ); - } - - /// The tumbling-window chains in `roots`, one per distinct node. - fn chains(roots: &[QueryRoot]) -> Vec> { - let mut found: Vec> = Vec::new(); - for root in roots { - for node in OperatorNode::reachable(operator(root)) { - if matches!(node.asap(), Some(ASAPOp::SummaryTumblingWindowChain { .. })) - && !found.iter().any(|c| Rc::ptr_eq(c, &node)) - { - found.push(node); - } - } - } - found - } - - /// The `newest_tumbling_windows` of the one merge `root` reads. - fn newest_tumbling_windows(root: &QueryRoot) -> Option { - OperatorNode::reachable(operator(root)) - .into_iter() - .find_map(|n| match n.asap() { - Some(ASAPOp::SummaryMerge { - newest_tumbling_windows, - .. - }) => Some(*newest_tumbling_windows), - _ => None, - }) - .expect("a merge") - } - - /// A 1-min and a 5-min KLL quantile over one metric, both every 10 s, - /// are 6 and 30 tumbling windows of 10 s; the 1-min window is the newest 6 of the - /// 5-min one. Besides each query's own chain, the rule offers one chain - /// of 30 tumbling windows read by both: the 5-min query merges every tumbling window, the - /// 1-min one only the newest 6, and each keeps the guarantee of its own - /// whole-window build (a KLL merges exactly). - #[test] - fn nested_windows_on_equal_tumbling_windows_share_one_chain() { - let roots = chose( - &[ - ("quantile_over_time(0.5, m[1m])", 0.01), - ("quantile_over_time(0.9, m[5m])", 0.01), - ], - SketchAlgorithm::Kll, - ); - let rule_candidates = tumbling_window_candidates(&roots, &[Some(10_000), Some(10_000)]); - assert_eq!( - tumbling_windows(&roots, &[Some(10_000), Some(10_000)]), - [(10_000, 6), (10_000, 30)] - ); - let pass2_candidates = - window_composition_pass2_candidates(&roots, &rule_candidates).unwrap(); - let shared: Vec<_> = pass2_candidates - .iter() - .filter(|c| !c.shared_chains.is_empty()) - .collect(); - let [shared] = shared.as_slice() else { - panic!("one shared-chain candidate: {pass2_candidates:?}") - }; - assert_eq!( - shared.shared_chains, - [SharedTumblingWindowChain { - members: vec![0, 1], - tumbling_window_width_ms: 10_000, - tumbling_window_count: 30 - }] - ); - let shared_roots = shared.roots.as_ref().unwrap(); - let [chain] = chains(shared_roots) - .try_into() - .unwrap_or_else(|c: Vec<_>| panic!("one chain, found {}", c.len())); - assert!(matches!( - chain.asap(), - Some(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms: 10_000, - tumbling_window_count: 30, - .. - }) - )); - assert_eq!(newest_tumbling_windows(&shared_roots[0]), Some(6)); - assert_eq!(newest_tumbling_windows(&shared_roots[1]), None); - for (root, whole) in shared_roots.iter().zip(&roots) { - let root = operator(root); - root.validate_structure().unwrap(); - assert!(root.guarantee.is_some()); - assert_eq!(root.guarantee, operator(whole).guarantee); - } - // The per-query chains stay candidates: two chains, each read in full. - let separate = pass2_candidates - .iter() - .find(|c| c.tumbling_window_candidates.len() == 2 && c.shared_chains.is_empty()) - .expect("both builds in their own tumbling windows"); - assert_eq!(chains(separate.roots.as_ref().unwrap()).len(), 2); - } - - /// A 15-min quantile every minute and the same quantile every 5 min - /// are 15 × 1-min and 3 × 5-min tumbling windows. The rule also offers one chain - /// at the finer width, 15 × 1 min, that both read in full, each at its - /// own interval; a 1-hour quantile every 5 min (12 × 5 min) reads the - /// newest 60 of one 60 × 1-min chain with them. - #[test] - fn queries_at_different_intervals_share_one_chain_at_the_finer_width() { - let roots = chose( - &[ - ("quantile_over_time(0.5, m[15m])", 0.01), - ("quantile_over_time(0.9, m[15m])", 0.01), - ], - SketchAlgorithm::Kll, - ); - let grids = [Some(60_000), Some(300_000)]; - assert_eq!( - tumbling_windows(&roots, &grids), - [(60_000, 15), (300_000, 3)] - ); - let rule_candidates = tumbling_window_candidates(&roots, &grids); - let pass2_candidates = - window_composition_pass2_candidates(&roots, &rule_candidates).unwrap(); - let shared = pass2_candidates - .iter() - .find(|c| !c.shared_chains.is_empty()) - .expect("a shared-chain candidate"); - assert_eq!( - shared.shared_chains, - [SharedTumblingWindowChain { - members: vec![0, 1], - tumbling_window_width_ms: 60_000, - tumbling_window_count: 15 - }] - ); - let shared_roots = shared.roots.as_ref().unwrap(); - let [chain] = chains(shared_roots) - .try_into() - .unwrap_or_else(|c: Vec<_>| panic!("one chain, found {}", c.len())); - assert!(matches!( - chain.asap(), - Some(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms: 60_000, - tumbling_window_count: 15, - .. - }) - )); - assert_eq!(newest_tumbling_windows(&shared_roots[0]), None); - assert_eq!(newest_tumbling_windows(&shared_roots[1]), None); - for (root, whole) in shared_roots.iter().zip(&roots) { - operator(root).validate_structure().unwrap(); - assert_eq!(operator(root).guarantee, operator(whole).guarantee); - } - - let roots = chose( - &[ - ("quantile_over_time(0.5, m[15m])", 0.01), - ("quantile_over_time(0.9, m[1h])", 0.01), - ], - SketchAlgorithm::Kll, - ); - let rule_candidates = tumbling_window_candidates(&roots, &grids); - let shared = window_composition_pass2_candidates(&roots, &rule_candidates) - .unwrap() - .into_iter() - .find(|c| !c.shared_chains.is_empty()) - .expect("a shared-chain candidate"); - let shared_roots = shared.roots.unwrap(); - assert_eq!(newest_tumbling_windows(&shared_roots[0]), Some(15)); - assert_eq!(newest_tumbling_windows(&shared_roots[1]), None); - assert!(matches!( - chains(&shared_roots)[0].asap(), - Some(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms: 60_000, - tumbling_window_count: 60, - .. - }) - )); - } - - /// A build whose tumbling windows the group's finest tumbling windows do not divide gets its - /// own chain without breaking the group: 20-min and 10-min quantiles - /// every 2 min (2-min tumbling windows) share one chain, and a 15-min quantile - /// every 3 min (3-min tumbling windows) stays apart. - #[test] - fn a_build_on_an_incompatible_grid_does_not_break_a_shared_chain() { - let roots = chose( - &[ - ("quantile_over_time(0.5, m[20m])", 0.01), - ("quantile_over_time(0.9, m[10m])", 0.01), - ("quantile_over_time(0.99, m[15m])", 0.01), - ], - SketchAlgorithm::Kll, - ); - let grids = [Some(120_000), Some(120_000), Some(180_000)]; - assert_eq!( - tumbling_windows(&roots, &grids), - [(120_000, 10), (120_000, 5), (180_000, 5)] - ); - let rule_candidates = tumbling_window_candidates(&roots, &grids); - let shared = window_composition_pass2_candidates(&roots, &rule_candidates) - .unwrap() - .into_iter() - .find(|c| c.tumbling_window_candidates.len() == 3 && !c.shared_chains.is_empty()) - .expect("a shared chain among all three builds in tumbling windows"); - assert_eq!( - shared.shared_chains, - [SharedTumblingWindowChain { - members: vec![0, 1], - tumbling_window_width_ms: 120_000, - tumbling_window_count: 10 - }] - ); - assert_eq!(chains(shared.roots.as_ref().unwrap()).len(), 2); - } - - /// The shared chains (width, count) of every Pass 2 candidate that - /// windowed all of `rule_candidates`. - fn shared_grids( - roots: &[QueryRoot], - rule_candidates: &[TumblingWindowCandidate], - ) -> Vec> { - window_composition_pass2_candidates(roots, rule_candidates) - .unwrap() - .into_iter() - .filter(|c| c.tumbling_window_candidates.len() == rule_candidates.len()) - .map(|c| { - c.shared_chains - .iter() - .map(|shared| { - ( - shared.tumbling_window_width_ms, - shared.tumbling_window_count, - ) - }) - .collect() - }) - .collect() - } - - /// Tumbling windows neither member's divide (3-min and 5-min tumbling windows of a 15-min - /// window) are shared at their gcd, 1 min, as one more candidate beside - /// the per-query chains; no member's own width serves both. - #[test] - fn a_shared_grid_at_the_gcd_of_the_members_widths_is_offered() { - let roots = chose( - &[ - ("quantile_over_time(0.5, m[15m])", 0.01), - ("quantile_over_time(0.9, m[15m])", 0.01), - ], - SketchAlgorithm::Kll, - ); - let grids = [Some(180_000), Some(300_000)]; - assert_eq!( - tumbling_windows(&roots, &grids), - [(180_000, 5), (300_000, 3)] - ); - let rule_candidates = tumbling_window_candidates(&roots, &grids); - assert_eq!( - shared_grids(&roots, &rule_candidates), - [vec![], vec![(60_000, 15)]] - ); - } - - /// A 15-min quantile every minute (15 × 1-min tumbling windows) and a 6-min one - /// every 90 s (4 × 90-s tumbling windows): one chain of 30-s tumbling windows over 15 min - /// serves both, the 6-min query merging the newest 12. It comes beside - /// the per-query chains, and Stage 3 prices which is cheaper. - #[test] - fn queries_at_60_and_90_seconds_share_one_chain_of_30_second_tumbling_windows() { - let roots = chose( - &[ - ("quantile_over_time(0.5, m[15m])", 0.01), - ("quantile_over_time(0.9, m[6m])", 0.01), - ], - SketchAlgorithm::Kll, - ); - let grids = [Some(60_000), Some(90_000)]; - assert_eq!( - tumbling_windows(&roots, &grids), - [(60_000, 15), (90_000, 4)] - ); - let rule_candidates = tumbling_window_candidates(&roots, &grids); - assert_eq!( - shared_grids(&roots, &rule_candidates), - [vec![], vec![(30_000, 30)]] - ); - let shared = window_composition_pass2_candidates(&roots, &rule_candidates) - .unwrap() - .into_iter() - .find(|c| !c.shared_chains.is_empty()) - .unwrap(); - let shared_roots = shared.roots.unwrap(); - assert_eq!(newest_tumbling_windows(&shared_roots[0]), None); - assert_eq!(newest_tumbling_windows(&shared_roots[1]), Some(12)); - for (root, whole) in shared_roots.iter().zip(&roots) { - operator(root).validate_structure().unwrap(); - assert_eq!(operator(root).guarantee, operator(whole).guarantee); - } - } - - /// When the members' finest width divides the others, the gcd is that - /// width: one shared candidate, not two equal ones. With 1-min, 2-min - /// and 3-min tumbling windows, the divisible group (1 and 2 min, the 3-min build - /// joining at 1 min too) is already the gcd grid. - #[test] - fn the_gcd_grid_is_not_listed_twice_when_a_member_width_is_the_gcd() { - let roots = chose( - &[ - ("quantile_over_time(0.5, m[12m])", 0.01), - ("quantile_over_time(0.9, m[12m])", 0.01), - ("quantile_over_time(0.99, m[12m])", 0.01), - ], - SketchAlgorithm::Kll, - ); - let grids = [Some(60_000), Some(120_000), Some(180_000)]; - let rule_candidates = tumbling_window_candidates(&roots, &grids); - assert_eq!( - shared_grids(&roots, &rule_candidates), - [vec![], vec![(60_000, 12)]] - ); - } - - /// The shared chain is sized for its strictest reader - /// ([`size_for_all_readers`]): a 1-min quantile at ε = 0.01 and a - /// 5-min one at ε = 0.05 read one chain of the 1-min query's `k`. - #[test] - fn a_shared_chain_is_sized_for_its_strictest_reader() { - let roots = chose( - &[ - ("quantile_over_time(0.5, m[1m])", 0.01), - ("quantile_over_time(0.9, m[5m])", 0.05), - ], - SketchAlgorithm::Kll, - ); - let family = |node: &OperatorNode| match node.asap() { - Some( - ASAPOp::SummaryAgg { family, .. } - | ASAPOp::SummaryTumblingWindowChain { family, .. }, - ) => family.clone(), - _ => unreachable!(), - }; - let strict = family(&builds(&roots[0])[0]); - assert_ne!(strict, family(&builds(&roots[1])[0])); - let rule_candidates = tumbling_window_candidates(&roots, &[Some(10_000), Some(10_000)]); - let pass2_candidates = - window_composition_pass2_candidates(&roots, &rule_candidates).unwrap(); - let shared = pass2_candidates - .iter() - .find(|c| !c.shared_chains.is_empty()) - .expect("a shared-chain candidate"); - let shared_roots = shared.roots.as_ref().unwrap(); - let [chain] = chains(shared_roots) - .try_into() - .unwrap_or_else(|c: Vec<_>| panic!("one chain, found {}", c.len())); - assert_eq!(family(&chain), strict); - // The 1-min query keeps its own guarantee; the 5-min one reads the - // stricter state, so its guarantee is the 1-min build's size's, - // not its own looser one. - let guarantee = |root: &QueryRoot| operator(root).guarantee.clone().unwrap(); - assert_eq!(guarantee(&shared_roots[0]), guarantee(&roots[0])); - assert_ne!(guarantee(&shared_roots[1]), guarantee(&roots[1])); - assert_eq!( - guarantee(&shared_roots[1]).spec, - guarantee(&shared_roots[0]).spec - ); - } - /// The composed candidate of `queries` with every target at its exact - /// option (an exact accumulator, or plain exact execution). - fn exact(queries: &[&str]) -> Vec { - use asap_types::stages::HowToComputeAggregateMeasure as C; - let queries: Vec<(&str, f64)> = queries.iter().map(|q| (*q, 0.01)).collect(); - chose_where(&queries, |c| matches!(c, C::Exact | C::ExactAccumulator(_))) - } - - fn operator(root: &QueryRoot) -> &Rc { - let QueryRoot::Operator(node) = root else { - panic!("operator root") - }; - node - } - - fn builds(root: &QueryRoot) -> Vec> { - OperatorNode::reachable(operator(root)) - .into_iter() - .filter(|n| matches!(n.asap(), Some(ASAPOp::SummaryAgg { .. }))) - .collect() - } - - /// One coarse build's rollup candidate, computed alone, is the one - /// [`rollup_candidates`] lists for it, and needs a finer build to exist. - #[test] - fn one_coarse_builds_rollup_candidate_is_the_listed_one() { - let roots = exact(&[ - "sum by (job) (m)", - "sum by (job, instance) (m)", - "sum by (job, instance, zone) (m)", - ]); - let listed = rollup_candidates(&roots); - assert!(!listed.is_empty()); - for candidate in &listed { - let alone = rollup_candidate_for(&roots, Rc::as_ptr(&candidate.coarse)).unwrap(); - assert!(Rc::ptr_eq(&alone.finer, &candidate.finer)); - } - let coarse = &builds(&roots[0])[0]; - assert!(rollup_candidate_for(&roots[..1], Rc::as_ptr(coarse)).is_none()); - } - - /// `sum by (job)` is answered from `sum by (job, instance)`, which the - /// workload computes anyway: the coarse query reads the finer query's - /// finalized sums, summed by `job`, so it builds nothing over the raw - /// rows; both values stay exact and keep their columns. - #[test] - fn rollup_from_a_finer_exact_sum() { - let roots = exact(&["sum by (job) (m)", "sum by (job, instance) (m)"]); - let rule_candidates = rollup_candidates(&roots); - let [rule_candidate] = rule_candidates.as_slice() else { - panic!("one candidate: {rule_candidates:?}") - }; - assert!(Rc::ptr_eq(&rule_candidate.coarse, &builds(&roots[0])[0])); - let pass2_candidates = rollup_pass2_candidates(&roots, &rule_candidates).unwrap(); - let [pass2_candidate] = pass2_candidates.as_slice() else { - panic!("one Pass 2 candidate") - }; - let rolled = pass2_candidate.roots.as_ref().unwrap(); - let coarse = builds(&rolled[0]); - let fine = builds(&rolled[1]); - assert_eq!(fine.len(), 1); - // The finer build, then the sum over its values. - assert_eq!(coarse.len(), 2); - assert!(Rc::ptr_eq(&coarse[0], &fine[0]) || Rc::ptr_eq(&coarse[1], &fine[0])); - let rollup = coarse.iter().find(|b| !Rc::ptr_eq(b, &fine[0])).unwrap(); - let Some(ASAPOp::SummaryAgg { child, family, .. }) = rollup.asap() else { - unreachable!() - }; - assert!(matches!( - child.asap(), - Some(ASAPOp::FinalizeExactAccumulator { .. }) - )); - assert_eq!( - *family, - FieldDataType::ExactAggregate(ExactKind::Sum, asap_types::ir::schema::ExactParams::Sum) - ); - for (before, after) in roots.iter().zip(rolled) { - let (before, after) = (operator(before), operator(after)); - after.validate_structure().unwrap(); - assert_eq!(before.schema, after.schema); - assert!(after - .guarantee - .as_ref() - .is_some_and(asap_types::ir::properties::AccuracyGuarantee::is_exact)); - } - } - - /// Identical per-series averages retain their values under label projection; - /// different windows, functions, and source predicates must stay separate. - #[test] - fn rollup_crosses_only_identical_per_series_reductions() { - let input = |query: &str| { - let root = lower_promql(query, asap_types::ir::properties::AccuracyTarget::exact()); - OperatorNode::reachable(&root) - .into_iter() - .find(|node| { - matches!( - node.non_asap(), - Some(NonASAPOp::Aggregate { - reduction: Reduction::PerEntity, - .. - }) - ) - }) - .unwrap() - }; - let fine = input("sum by (job, instance) (avg_over_time(m[1m]))"); - assert!(same_rows( - &input("sum by (job) (avg_over_time(m[1m]))"), - &fine - )); - for coarse in [ - "sum by (job) (avg_over_time(m[2m]))", - "sum by (job) (max_over_time(m[1m]))", - "sum by (job) (avg_over_time(m{zone=\"east\"}[1m]))", - ] { - assert!(!same_rows(&input(coarse), &fine), "{coarse}"); - } - } - - /// Minima and maxima roll up as extrema; counts merge their integer states. - #[test] - fn extrema_and_integer_counts_roll_up() { - assert!(!rollup_candidates(&exact(&[ - "count by (job) (m)", - "count by (job, instance) (m)" - ])) - .is_empty()); - for (query, kind) in [ - ("min by (job) (m)", ExactKind::Min), - ("max by (job) (m)", ExactKind::Max), - ] { - let finer = query.replace("(job)", "(job, instance)"); - let roots = exact(&[query, &finer]); - let rule_candidates = rollup_candidates(&roots); - let [rule_candidate] = rule_candidates.as_slice() else { - panic!("{query}: one candidate: {rule_candidates:?}") - }; - let mut value = Rc::clone(&rule_candidate.rollup); - if let Some(NonASAPOp::Project { child, .. }) = value.non_asap() { - value = Rc::clone(child); - } - let Some(ASAPOp::FinalizeExactAccumulator { child: state }) = value.asap() else { - panic!("a finalized value") - }; - let Some(ASAPOp::SummaryAgg { - family: FieldDataType::ExactAggregate(rolled, _), - .. - }) = state.asap() - else { - panic!("an exact accumulator") - }; - assert_eq!(*rolled, kind, "{query}"); - let pass2_candidate = rollup_pass2_candidates(&roots, &rule_candidates) - .unwrap() - .remove(0); - assert_eq!( - operator(&pass2_candidate.roots.unwrap()[0]).schema, - operator(&roots[0]).schema, - "{query}" - ); - } - } - - /// The rule reuses only a finer computation the workload already has: - /// keys that are not a superset, another metric, window or measure, or - /// a lone query, give no tumbling-window candidate. - #[test] - fn rollup_needs_a_finer_computation_of_the_same_rows() { - for queries in [ - &["sum by (job) (m)"][..], - &["sum by (job) (m)", "sum by (instance) (m)"], - &["sum by (job) (m)", "sum by (job, instance) (n)"], - &["sum by (job) (m)", "max by (job, instance) (m)"], - &[ - "sum by (job) (sum_over_time(m[5m]))", - "sum by (job, instance) (sum_over_time(m[1m]))", - ], - &["sum by (job) (m)", "sum by (job) (m)"], - &["sum by (job) (m)", "sum without (instance) (m)"], - &["sum without (job) (m)", "sum by (job, instance) (m)"], - ] { - assert!(rollup_candidates(&exact(queries)).is_empty(), "{queries:?}"); - } - } - - /// Rollup follows the family-capability table: sums, counts and - /// extrema merge exactly across groups; counter states merge only in - /// time, IRate not at all; a heap sketch's merged top-k has no - /// guarantee, and no sketch's states can be regrouped in the IR. - #[test] - fn rollup_is_refused_for_families_without_an_exact_merge_across_groups() { - use asap_types::ir::schema::{ExactParams as P, SketchParams}; - let exact = |kind, params| FieldDataType::ExactAggregate(kind, params); - assert_eq!( - rollup_kind(&exact(ExactKind::Sum, P::Sum)), - Some(ExactKind::Sum) - ); - assert_eq!( - rollup_kind(&exact(ExactKind::Count, P::Count)), - Some(ExactKind::Sum) - ); - for family in [ - exact(ExactKind::Rate, P::Rate), - exact(ExactKind::Increase, P::Increase), - exact(ExactKind::IRate, P::IRate), - FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::CmsWithHeap, - SketchParams::CmsWithHeap { - width: 64, - depth: 5, - heap_size: 10, - }, - ), - GroupingStrategy::default(), - ), - FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: 200, - failure_probability: None, - }, - ), - GroupingStrategy::default(), - ), - ] { - assert_eq!(rollup_kind(&family), None, "{family:?}"); - } - } - - /// #840's `quantile by (host, job)` and `quantile by (job)`, both KLL, - /// the coarse one at a tighter ε: the coarse query reads the regrouping - /// merge of the finer KLL's states by `job`, so one KLL is built, sized - /// for its strictest reader (the larger `k`), and the coarse query keeps - /// a guarantee within its own target. - #[test] - fn a_coarse_sketch_rolls_up_from_the_finer_build() { - let roots = chose( - &[ - ("quantile by (host, job) (0.5, data)", 0.01), - ("quantile by (job) (0.5, data)", 0.001), - ], - SketchAlgorithm::Kll, - ); - let ks: Vec = kll_builds(&roots).into_iter().map(|(_, k)| k).collect(); - let rule_candidates = rollup_candidates(&roots); - let [rule_candidate] = rule_candidates.as_slice() else { - panic!("one candidate: {rule_candidates:?}") - }; - assert!(Rc::ptr_eq(&rule_candidate.coarse, &builds(&roots[1])[0])); - assert!(Rc::ptr_eq(&rule_candidate.finer, &builds(&roots[0])[0])); - let pass2_candidates = rollup_pass2_candidates(&roots, &rule_candidates).unwrap(); - let [pass2_candidate] = pass2_candidates.as_slice() else { - panic!("one Pass 2 candidate") - }; - let rolled = pass2_candidate.roots.as_ref().unwrap(); - // One KLL, at the larger k, read by both queries. - assert_eq!( - kll_builds(rolled) - .into_iter() - .map(|(_, k)| k) - .collect::>(), - [*ks.iter().max().unwrap()] - ); - let merge = OperatorNode::reachable(operator(&rolled[1])) - .into_iter() - .find(|n| matches!(n.asap(), Some(ASAPOp::SummaryMerge { .. }))) - .expect("the coarse query reads a rollup"); - let Some(ASAPOp::SummaryMerge { - children, group_by, .. - }) = merge.asap() - else { - unreachable!() - }; - assert_eq!(group_by, &Some(vec![1])); - assert!(Rc::ptr_eq(&children[0], &builds(&rolled[0])[0])); - for (before, after) in roots.iter().zip(rolled) { - let (before, after) = (operator(before), operator(after)); - after.validate_structure().unwrap(); - assert_eq!(before.schema, after.schema); - } - // Each query's guarantee is the shared KLL's, within its target. - assert!( - root_bound(&rolled[1]) <= 0.001, - "{}", - root_bound(&rolled[1]) - ); - assert!(root_bound(&rolled[0]) <= 0.001); - } - - /// Every coarser grouping of one finest build is one choice: two coarse - /// builds of a three-grouping chain give one Pass 2 candidate, in which - /// both read the finest build's states. A finer build that is itself - /// coarse is not a source. - #[test] - fn coarse_groupings_of_one_finest_build_roll_up_together() { - let roots = chose( - &[ - ("quantile by (job) (0.9, m)", 0.01), - ("quantile by (job, instance) (0.9, m)", 0.01), - ("quantile by (job, instance, zone) (0.9, m)", 0.01), - ], - SketchAlgorithm::Kll, - ); - let rule_candidates = rollup_candidates(&roots); - assert_eq!(rule_candidates.len(), 2); - let finest = &builds(&roots[2])[0]; - assert!(rule_candidates - .iter() - .all(|candidate| Rc::ptr_eq(&candidate.finer, finest))); - assert_eq!(rollup_choices(&rule_candidates).len(), 1); - let pass2_candidates = rollup_pass2_candidates(&roots, &rule_candidates).unwrap(); - let [pass2_candidate] = pass2_candidates.as_slice() else { - panic!("one Pass 2 candidate") - }; - let rolled = pass2_candidate.roots.as_ref().unwrap(); - assert_eq!(kll_builds(rolled).len(), 1); - } - - /// A quantile over every series rolls up from the KLLs by job: a - /// rollup to no keys. - #[test] - fn a_global_quantile_rolls_up_to_no_keys() { - let roots = chose( - &[ - ("quantile(0.9, m)", 0.01), - ("quantile by (job) (0.9, m)", 0.01), - ], - SketchAlgorithm::Kll, - ); - let rule_candidates = rollup_candidates(&roots); - let [rule_candidate] = rule_candidates.as_slice() else { - panic!("one candidate: {rule_candidates:?}") - }; - assert!(matches!( - rule_candidate.rollup.asap(), - Some(ASAPOp::SummaryMerge { group_by: Some(keys), .. }) if keys.is_empty() - )); - let pass2 = rollup_pass2_candidates(&roots, &rule_candidates).unwrap(); - let rolled = pass2[0].roots.as_ref().unwrap(); - assert_eq!(kll_builds(rolled).len(), 1); - operator(&rolled[0]).validate_structure().unwrap(); - } - - /// Groupings that are not nested: `by (job)` rolls up from `by (job, - /// instance)` or `by (job, zone)`; the source is the one whose sorted - /// key names come first, whatever the query order. - #[test] - fn the_rollup_source_does_not_depend_on_query_order() { - for queries in [ - [ - "quantile by (job) (0.9, m)", - "quantile by (job, zone) (0.9, m)", - "quantile by (job, instance) (0.9, m)", - ], - [ - "quantile by (job) (0.9, m)", - "quantile by (job, instance) (0.9, m)", - "quantile by (job, zone) (0.9, m)", - ], - ] { - let roots = chose(&queries.map(|q| (q, 0.01)), SketchAlgorithm::Kll); - let rule_candidates = rollup_candidates(&roots); - let [rule_candidate] = rule_candidates.as_slice() else { - panic!("one candidate: {rule_candidates:?}") - }; - let instance = queries.iter().position(|q| q.contains("instance")).unwrap(); - assert!(Rc::ptr_eq( - &rule_candidate.finer, - &builds(&roots[instance])[0] - )); - } - } - - /// A rollup needs the same rows: a coarse KLL over another metric is - /// none. - #[test] - fn sketch_rollup_needs_the_same_rows() { - let other_metric = chose( - &[ - ("quantile by (job) (0.9, n)", 0.01), - ("quantile by (job, instance) (0.9, m)", 0.01), - ], - SketchAlgorithm::Kll, - ); - assert!(rollup_candidates(&other_metric).is_empty()); - } - - /// Top-k over other partitions or inputs, or of one `k`, share no - /// ranking; a top-k a heap sketch answers is not reused. - #[test] - fn topk_reuse_needs_exact_topk_of_one_input_and_partition() { - for queries in [ - &[ - "topk(5, sum by (job) (m))", - "topk by (job) (10, sum by (job, instance) (m))", - ][..], - &["topk(5, sum by (job) (m))", "topk(10, sum by (job) (n))"], - &["topk(5, sum by (job) (m))", "topk(5, max by (job) (m))"], - &[ - "topk(5, sum by (job) (m))", - "topk by (job) (10, sum by (job) (m))", - ], - ] { - assert!( - topk_reuse_candidates(&exact(queries)).is_empty(), - "{queries:?}" - ); - } - use crate::pass1::per_computation_candidates::{ - compose_logical_candidate, generate_per_computation_candidates, - }; - use asap_types::stages::HowToComputeAggregateMeasure as C; - let inventory = generate_per_computation_candidates( - &crate::stage0::generate_dag_of_logical_memo_groups( - &[ - "topk by (job) (5, sum_over_time(m[1m]))", - "topk by (job) (10, sum_over_time(m[1m]))", - ] - .iter() - .map(|q| { - let root = lower_promql( - q, - asap_types::ir::properties::AccuracyTarget::max_error(0.01) - .failure_probability(0.01), - ); - QueryRoot::Operator(root) - }) - .collect::>(), - ), - ) - .unwrap(); - // The CountSketch heap, the top-k's fused whole-expression one. - let heap = |c: &C| matches!(c, C::Summary(SketchAlgorithm::CountSketchWithHeap)); - let choice: Vec = inventory - .targets - .iter() - .map(|t| { - (0..t.choice_count()) - .position(|i| t.computation_for(i).is_some_and(|(c, _)| heap(c))) - .unwrap_or(0) - }) - .collect(); - let heaps: Vec = compose_logical_candidate(&inventory, &choice).unwrap(); - assert!(OperatorNode::reachable(operator(&heaps[0])) - .iter() - .any(|n| matches!(n.asap(), Some(ASAPOp::SummaryEstimate { .. })))); - assert!(topk_reuse_candidates(&heaps).is_empty()); - } -} diff --git a/crates/logical-asap-optimization/src/pass2/finer_aggregate.rs b/crates/logical-asap-optimization/src/pass2/finer_aggregate.rs deleted file mode 100644 index df857c35c..000000000 --- a/crates/logical-asap-optimization/src/pass2/finer_aggregate.rs +++ /dev/null @@ -1,146 +0,0 @@ -//! Whether a query aggregate can be computed from a finer aggregate over the -//! same source and window, and with what on top (view matching for -//! aggregation queries, as in *Optimizing Queries Using Materialized Views: -//! A Practical, Scalable Solution*, SIGMOD 2001, §3.3, and Apache Calcite's -//! `SubstitutionVisitor.unifyAggregates`). -//! -//! The finer result holds at least the rows the query needs. A -//! *compensating* operation closes the gap: a compensating filter removes -//! the extra rows, and a rollup merges finer groups into the query's. A -//! filter that reads only finer group-by columns is constant within a finer -//! group, so filtering finer groups equals filtering input rows. -use asap_types::ir::{NonASAPOp, Operator, OperatorNode, Predicate, ScalarExpr}; - -use asap_accuracy::propagation::{pinned_column, split_and}; - -/// How a query aggregate is computed from a finer one: keep the finer rows -/// that pass `compensating_filter`, then, when `needs_rollup`, merge them -/// into the query's groups. -#[derive(Debug, Clone, PartialEq)] -pub(crate) struct FinerAggregateMatch { - /// The query's conjuncts that the finer aggregate lacks, over the finer - /// group-by (column `i` is the finer aggregate's `i`-th key); `None` - /// when the finer aggregate has every conjunct. - pub compensating_filter: Option, - /// Each query group key, as an index into the finer group-by. - pub query_keys: Vec, - /// Whether finer groups merge into query groups: a finer key the query - /// drops is not pinned by an equality in `compensating_filter`. When - /// `false`, each query group is at most one finer row. - pub needs_rollup: bool, -} - -/// How a query aggregate over `query_rows` grouped by `query_keys` is -/// computed from a finer aggregate over `finer_rows` grouped by -/// `finer_keys` (keys are columns of the rows); `None` when it cannot be. -/// -/// Both rows must be the same scan (source and schema) under the same time -/// ranges; the finer conjuncts (scan predicates and filters) must be among -/// the query's; every other query conjunct must read only finer group-by -/// columns; and the query keys must be finer keys. -pub(crate) fn match_finer( - query_rows: &OperatorNode, - query_keys: &[usize], - finer_rows: &OperatorNode, - finer_keys: &[usize], -) -> Option { - let query = Rows::of(query_rows)?; - let finer = Rows::of(finer_rows)?; - if query.spine != finer.spine { - return None; - } - let mut extra = query.conjuncts; - for conjunct in finer.conjuncts { - let position = extra.iter().position(|other| *other == conjunct)?; - extra.remove(position); - } - let finer_key = |column: usize| finer_keys.iter().position(|&key| key == column); - let query_key_positions = query_keys - .iter() - .map(|&key| finer_key(key)) - .collect::>>()?; - if !extra.iter().all(|conjunct| { - conjunct.operator_refs().is_empty() - && conjunct - .columns_referenced() - .into_iter() - .all(|column| finer_key(column).is_some()) - }) { - return None; - } - let pinned: Vec = extra.iter().filter_map(pinned_column).collect(); - let needs_rollup = finer_keys - .iter() - .any(|key| !query_keys.contains(key) && !pinned.contains(key)); - // Without extra conjuncts or a rollup, the two aggregates have the same - // groups, perhaps in another key order: they are one computation, not - // one read from the other (each read from the other would be a cycle). - if extra.is_empty() && !needs_rollup { - return None; - } - let compensating_filter = (!extra.is_empty()).then(|| { - let remapped: Vec = extra - .iter() - .map(|conjunct| conjunct.map_columns(&mut |column| finer_key(column).unwrap())) - .collect(); - Predicate(match <[ScalarExpr; 1]>::try_from(remapped) { - Ok([one]) => one, - Err(many) => ScalarExpr::BoolAnd(many), - }) - }); - Some(FinerAggregateMatch { - compensating_filter, - query_keys: query_key_positions, - needs_rollup, - }) -} - -/// An aggregate's input rows: the scan and time ranges they come from, and -/// the conjuncts that select them. -struct Rows { - /// Each time range above the scan, outermost first, then the scan - /// without its predicates. - spine: Vec>, - /// Scan predicates and filter predicates, split at `AND`. - conjuncts: Vec, -} - -impl Rows { - /// The rows of `node`, a chain of `TimeRange`s and `Filter`s over a - /// `Scan`, none changing the scan's columns; `None` for anything else. - fn of(node: &OperatorNode) -> Option { - let mut rows = Rows { - spine: Vec::new(), - conjuncts: Vec::new(), - }; - let mut node = node; - loop { - match node.non_asap()? { - NonASAPOp::TimeRange { child, .. } if child.schema == node.schema => { - rows.spine.push(node.operator.map_children(|_| ())); - node = child; - } - NonASAPOp::Filter { pred, child } if child.schema == node.schema => { - split_and(&pred.0, &mut rows.conjuncts); - node = child; - } - NonASAPOp::Scan { - source, - schema, - predicates, - } => { - for predicate in predicates { - split_and(&predicate.0, &mut rows.conjuncts); - } - rows.spine.push(Operator::NonASAP(NonASAPOp::Scan { - source: source.clone(), - schema: schema.clone(), - predicates: Vec::new(), - })); - return Some(rows); - } - _ => return None, - } - } - } -} diff --git a/crates/logical-asap-optimization/src/pass2/logical_memo_candidates.rs b/crates/logical-asap-optimization/src/pass2/logical_memo_candidates.rs index 678ba5c0d..02fa8f6c5 100644 --- a/crates/logical-asap-optimization/src/pass2/logical_memo_candidates.rs +++ b/crates/logical-asap-optimization/src/pass2/logical_memo_candidates.rs @@ -1,71 +1,53 @@ -//! Stage 1, Pass 2's entry and output: the Pass 1 memo extended with the -//! candidates and groups that share work across computations -//! ([`asap_types::stages::LogicalASAPMemoCandidates`]), which Stage 2 implements. +//! Stage 1, Pass 2's entry and output: the Pass 1 memo extended by the +//! rules over groups, and the logical edges +//! ([`asap_types::stages::LogicalASAPMemoCandidates`]), which Stage 2 expands. use crate::pass2::memo::rules::{ - finer_aggregate, rollup, scan_labels, shared_configuration_readers, summary_inputs, - summary_results, topk, -}; -use crate::pass2::memo::state_groups::{ - add_shared_summary_state_groups, check_unexpanded_memo, Pass2RuleError, + finer_aggregate, rollup, scan_labels, summary_inputs, topk, Pass2RuleError, }; use crate::pass2::memo::window_candidates::add_local_window_composition; -use asap_types::stages::Memo; -use asap_types::stages::{LogicalASAPMemoCandidates, PerComputationLogicalMemoCandidates}; +use asap_types::stages::{LogicalASAPMemoCandidates, Memo}; use asap_types::workload::QueryWorkloadEntry; -/// Stage 1, Pass 2: adds to Pass 1's memo the local tumbling-window -/// candidates for every legal tumbling-window width (rule 1), then runs -/// rules 2–9 and `add_shared_summary_state_groups`. +/// Stage 1, Pass 2: rule 1 ([`pass2_local_window_composition`]), then rules +/// 2–7 and the logical edges ([`pass2_after_local_window_composition`]). /// `entries[i]` is query `i`'s workload entry. pub fn generate_logical_asap_memo_candidates( - pass1: PerComputationLogicalMemoCandidates, + pass1: Memo, entries: &[QueryWorkloadEntry], ) -> Result { - pass2_after_local_window_composition(pass2_local_window_composition(pass1, entries)) + pass2_after_local_window_composition(pass2_local_window_composition(pass1, entries)?) } -/// Pass 2's rule 1 alone: Pass 1's memo with the local tumbling-window -/// candidates for every legal width (`asap_planner` prints this step). +/// Pass 2's rule 1 alone: Pass 1's memo with the tumbling-window layouts +/// of each summary state group (`asap_planner` prints this step). pub fn pass2_local_window_composition( - pass1: PerComputationLogicalMemoCandidates, + pass1: Memo, entries: &[QueryWorkloadEntry], -) -> Memo { - let mut memo = pass1.memo; +) -> Result { + let mut memo = pass1; // 1. Local window composition. - add_local_window_composition(&mut memo, entries); - memo + add_local_window_composition(&mut memo, entries)?; + Ok(memo) } /// The rest of Pass 2 on [`pass2_local_window_composition`]'s memo: rules -/// 2–9 and `add_shared_summary_state_groups`. +/// 2–7, then the logical edges with their pruning caps. pub fn pass2_after_local_window_composition( mut memo: Memo, ) -> Result { - apply_sharing_rules(&mut memo)?; - // 10. One group per shared summary build, read by its reader groups. - add_shared_summary_state_groups(&mut memo)?; - Ok(LogicalASAPMemoCandidates { memo }) -} - -/// Pass 2's rules 2–9, in order, on a memo after local window composition. -pub(crate) fn apply_sharing_rules(memo: &mut Memo) -> Result<(), Pass2RuleError> { - check_unexpanded_memo(memo)?; // 2. Scan widening. - scan_labels::add_scan_label_candidates(memo)?; + scan_labels::add_scan_label_candidates(&mut memo)?; // 3. Summary inputs. - summary_inputs::factor_summary_inputs(memo)?; + summary_inputs::factor_summary_inputs(&mut memo)?; // 4. Top-k reuse. - topk::add_topk_candidates(memo)?; + topk::add_topk_candidates(&mut memo)?; // 5. Finer aggregate. - finer_aggregate::add_finer_aggregate_candidates(memo)?; + finer_aggregate::add_finer_aggregate_candidates(&mut memo)?; // 6. Rollup. - rollup::add_rollup_candidates(memo)?; + rollup::add_rollup_candidates(&mut memo)?; // 7. Window rollup. - rollup::add_window_rollup_candidates(memo)?; - // 8. Summary results. - summary_results::factor_summary_results(memo)?; - // 9. Shared configurations. - shared_configuration_readers::add_state_configuration_candidates(memo)?; - Ok(()) + rollup::add_window_rollup_candidates(&mut memo)?; + let _ = memo; + todo!("#1207 PR 4: the logical edges, each input's floor tightened by its pruning cap") } diff --git a/crates/logical-asap-optimization/src/pass2/memo.rs b/crates/logical-asap-optimization/src/pass2/memo.rs index c9bc66a80..754a06a5c 100644 --- a/crates/logical-asap-optimization/src/pass2/memo.rs +++ b/crates/logical-asap-optimization/src/pass2/memo.rs @@ -1,12 +1,6 @@ -//! The logical memo's candidates, rules and enumeration, over the memo -//! groups Stage 0 builds (`crate::stage0`). +//! The logical memo's rules over the memo groups Stage 0 builds +//! (`crate::stage0`). -/// The candidate DAGs a memo stands for: choices and their composition. -pub(crate) mod enumeration; pub(crate) mod rules; -/// Shared finite state configuration enumeration. -pub(crate) mod shared_configurations; -/// Concrete summary-state groups exposed by sharing rules. -pub(crate) mod state_groups; -/// Local tumbling-window candidates with selected-reader grid configurations. +/// Rule 1: tumbling-window layouts of a summary state group. pub(crate) mod window_candidates; diff --git a/crates/logical-asap-optimization/src/pass2/memo/enumeration.rs b/crates/logical-asap-optimization/src/pass2/memo/enumeration.rs deleted file mode 100644 index d849a5e75..000000000 --- a/crates/logical-asap-optimization/src/pass2/memo/enumeration.rs +++ /dev/null @@ -1,236 +0,0 @@ -//! The candidate DAGs a memo stands for: each choice of one implementation -//! candidate per needed group, and the complete DAG that choice composes. -//! -//! A group is needed when a query's root is in it or a chosen candidate reads -//! it. A group read by several chosen candidates takes one candidate, so its -//! nodes are built once and shared; which groups are needed depends on the -//! candidates chosen above them. The count is therefore not a product over -//! groups. -//! -//! A choice is counted, not the DAG it composes: two candidates of a group -//! that differ only in what recomposing guarantees re-derives compose equal -//! DAGs. Composing checks only that a choice composes; input contracts, -//! accuracy at the roots, Stage 2 legality and Stage 3 cost stay with the -//! validator and the later stages. The memo's groups are acyclic, as every -//! memo step builds them, but candidates may read each other (a rollup of -//! a finer grouping whose candidate reads the coarser one): a choice taking -//! such candidates together composes no DAG and is not enumerated. - -use std::collections::{BTreeMap, HashMap, HashSet}; -use std::rc::Rc; - -use asap_types::ir::{OperatorNode, QueryRoot, SchemaDerivationError}; - -use super::rules::reuse::replace; -use asap_accuracy::propagation::recompose_guarantees; -use asap_types::stages::Memo; -use asap_types::stages::MemoGroupId; - -/// One candidate (an index into `Memo::candidates[group]`) for each -/// group it names. [`build_logical_asap_dags_from_chosen_candidates`] gives a group it does not name its first -/// candidate. -pub type ChosenCandidatePerMemoGroup = BTreeMap; - -/// The choices [`enumerate`] found, in depth-first order. -#[derive(Debug, Clone, Default, PartialEq, Eq)] -pub struct AllChosenCandidatesPerMemoGroup { - /// Each choice names exactly the groups it needs. - pub all_chosen_candidates: Vec, - /// False when the limit stopped the enumeration before every choice was found. - pub complete: bool, -} - -/// A choice that does not compose. -#[derive(Debug, thiserror::Error)] -pub enum ErrorBuildingLogicalASAPDagsFromChosenCandidates { - #[error("group {group:?} has no candidate {candidate}")] - NoSuchCandidate { - group: MemoGroupId, - candidate: usize, - }, - #[error("group {group:?}'s candidate {candidate} reads group {child:?} but has no node of it to replace")] - ChildNotInCandidate { - group: MemoGroupId, - candidate: usize, - child: MemoGroupId, - }, - /// The chosen candidates read each other in a cycle (e.g. a coarse - /// group's rollup reads a finer group whose chosen candidate reads it). - #[error("group {group:?} reads itself through the chosen candidates")] - Cycle { group: MemoGroupId }, - /// Re-deriving the composed nodes' schemas and guarantees failed. - #[error("recomposing the composed DAG: {0}")] - Recompose(#[from] SchemaDerivationError), -} - -/// Every choice of one candidate per needed group, starting from the query -/// roots, stopping after `limit` choices. A choice whose candidates read -/// each other in a cycle composes no DAG and is left out. -pub fn enumerate(memo: &Memo, limit: usize) -> AllChosenCandidatesPerMemoGroup { - fn walk( - memo: &Memo, - mut pending: Vec, - chosen_candidates: &mut ChosenCandidatePerMemoGroup, - limit: usize, - all_chosen_candidates: &mut Vec, - ) -> bool { - let group = loop { - match pending.pop() { - None => { - if all_chosen_candidates.len() == limit { - return false; - } - all_chosen_candidates.push(chosen_candidates.clone()); - return true; - } - Some(group) if !chosen_candidates.contains_key(&group) => break group, - Some(_) => {} - } - }; - for (index, candidate) in memo.candidates[group.0 as usize].iter().enumerate() { - // A candidate reading a group that already reads this one closes - // a cycle; every edge is checked as its group is chosen. - if candidate - .input_groups - .iter() - .any(|&child| reaches(memo, chosen_candidates, child, group)) - { - continue; - } - chosen_candidates.insert(group, index); - let mut next = pending.clone(); - next.extend(candidate.input_groups.iter().rev()); - let finished = walk(memo, next, chosen_candidates, limit, all_chosen_candidates); - chosen_candidates.remove(&group); - if !finished { - return false; - } - } - true - } - let roots = root_groups(memo).rev().collect(); - let mut all_chosen_candidates = Vec::new(); - let complete = walk( - memo, - roots, - &mut ChosenCandidatePerMemoGroup::new(), - limit, - &mut all_chosen_candidates, - ); - AllChosenCandidatesPerMemoGroup { - all_chosen_candidates, - complete, - } -} - -/// Each root group (one with a query's root) and its DAG under `chosen_candidates`: -/// each needed group implemented by its chosen candidate over its children's -/// composed DAGs, a group read several times composed once and shared. Every -/// node is then rebuilt with its derived schema and composed guarantee, as -/// in plan reconstruction. A scalar query's expression is not rebuilt: its -/// operator roots are root groups like any other query's. -pub fn build_logical_asap_dags_from_chosen_candidates( - memo: &Memo, - chosen_candidates: &ChosenCandidatePerMemoGroup, -) -> Result)>, ErrorBuildingLogicalASAPDagsFromChosenCandidates> -{ - fn composed( - memo: &Memo, - group: MemoGroupId, - chosen_candidates: &ChosenCandidatePerMemoGroup, - done: &mut HashMap>, - composing: &mut HashSet, - ) -> Result, ErrorBuildingLogicalASAPDagsFromChosenCandidates> { - if let Some(found) = done.get(&group) { - return Ok(Rc::clone(found)); - } - if !composing.insert(group) { - return Err(ErrorBuildingLogicalASAPDagsFromChosenCandidates::Cycle { group }); - } - let index = chosen_candidates.get(&group).copied().unwrap_or(0); - let candidate = memo.candidates[group.0 as usize].get(index).ok_or( - ErrorBuildingLogicalASAPDagsFromChosenCandidates::NoSuchCandidate { - group, - candidate: index, - }, - )?; - let own_nodes: HashSet<_> = OperatorNode::reachable(&candidate.root_node) - .iter() - .map(Rc::as_ptr) - .collect(); - let mut replacements = HashMap::new(); - for &child in &candidate.input_groups { - let old = &memo.groups[child.0 as usize].logical_node; - if !own_nodes.contains(&Rc::as_ptr(old)) { - return Err( - ErrorBuildingLogicalASAPDagsFromChosenCandidates::ChildNotInCandidate { - group, - candidate: index, - child, - }, - ); - } - replacements.insert( - Rc::as_ptr(old), - composed(memo, child, chosen_candidates, done, composing)?, - ); - } - let node = replace(&candidate.root_node, &replacements); - done.insert(group, Rc::clone(&node)); - Ok(node) - } - let mut done = HashMap::new(); - let mut composing = HashSet::new(); - let (groups, roots): (Vec<_>, Vec<_>) = root_groups(memo) - .map(|group| { - Ok(( - group, - QueryRoot::Operator(composed( - memo, - group, - chosen_candidates, - &mut done, - &mut composing, - )?), - )) - }) - .collect::, ErrorBuildingLogicalASAPDagsFromChosenCandidates>>()? - .into_iter() - .unzip(); - let roots = recompose_guarantees(&roots)?; - Ok(groups - .into_iter() - .zip(roots) - .map(|(group, root)| match root { - QueryRoot::Operator(node) => (group, node), - QueryRoot::Scalar(_) => unreachable!("composed from operator roots"), - }) - .collect()) -} - -/// Whether `from` is `to` or reads it through the candidates `chosen_candidates` names. -fn reaches( - memo: &Memo, - chosen_candidates: &ChosenCandidatePerMemoGroup, - from: MemoGroupId, - to: MemoGroupId, -) -> bool { - let mut pending = vec![from]; - let mut seen = HashSet::new(); - while let Some(group) = pending.pop() { - if group == to { - return true; - } - if !seen.insert(group) { - continue; - } - if let Some(&index) = chosen_candidates.get(&group) { - pending.extend(&memo.candidates[group.0 as usize][index].input_groups); - } - } - false -} - -fn root_groups(memo: &Memo) -> impl DoubleEndedIterator { - asap_types::stages::root_groups(&memo.query_roots).into_iter() -} diff --git a/crates/logical-asap-optimization/src/pass2/memo/rules.rs b/crates/logical-asap-optimization/src/pass2/memo/rules.rs index e8eec23f0..c9b38ed85 100644 --- a/crates/logical-asap-optimization/src/pass2/memo/rules.rs +++ b/crates/logical-asap-optimization/src/pass2/memo/rules.rs @@ -1,11 +1,19 @@ -//! Bounded logical memo rule expansion. Rules expose computations and typed -//! dependencies; physical configuration and cost selection live downstream. +//! Pass 2's rules over groups (concepts doc, section 3): each reads the +//! memo and adds groups (`Memo::find_or_add_group`) or candidates +//! (`Memo::add_candidate`); none copies a consumer. pub(crate) mod finer_aggregate; -pub(super) mod reuse; pub(crate) mod rollup; pub(crate) mod scan_labels; -pub(crate) mod shared_configuration_readers; -pub(super) mod shared_producer; pub(crate) mod summary_inputs; -pub(crate) mod summary_results; pub(crate) mod topk; + +/// Why a Pass 2 rule failed on a memo. +#[derive(Debug, thiserror::Error)] +pub enum Pass2RuleError { + #[error("memo contains an out-of-range group reference")] + InvalidGroupReference, + #[error("memo group capacity exceeds the group identifier range")] + TooManyGroups, + #[error("memo dependencies contain a cycle")] + CyclicDependencies, +} diff --git a/crates/logical-asap-optimization/src/pass2/memo/rules/finer_aggregate.rs b/crates/logical-asap-optimization/src/pass2/memo/rules/finer_aggregate.rs index 78aee9654..95c8c7c0a 100644 --- a/crates/logical-asap-optimization/src/pass2/memo/rules/finer_aggregate.rs +++ b/crates/logical-asap-optimization/src/pass2/memo/rules/finer_aggregate.rs @@ -1,177 +1,12 @@ //! Read a SQL aggregate from a finer aggregate's result //! ([`match_finer`](crate::pass2::finer_aggregate::match_finer)): filter the finer //! rows, then merge them into the query's groups with each measure's rollup. -use super::reuse::append_reader; -use crate::pass2::finer_aggregate::match_finer; -use crate::pass2::memo::state_groups::Pass2RuleError; -use asap_types::ir::operator::{AggIntent, Reduction}; -use asap_types::ir::{NonASAPOp, Operator, OperatorNode, OperatorResultKind}; -use asap_types::stages::{Memo, OriginOfCandidateForMemoGroup}; -use std::collections::HashMap; -use std::rc::Rc; -/// A memo group's SQL aggregate: no `without`, measure filter or `HAVING`. -struct Aggregate<'a> { - keys: &'a [usize], - measures: &'a [AggIntent], - output_names: &'a [String], - rows: &'a Rc, -} - -impl<'a> Aggregate<'a> { - fn of(node: &'a OperatorNode) -> Option { - if node.result_kind != OperatorResultKind::Relation { - return None; - } - let NonASAPOp::Aggregate { - reduction: Reduction::Reduce(by), - measures, - output_names, - filters, - having: None, - child, - } = node.non_asap()? - else { - return None; - }; - // A same-level read takes the one row of each group, so a measure - // that gives several rows per group (a top-k, a count of values) - // does not qualify. - let one_row_per_group = !measures.iter().any(|measure| { - matches!( - measure, - AggIntent::TopK { .. } | AggIntent::CountValues { .. } - ) - }); - (!by.is_without() && filters.iter().all(Option::is_none) && one_row_per_group).then_some( - Self { - keys: by.keys(), - measures, - output_names, - rows: child, - }, - ) - } -} - -/// The measure that merges finer values of `measure` (at `col` of the finer -/// result) into a coarser group (Calcite's `SqlAggFunction.getRollup`); -/// `None` when its values do not roll up. A count rolls up as a sum only -/// into grouped queries: a global count of no rows is 0, a sum NULL. -fn value_rollup(measure: &AggIntent, col: usize, global: bool) -> Option { - match measure { - AggIntent::Sum { .. } => Some(AggIntent::Sum { col: Some(col) }), - AggIntent::Count { .. } if !global => Some(AggIntent::Sum { col: Some(col) }), - AggIntent::Min { .. } => Some(AggIntent::Min { col: Some(col) }), - AggIntent::Max { .. } => Some(AggIntent::Max { col: Some(col) }), - _ => None, - } -} - -pub(crate) fn add_finer_aggregate_candidates(memo: &mut Memo) -> Result { - // Only aggregates of equal measures can match; bucket by them (the - // first levels of the paper's filter tree) instead of trying all pairs. - let mut buckets: HashMap> = HashMap::new(); - for (group, entry) in memo.groups.iter().enumerate() { - if let Some(aggregate) = Aggregate::of(&entry.logical_node) { - buckets - .entry(format!("{:?}", aggregate.measures)) - .or_default() - .push(group); - } - } - let mut added = 0; - for groups in buckets.values() { - for &owner in groups { - for &finer in groups { - if owner != finer { - added += add_read(memo, owner, finer)?; - } - } - } - } - Ok(added) -} +use crate::pass2::memo::rules::Pass2RuleError; +use asap_types::stages::Memo; -/// Adds to group `owner` a read of group `finer`'s result when it computes -/// `owner`'s aggregate; the number of candidates added (0 or 1). -fn add_read(memo: &mut Memo, owner: usize, finer: usize) -> Result { - let query_node = Rc::clone(&memo.groups[owner].logical_node); - let finer_node = Rc::clone(&memo.groups[finer].logical_node); - let (Some(query), Some(fine)) = (Aggregate::of(&query_node), Aggregate::of(&finer_node)) else { - return Ok(0); - }; - let Some(matched) = match_finer(query.rows, query.keys, fine.rows, fine.keys) else { - return Ok(0); - }; - let global = query.keys.is_empty(); - // A finer measure's value column follows the finer keys. When each query - // group is at most one finer row, `MIN` reads that row's value, and over - // no row gives a global query SQL's one NULL row. - let measures = query - .measures - .iter() - .enumerate() - .map(|(index, measure)| { - let col = fine.keys.len() + index; - if !matched.needs_rollup && !(global && matches!(measure, AggIntent::Count { .. })) { - Some(AggIntent::Min { col: Some(col) }) - } else if matched.needs_rollup { - value_rollup(measure, col, global) - } else { - None - } - }) - .collect::>>(); - let Some(measures) = measures else { - return Ok(0); - }; - let reuse_error = - |error: asap_types::ir::SchemaDerivationError| Pass2RuleError::ReuseRule(error.to_string()); - let rows = match matched.compensating_filter { - Some(pred) => OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Filter { - pred, - child: Rc::clone(&finer_node), - })) - .map_err(reuse_error)?, - None => Rc::clone(&finer_node), - }; - let reader = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Aggregate { - reduction: Reduction::by(matched.query_keys), - measures, - output_names: query.output_names.to_vec(), - filters: vec![], - having: None, - child: rows, - })) - .map_err(reuse_error)?; - let Some(original) = memo.candidates[owner] - .iter() - .find(|c| c.origin == OriginOfCandidateForMemoGroup::Pass1) - .cloned() - else { - return Ok(0); - }; - // The reader's columns take the query's qualifiers. A rollup whose type - // or nullability differs from the query's (a sum of counts) is not - // offered. - let mut schema = reader.schema.clone(); - for (field, expected) in schema - .fields - .iter_mut() - .zip(&original.root_node.schema.fields) - { - field.table = expected.table.clone(); - } - if schema != original.root_node.schema || reader.result_kind != original.root_node.result_kind { - return Ok(0); - } - let reader = Rc::new(OperatorNode::with_schema(reader.operator.clone(), schema)); - Ok(usize::from(append_reader( - memo, - owner, - &original, - reader, - OriginOfCandidateForMemoGroup::Rollup, - )?)) +/// Rule 5, finer aggregate: a candidate of a coarse group that filters and rolls up a finer group. +pub(crate) fn add_finer_aggregate_candidates(memo: &mut Memo) -> Result<(), Pass2RuleError> { + let _ = memo; + todo!("#1207 PR 4: rule 5 over groups") } diff --git a/crates/logical-asap-optimization/src/pass2/memo/rules/reuse.rs b/crates/logical-asap-optimization/src/pass2/memo/rules/reuse.rs deleted file mode 100644 index 5eac3feb1..000000000 --- a/crates/logical-asap-optimization/src/pass2/memo/rules/reuse.rs +++ /dev/null @@ -1,227 +0,0 @@ -//! Shared boundary handling for producer-level reuse rules. Rule outputs retain -//! canonical input identities after guarantee recomposition rebuilds their DAGs. -use crate::pass2::memo::state_groups::{boundary_groups, Pass2RuleError}; -use asap_types::ir::OperatorNode; -use asap_types::stages::{CandidateForMemoGroup, Memo, OriginOfCandidateForMemoGroup}; -use asap_types::stages::{MemoGroup, MemoGroupId}; -use std::collections::{HashMap, HashSet}; -use std::rc::Rc; - -pub(in crate::pass2::memo) fn eligible(candidate: &CandidateForMemoGroup) -> bool { - candidate.accuracy_budget_parts > 0 -} - -pub(in crate::pass2::memo) fn restore_inputs( - node: &Rc, - memo: &Memo, - inputs: &[MemoGroupId], -) -> Rc { - // These are the same declared logical input groups, rebound after a rule - // cloned the DAG. Value guarantees are recomposed from selected producers - // during reconstruction; state inputs additionally require the same error - // contract because the intermediate state's guarantee is part of its API. - let canonical: Vec<_> = inputs - .iter() - .flat_map(|id| OperatorNode::reachable(&memo.groups[id.0 as usize].logical_node)) - .collect(); - fn visit( - node: &Rc, - canonical: &[Rc], - done: &mut HashMap<*const OperatorNode, Rc>, - ) -> Rc { - if let Some(found) = done.get(&Rc::as_ptr(node)) { - return Rc::clone(found); - } - let operator = node - .operator - .map_children(|child| visit(child, canonical, done)); - let rebuilt = Rc::new(OperatorNode { - operator, - ..(**node).clone() - }); - let result = canonical - .iter() - .find(|known| { - same_node( - known, - &rebuilt, - known.result_kind == asap_types::ir::OperatorResultKind::State, - ) - }) - .cloned() - .unwrap_or(rebuilt); - done.insert(Rc::as_ptr(node), Rc::clone(&result)); - result - } - visit(node, &canonical, &mut HashMap::new()) -} - -pub(in crate::pass2::memo) fn same_node( - left: &OperatorNode, - right: &OperatorNode, - guarantees: bool, -) -> bool { - left.operator.map_children(|_| ()) == right.operator.map_children(|_| ()) - && left.schema == right.schema - && left.result_kind == right.result_kind - && left.timing == right.timing - && left.materialization == right.materialization - && left.children().len() == right.children().len() - && left - .children() - .iter() - .zip(right.children()) - .all(|(a, b)| Rc::ptr_eq(a, b)) - && serde_json::to_string(&( - left.operator.map_children(|_| ()), - &left.schema, - left.result_kind, - left.timing, - left.materialization, - )) - .ok() - == serde_json::to_string(&( - right.operator.map_children(|_| ()), - &right.schema, - right.result_kind, - right.timing, - right.materialization, - )) - .ok() - && (!guarantees || left.guarantee == right.guarantee) -} - -pub(in crate::pass2::memo) fn append_producer( - memo: &mut Memo, - node: Rc, - queries: &[usize], - origin: OriginOfCandidateForMemoGroup, - accuracy_budget_parts: usize, -) -> Result { - // Reuse only dedicated producer groups. A logical result group can contain - // other implementations and cannot enforce this producer's exact contract. - let root_groups = asap_types::stages::root_groups(&memo.query_roots); - if let Some((index, _)) = memo.groups.iter().enumerate().find(|(index, group)| { - !root_groups.contains(&MemoGroupId(*index as u32)) - && memo.candidates[*index].len() == 1 - && memo.candidates[*index][0].origin == origin - && memo.candidates[*index][0].accuracy_budget_parts == accuracy_budget_parts - && same_node(&group.logical_node, &node, true) - }) { - return Ok(MemoGroupId(index as u32)); - } - let id = - MemoGroupId(u32::try_from(memo.groups.len()).map_err(|_| Pass2RuleError::TooManyGroups)?); - let node = Rc::new((*node).clone()); - let boundaries = memo - .groups - .iter() - .enumerate() - .map(|(id, group)| (Rc::as_ptr(&group.logical_node), MemoGroupId(id as u32))) - .collect(); - let children = boundary_groups(&node, &boundaries); - memo.groups.push(MemoGroup { - logical_node: Rc::clone(&node), - child_groups: children.clone(), - parents: Vec::new(), - queries: queries.to_vec(), - }); - memo.candidates.push(vec![CandidateForMemoGroup { - fused_groups: Vec::new(), - root_node: node, - input_groups: children, - origin, - accuracy_budget_parts, - }]); - Ok(id) -} - -pub(in crate::pass2::memo) fn append_reader( - memo: &mut Memo, - owner: usize, - original: &CandidateForMemoGroup, - node: Rc, - origin: OriginOfCandidateForMemoGroup, -) -> Result { - if original.root_node.schema.fields != node.schema.fields - || original.root_node.schema.time_index != node.schema.time_index - || original.root_node.schema.unique_keys != node.schema.unique_keys - || (original.root_node.schema.closed && !node.schema.closed) - || original.root_node.result_kind != node.result_kind - { - return Err(Pass2RuleError::ReuseRule(format!( - "reuse changed the reader output contract ({origin:?}): expected {:?}, got {:?}", - original.root_node.schema, node.schema - ))); - } - let boundaries = memo - .groups - .iter() - .enumerate() - .filter(|(id, _)| *id != owner) - .map(|(id, group)| (Rc::as_ptr(&group.logical_node), MemoGroupId(id as u32))) - .collect(); - let children = boundary_groups(&node, &boundaries); - if memo.candidates[owner].iter().any(|known| { - known.input_groups == children - && known.accuracy_budget_parts == original.accuracy_budget_parts - && *known.root_node == *node - && serde_json::to_string(&known.root_node).ok() == serde_json::to_string(&node).ok() - }) { - return Ok(false); - } - let mut candidate = original.clone(); - candidate.root_node = node; - candidate.input_groups = children; - candidate.origin = origin; - memo.candidates[owner].push(candidate); - Ok(true) -} - -pub(in crate::pass2::memo) fn replace( - node: &Rc, - replacements: &HashMap<*const OperatorNode, Rc>, -) -> Rc { - fn visit( - node: &Rc, - replacements: &HashMap<*const OperatorNode, Rc>, - done: &mut HashMap<*const OperatorNode, Rc>, - ) -> Rc { - if let Some(found) = replacements - .get(&Rc::as_ptr(node)) - .or_else(|| done.get(&Rc::as_ptr(node))) - { - return Rc::clone(found); - } - let mut changed = false; - let operator = node.operator.map_children(|child| { - let replacement = visit(child, replacements, done); - changed |= !Rc::ptr_eq(child, &replacement); - replacement - }); - let rebuilt = if changed { - Rc::new(OperatorNode { - operator, - ..(**node).clone() - }) - } else { - Rc::clone(node) - }; - done.insert(Rc::as_ptr(node), Rc::clone(&rebuilt)); - rebuilt - } - visit(node, replacements, &mut HashMap::new()) -} - -pub(in crate::pass2::memo) fn combined_inputs( - left: &CandidateForMemoGroup, - right: &CandidateForMemoGroup, -) -> Vec { - let mut seen = HashSet::new(); - left.input_groups - .iter() - .chain(&right.input_groups) - .copied() - .filter(|id| seen.insert(*id)) - .collect() -} diff --git a/crates/logical-asap-optimization/src/pass2/memo/rules/rollup.rs b/crates/logical-asap-optimization/src/pass2/memo/rules/rollup.rs index 1f684e04f..de56a825b 100644 --- a/crates/logical-asap-optimization/src/pass2/memo/rules/rollup.rs +++ b/crates/logical-asap-optimization/src/pass2/memo/rules/rollup.rs @@ -1,360 +1,18 @@ //! Producer-level exact and sketch rollups, using the existing capability rules. -use super::reuse::{ - append_producer, append_reader, combined_inputs, eligible, replace, restore_inputs, -}; -use crate::pass2::asap_aware_cse::{rollup_candidate_for, rollup_pass2_candidates}; -use crate::pass2::memo::state_groups::{simple_build, Pass2RuleError}; -use asap_types::ir::schema::FieldDataType; -use asap_types::ir::{ - ASAPOp, NonASAPOp, Operator, OperatorNode, ProjectItem, QueryRoot, ScalarExpr, -}; -use asap_types::stages::{Memo, OriginOfCandidateForMemoGroup}; -use std::collections::HashMap; -use std::rc::Rc; -pub(crate) fn add_rollup_candidates(memo: &mut Memo) -> Result { - let mut candidates = Vec::new(); - for (group, entries) in memo.candidates.iter().enumerate() { - for candidate in entries { - if eligible(candidate) - && matches!( - candidate.origin, - OriginOfCandidateForMemoGroup::Pass1 - | OriginOfCandidateForMemoGroup::WidenedScans - ) - { - if let Some((build, _)) = simple_build(memo, candidate) { - candidates.push((group, candidate.clone(), build)); - } - } - } - } - let mut added = 0; - for (coarse_group, coarse, coarse_build) in &candidates { - for (fine_group, fine, _) in &candidates { - if coarse_group == fine_group { - continue; - } - let roots = [ - QueryRoot::Operator(Rc::clone(&coarse.root_node)), - QueryRoot::Operator(Rc::clone(&fine.root_node)), - ]; - let Some(rule) = rollup_candidate_for(&roots, Rc::as_ptr(coarse_build)) else { - continue; - }; - if !OperatorNode::reachable(&fine.root_node) - .iter() - .any(|node| Rc::ptr_eq(node, &rule.finer)) - { - continue; - } - let state_rollup = OperatorNode::reachable(&rule.rollup).iter().any(|node| { - matches!( - node.asap(), - Some(ASAPOp::SummaryMerge { - group_by: Some(_), - .. - }) - ) - }); - let is_sketch = matches!( - rule.finer.asap(), - Some(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(..), - .. - }) - ); - let rewritten = rollup_pass2_candidates(&roots, &[rule]) - .map_err(|error| Pass2RuleError::ReuseRule(error.to_string()))?; - let Some(result) = rewritten.into_iter().next() else { - continue; - }; - // Existing rule API marks individually noncomposable rewrites as - // invalid candidates; retain the local implementation in that case. - let Ok(rewritten) = result.roots else { - continue; - }; - let [QueryRoot::Operator(coarse_node), QueryRoot::Operator(fine_node)] = - rewritten.as_slice() - else { - unreachable!("two operator roots") - }; - let inputs = combined_inputs(coarse, fine); - let coarse_node = restore_inputs(coarse_node, memo, &inputs); - let fine_node = restore_inputs(fine_node, memo, &inputs); - if is_sketch || state_rollup { - // Concrete resized states become ordinary state groups after all - // producer-level configurations have been expanded. - added += usize::from(append_reader( - memo, - *coarse_group, - coarse, - coarse_node, - OriginOfCandidateForMemoGroup::Rollup, - )?); - added += usize::from(append_reader( - memo, - *fine_group, - fine, - fine_node, - OriginOfCandidateForMemoGroup::Rollup, - )?); - } else { - let Some(value) = - OperatorNode::reachable(&coarse_node) - .into_iter() - .find_map(|node| match node.asap() { - Some(ASAPOp::SummaryAgg { child, .. }) - if matches!( - child.asap(), - Some(ASAPOp::FinalizeExactAccumulator { .. }) - ) => - { - Some(Rc::clone(child)) - } - _ => None, - }) - else { - continue; - }; - // When the finer query's result is the value itself and every - // implementation of it is exact, read the finer query's group: - // one stored result then serves both queries, as the - // whole-workload rewrite does, without a separate producer the - // finer query would read through an identity projection. - // Exactness is checked when this rule runs; a later - // implementation of the group is still checked by plan - // reconstruction (input contract) and Stage 3 (guarantee). - let fine_result_is_value = *fine_node == *value - && memo.groups[*fine_group].logical_node.schema == value.schema - && memo.candidates[*fine_group].iter().all(|candidate| { - candidate - .root_node - .guarantee - .as_ref() - .is_some_and(|guarantee| guarantee.is_exact()) - }); - if fine_result_is_value { - let fine_result = Rc::clone(&memo.groups[*fine_group].logical_node); - let coarse_node = replace( - &coarse_node, - &HashMap::from([(Rc::as_ptr(&value), fine_result)]), - ); - added += usize::from(append_reader( - memo, - *coarse_group, - coarse, - coarse_node, - OriginOfCandidateForMemoGroup::Rollup, - )?); - continue; - } - let mut queries = memo.groups[*coarse_group].queries.clone(); - queries.extend(&memo.groups[*fine_group].queries); - queries.sort_unstable(); - queries.dedup(); - let producer = append_producer( - memo, - Rc::clone(&value), - &queries, - OriginOfCandidateForMemoGroup::Rollup, - 1, - )?; - let canonical = Rc::clone(&memo.groups[producer.0 as usize].logical_node); - let coarse_node = replace( - &coarse_node, - &HashMap::from([(Rc::as_ptr(&value), Rc::clone(&canonical))]), - ); - added += usize::from(append_reader( - memo, - *coarse_group, - coarse, - coarse_node, - OriginOfCandidateForMemoGroup::Rollup, - )?); - if let Some(own_value) = OperatorNode::reachable(&fine_node) - .into_iter() - .find(|node| **node == *value) - { - let fine_reader = if Rc::ptr_eq(&own_value, &fine_node) { - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Project { - cols: canonical - .schema - .fields - .iter() - .enumerate() - .map(|(column, field)| ProjectItem { - alias: Some(field.name.clone()), - expr: ScalarExpr::Column(column), - }) - .collect(), - qualifier: None, - child: Rc::clone(&canonical), - })) - .map_err(|error| Pass2RuleError::ReuseRule(error.to_string()))? - } else { - replace( - &fine_node, - &HashMap::from([(Rc::as_ptr(&own_value), Rc::clone(&canonical))]), - ) - }; - added += usize::from(append_reader( - memo, - *fine_group, - fine, - fine_reader, - OriginOfCandidateForMemoGroup::Rollup, - )?); - } - } - } - } - Ok(added) -} +use crate::pass2::memo::rules::Pass2RuleError; +use asap_types::stages::Memo; -/// Existing tumbling window rollup rules applied to producer pairs on already generated -/// legal local grids. No whole-workload choice combinations are constructed. -pub(crate) fn add_window_rollup_candidates(memo: &mut Memo) -> Result { - use crate::pass2::asap_aware_cse::{ - tumbling_window_candidates, window_composition_shared_chain_candidates, - }; - let mut grids = vec![std::collections::BTreeSet::new(); memo.groups.len()]; - for (group, candidates) in memo.candidates.iter().enumerate() { - for candidate in candidates.iter().filter(|candidate| { - candidate.origin == OriginOfCandidateForMemoGroup::TumblingWindows - && eligible(candidate) - }) { - if let Some((build, _)) = simple_build(memo, candidate) { - if let Some(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms, - .. - }) = build.asap() - { - grids[group].insert(*tumbling_window_width_ms); - } - } - } - } - let candidates: Vec<_> = memo - .candidates - .iter() - .enumerate() - .flat_map(|(group, candidates)| { - candidates - .iter() - .filter(|candidate| { - eligible(candidate) && candidate.origin == OriginOfCandidateForMemoGroup::Pass1 - }) - .map(move |candidate| (group, candidate.clone())) - }) - .collect(); - let mut added = 0; - for (coarse_group, coarse) in &candidates { - for (fine_group, fine) in &candidates { - if coarse_group == fine_group || grids[*fine_group].is_empty() { - continue; - } - for grid in &grids[*coarse_group] { - let roots = [ - QueryRoot::Operator(Rc::clone(&coarse.root_node)), - QueryRoot::Operator(Rc::clone(&fine.root_node)), - ]; - let tumbling_windows = - tumbling_window_candidates(&roots, &[Some(*grid), Some(*grid)]); - if tumbling_windows.len() != 2 { - continue; - } - for rewritten in - window_composition_shared_chain_candidates(&roots, &tumbling_windows) - { - if rewritten.shared_chains.is_empty() { - continue; - } - let Ok(nodes) = rewritten.roots else { continue }; - let [QueryRoot::Operator(coarse_node), QueryRoot::Operator(fine_node)] = - nodes.as_slice() - else { - continue; - }; - if !OperatorNode::reachable(coarse_node).iter().any(|node| { - matches!( - node.asap(), - Some(ASAPOp::SummaryMerge { - group_by: Some(_), - .. - }) - ) - }) { - continue; - } - let inputs = combined_inputs(coarse, fine); - let coarse_node = restore_inputs(coarse_node, memo, &inputs); - let fine_node = restore_inputs(fine_node, memo, &inputs); - added += usize::from(append_reader( - memo, - *coarse_group, - coarse, - coarse_node, - OriginOfCandidateForMemoGroup::Rollup, - )?); - added += usize::from(append_reader( - memo, - *fine_group, - fine, - fine_node, - OriginOfCandidateForMemoGroup::Rollup, - )?); - } - } - } - } - Ok(added) +/// Rule 6, rollup: a candidate of a coarse aggregate or summary group reading +/// a finer group whose state rolls up (exact or sketch merge). +pub(crate) fn add_rollup_candidates(memo: &mut Memo) -> Result<(), Pass2RuleError> { + let _ = memo; + todo!("#1207 PR 4: rule 6 over groups") } -#[cfg(test)] -mod tests { - use crate::pass1::memo_population::try_populate_pass1; - use crate::pass1::per_computation_candidates::generate_per_computation_candidates; - use crate::stage0::generate_dag_of_logical_memo_groups; - use crate::test_support::{lower_promql, sharing_rules_and_shared_state_groups}; - use asap_types::ir::properties::AccuracyTarget; - use asap_types::ir::QueryRoot; - use asap_types::stages::OriginOfCandidateForMemoGroup; - - // When the finer query's own result is the finalized value the coarse - // query rolls up, the rollup reads the finer query's group, so one - // stored result serves both queries without a copy or an identity - // projection. - #[test] - fn a_rollup_reads_the_finer_query_result_itself() { - let roots: Vec<_> = ["sum by (job) (m)", "sum by (job, instance) (m)"] - .into_iter() - .map(|text| QueryRoot::Operator(lower_promql(text, AccuracyTarget::exact()))) - .collect(); - let initial = generate_dag_of_logical_memo_groups(&roots); - let inventory = - generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&roots)) - .unwrap(); - let memo = - sharing_rules_and_shared_state_groups(try_populate_pass1(initial, &inventory).unwrap()) - .unwrap(); - let root_of = |query: usize| memo.query_roots[query].as_operator().unwrap().0 as usize; - let (coarse, fine) = (root_of(0), root_of(1)); - assert!( - memo.candidates[coarse].iter().any(|candidate| { - candidate.origin == OriginOfCandidateForMemoGroup::Rollup - && candidate - .input_groups - .iter() - .any(|child| child.0 as usize == fine) - }), - "the coarse rollup reads the finer query's group" - ); - assert!( - !memo.candidates[fine] - .iter() - .any(|candidate| candidate.origin == OriginOfCandidateForMemoGroup::Rollup), - "the finer query needs no reader of a separate producer" - ); - } +/// Rule 7, window rollup: a candidate of a longer-range group merging a +/// shorter-range group's windows. +pub(crate) fn add_window_rollup_candidates(memo: &mut Memo) -> Result<(), Pass2RuleError> { + let _ = memo; + todo!("#1207 PR 4: rule 7 over groups") } diff --git a/crates/logical-asap-optimization/src/pass2/memo/rules/scan_labels.rs b/crates/logical-asap-optimization/src/pass2/memo/rules/scan_labels.rs index 1410c426a..a808eb48f 100644 --- a/crates/logical-asap-optimization/src/pass2/memo/rules/scan_labels.rs +++ b/crates/logical-asap-optimization/src/pass2/memo/rules/scan_labels.rs @@ -1,237 +1,11 @@ //! Scan-label widening for explicit memo state sharing. Reuses the existing //! Pass 2 widening rule; only its source-chain boundaries are interned here. -use std::collections::{HashMap, HashSet}; -use std::rc::Rc; -use asap_accuracy::propagation::recompose_guarantees; -use asap_types::ir::operator::Reduction; -use asap_types::ir::{ASAPOp, NonASAPOp, OperatorNode, QueryRoot}; +use crate::pass2::memo::rules::Pass2RuleError; +use asap_types::stages::Memo; -use crate::pass2::memo::state_groups::{boundary_groups, simple_build, Pass2RuleError}; -use asap_types::stages::{CandidateForMemoGroup, Memo, OriginOfCandidateForMemoGroup}; -use asap_types::stages::{MemoGroup, MemoGroupId}; - -/// Preserve local implementations and append readers over union-label inputs. -/// Parent/query reachability is rebuilt by the enclosing state-group expansion. -pub(crate) fn add_scan_label_candidates(memo: &mut Memo) -> Result { - let mut owners = Vec::new(); - let mut roots = Vec::new(); - for (group, candidates) in memo.candidates.iter().enumerate() { - for (index, candidate) in candidates.iter().enumerate() { - // Source-group candidates expose scan reuse to every downstream - // operator, including exact/non-summary computations. Their public - // schemas stay narrow through the existing projection rewrite. - if is_source_chain(&candidate.root_node) && super::reuse::eligible(candidate) { - owners.push((group, index)); - roots.push(QueryRoot::Operator(Rc::clone(&candidate.root_node))); - continue; - } - let Some((build, children)) = simple_build(memo, candidate) else { - continue; - }; - if !matches!( - build.asap(), - Some( - ASAPOp::SummaryAgg { - reduction: Reduction::PerEntity, - filter: None, - .. - } | ASAPOp::SummaryTumblingWindowChain { - reduction: Reduction::PerEntity, - filter: None, - .. - } - ) - ) || candidate - .input_groups - .iter() - .any(|child| !children.contains(child)) - || children - .iter() - .any(|child| !is_source_chain(&memo.groups[child.0 as usize].logical_node)) - { - continue; - } - owners.push((group, index)); - roots.push(QueryRoot::Operator(Rc::clone(&candidate.root_node))); - } - } - let Some(widened) = crate::pass2::asap_aware_cse::with_union_scan_labels(&roots) - .map_err(|error| Pass2RuleError::ScanLabelWidening(error.to_string()))? - else { - return Ok(0); - }; - // Widening recomposes every root's guarantees, also those it left - // unchanged, so a root is unchanged when it equals its recomposed - // original. Compared with the stored original instead, an unchanged root - // (a scan that already has every label) that differs only in its - // guarantee annotation would pass for a widened reader, and interning - // would map it back to its own group: a candidate reading the group it - // implements. - let unwidened = recompose_guarantees(&roots) - .map_err(|error| Pass2RuleError::ScanLabelWidening(error.to_string()))?; - let mut added = 0; - let mut canonical = HashMap::new(); - for (((group, index), root), unwidened) in owners.into_iter().zip(widened).zip(unwidened) { - let (QueryRoot::Operator(node), QueryRoot::Operator(unwidened)) = (root, unwidened) else { - unreachable!("operator readers") - }; - let original = memo.candidates[group][index].clone(); - if *node == *unwidened || !same_output_contract(&original.root_node, &node) { - continue; - } - let queries = memo.groups[group].queries.clone(); - let mut node = intern_inputs(&node, memo, &queries, &mut canonical)?; - if memo - .groups - .iter() - .any(|group| Rc::ptr_eq(&group.logical_node, &node)) - { - // Interning an equivalent range/source can return another group's - // canonical value. Keep a concrete reader instead of a root alias. - node = - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Project { - cols: node - .schema - .fields - .iter() - .enumerate() - .map(|(column, field)| asap_types::ir::ProjectItem { - alias: Some(field.name.clone()), - expr: asap_types::ir::ScalarExpr::Column(column), - }) - .collect(), - qualifier: None, - child: node, - })) - .map_err(|error| Pass2RuleError::ScanLabelWidening(error.to_string()))?; - if !same_output_contract(&original.root_node, &node) { - continue; - } - } - let boundaries = memo - .groups - .iter() - .enumerate() - .map(|(id, group)| (Rc::as_ptr(&group.logical_node), MemoGroupId(id as u32))) - .collect(); - let mut reader = original; - reader.root_node = node; - reader.input_groups = boundary_groups(&reader.root_node, &boundaries); - reader.origin = OriginOfCandidateForMemoGroup::WidenedScans; - if memo.candidates[group].iter().any(|known| { - known.origin == OriginOfCandidateForMemoGroup::WidenedScans - && known.input_groups == reader.input_groups - && *known.root_node == *reader.root_node - }) { - continue; - } - memo.candidates[group].push(reader); - added += 1; - } - Ok(added) -} - -fn same_output_contract(expected: &OperatorNode, actual: &OperatorNode) -> bool { - expected.result_kind == actual.result_kind - && expected.schema.fields == actual.schema.fields - && expected.schema.time_index == actual.schema.time_index - && expected.schema.unique_keys == actual.schema.unique_keys - // A projection may enumerate every output of an originally open schema. - && (!expected.schema.closed || actual.schema.closed) -} - -fn is_source_chain(node: &OperatorNode) -> bool { - match node.non_asap() { - Some(NonASAPOp::Scan { .. }) => true, - Some(NonASAPOp::TimeRange { child, .. } | NonASAPOp::TimeShift { child, .. }) => { - is_source_chain(child) - } - _ => false, - } -} - -fn same_source_node(left: &OperatorNode, right: &OperatorNode) -> bool { - // These operators preserve exact source rows; recomposition's exact - // guarantee annotation does not change their logical/execution contract. - left.operator.map_children(|_| ()) == right.operator.map_children(|_| ()) - && left.schema == right.schema - && left.result_kind == right.result_kind - && left.timing == right.timing - && left.materialization == right.materialization - && left - .children() - .iter() - .zip(right.children()) - .all(|(a, b)| Rc::ptr_eq(a, b)) - && left.children().len() == right.children().len() -} - -fn intern_inputs( - node: &Rc, - memo: &mut Memo, - queries: &[usize], - done: &mut HashMap<*const OperatorNode, Rc>, -) -> Result, Pass2RuleError> { - if let Some(known) = done.get(&Rc::as_ptr(node)) { - return Ok(Rc::clone(known)); - } - let mut children = HashMap::new(); - for child in node.children() { - children.insert( - Rc::as_ptr(child), - intern_inputs(child, memo, queries, done)?, - ); - } - let rewritten = Rc::new(OperatorNode { - operator: node - .operator - .map_children(|child| Rc::clone(&children[&Rc::as_ptr(child)])), - ..(**node).clone() - }); - let result = if is_source_chain(&rewritten) { - if let Some(existing) = memo - .groups - .iter() - .find(|group| same_source_node(&group.logical_node, &rewritten)) - { - Rc::clone(&existing.logical_node) - } else { - // The new group's id must fit a `MemoGroupId`. - u32::try_from(memo.groups.len()).map_err(|_| Pass2RuleError::TooManyGroups)?; - let mut seen = HashSet::new(); - let inputs: Vec<_> = rewritten - .children() - .iter() - .map(|child| { - MemoGroupId( - memo.groups - .iter() - .position(|group| Rc::ptr_eq(child, &group.logical_node)) - .expect("source-chain children were interned") - as u32, - ) - }) - .filter(|input| seen.insert(*input)) - .collect(); - memo.groups.push(MemoGroup { - logical_node: Rc::clone(&rewritten), - child_groups: inputs.clone(), - parents: Vec::new(), - queries: queries.to_vec(), - }); - memo.candidates.push(vec![CandidateForMemoGroup { - fused_groups: Vec::new(), - root_node: Rc::clone(&rewritten), - input_groups: inputs, - origin: OriginOfCandidateForMemoGroup::WidenedScans, - accuracy_budget_parts: 1, - }]); - rewritten - } - } else { - rewritten - }; - done.insert(Rc::as_ptr(node), Rc::clone(&result)); - Ok(result) +/// Rule 2, scan widening: a group for the widened scan, and in each original scan group a candidate `Project(widened)`. +pub(crate) fn add_scan_label_candidates(memo: &mut Memo) -> Result<(), Pass2RuleError> { + let _ = memo; + todo!("#1207 PR 4: rule 2 over groups") } diff --git a/crates/logical-asap-optimization/src/pass2/memo/rules/shared_configuration_readers.rs b/crates/logical-asap-optimization/src/pass2/memo/rules/shared_configuration_readers.rs deleted file mode 100644 index 5d45eac0d..000000000 --- a/crates/logical-asap-optimization/src/pass2/memo/rules/shared_configuration_readers.rs +++ /dev/null @@ -1,637 +0,0 @@ -//! Expand compatible readers at finite shared sizes and tumbling-window horizons before -//! interning concrete states. Local candidates remain available. -use crate::stage0::cse::same_value; -use std::collections::{HashMap, HashSet}; -use std::rc::Rc; - -use asap_types::ir::flat::{flatten, FlatDag}; -use asap_types::ir::schema::{sketch::SketchKind, FieldDataType}; -use asap_types::ir::{ASAPOp, Operator, OperatorNode, QueryRoot}; - -use super::shared_producer::{ - add_shared_producer_candidates, spans_groups, Reader, SharedProducer, -}; -use crate::pass2::asap_aware_cse::{with_resized_build, with_tumbling_window_chain}; -use crate::pass2::memo::shared_configurations::{ - shared_size, state_size_configurations, tumbling_window_configurations, - TumblingWindowConfiguration, TumblingWindowRequirement, -}; -use crate::pass2::memo::state_groups::{ - boundary_ptrs, same_range_input, simple_build, Pass2RuleError, -}; -use asap_types::stages::Memo; - -/// `resized` (a rewrite of `original` with the same shape) with each node -/// that is a child group's node in `original` put back, and the nodes above -/// rebuilt over them, keeping their own schema and guarantee. -fn with_child_nodes( - original: &Rc, - resized: &Rc, - child_nodes: &HashSet<*const OperatorNode>, - done: &mut HashMap<*const OperatorNode, Rc>, -) -> Rc { - if child_nodes.contains(&Rc::as_ptr(original)) { - return Rc::clone(original); - } - if let Some(found) = done.get(&Rc::as_ptr(resized)) { - return Rc::clone(found); - } - let pairs: HashMap<*const OperatorNode, Rc> = original - .children() - .into_iter() - .zip(resized.children()) - .map(|(o, r)| (Rc::as_ptr(r), with_child_nodes(o, r, child_nodes, done))) - .collect(); - let mut changed = false; - let operator = resized.operator.map_children(|child| { - let new = Rc::clone(&pairs[&Rc::as_ptr(child)]); - changed |= !Rc::ptr_eq(&new, child); - new - }); - let out = match changed { - false => Rc::clone(resized), - true => Rc::new( - OperatorNode::with_schema(operator, resized.schema.clone()) - .with_guarantee(resized.guarantee.clone()) - .with_timing(resized.timing) - .with_materialization(resized.materialization), - ), - }; - done.insert(Rc::as_ptr(resized), Rc::clone(&out)); - out -} - -/// A summary build one candidate reads: its own need of the shared producer. -struct StateNeed { - candidate: usize, - build: Rc, - merge: Option>, - family: FieldDataType, - /// The reader candidate's `accuracy_budget_parts`: the shares its build - /// is sized for. - accuracy_budget_parts: usize, - /// The reader's normalized tumbling-window layout, when its set shares tumbling windows: equal - /// layouts of one set have one index, so a configuration rewrites them once. - normalized_layout: Option, -} - -/// A shared build's need: one tumbling-window layout (its index among the set's tumbling window -/// configurations, for request deduplication) or one size, with the shares -/// of the budget its build is sized for ([`covered_parts`]). -enum StateCovering { - TumblingWindows { - configuration_index: usize, - configuration: TumblingWindowConfiguration, - accuracy_budget_parts: usize, - }, - Size { - family: FieldDataType, - accuracy_budget_parts: usize, - }, -} - -/// Summary builds of one family over one input at different sizes or tumbling window -/// layouts: each reader is rewritten to a build at each covering size or -/// layout that serves it, and equal concrete builds become one state group -/// later ([`super::super::state_groups`]). -struct SummaryStates { - deduplicate_requests: bool, - /// Every candidate's graph signature by group, so no rewrite repeats one. - known: HashMap<(usize, String), Vec>>, - rewrite_count: usize, - /// The (normalized layout, configuration index) rewrites already made. - requests: HashSet<(usize, usize)>, -} - -pub(crate) fn add_state_configuration_candidates(memo: &mut Memo) -> Result<(), Pass2RuleError> { - expand_configurations(memo, true).map(|_| ()) -} - -fn expand_configurations( - memo: &mut Memo, - deduplicate_requests: bool, -) -> Result { - let mut states = SummaryStates { - deduplicate_requests, - known: HashMap::new(), - rewrite_count: 0, - requests: HashSet::new(), - }; - add_shared_producer_candidates(memo, &mut states)?; - Ok(states.rewrite_count) -} - -impl SharedProducer for SummaryStates { - type Need = StateNeed; - type Covering = StateCovering; - - fn sets(&mut self, memo: &Memo) -> Result>>, Pass2RuleError> { - let mut sets: Vec>> = Vec::new(); - for (group, candidates) in memo.candidates.iter().enumerate() { - for (candidate, entry) in candidates.iter().enumerate() { - let Some((build, _)) = simple_build(memo, entry) else { - continue; - }; - // Configuration expansion precedes Stage 2. The rewrite helpers - // create unassigned state; preserve any already assigned build. - if build.timing.is_some() - || build.materialization - != asap_types::physical::Materialization::NotMaterialized - { - continue; - } - let family = match build.asap() { - Some( - ASAPOp::SummaryAgg { family, .. } - | ASAPOp::SummaryTumblingWindowChain { family, .. }, - ) => family.clone(), - _ => unreachable!(), - }; - let merge = if matches!( - build.asap(), - Some(ASAPOp::SummaryTumblingWindowChain { .. }) - ) { - let merges: Vec<_> = OperatorNode::reachable(&entry.root_node).into_iter().filter(|node| { - matches!(node.asap(), Some(ASAPOp::SummaryMerge {children, group_by: None, ..}) - if children.len() == 1 && Rc::ptr_eq(&children[0], &build)) - }).collect(); - // The existing rewrite changes one plain merge. Other tumbling window uses - // need their own explicit transformation, not guessed semantics. - if merges.len() != 1 { - continue; - } - Some(Rc::clone(&merges[0])) - } else { - None - }; - let reader = Reader { - group, - need: StateNeed { - candidate, - build, - merge, - family, - accuracy_budget_parts: entry.accuracy_budget_parts, - normalized_layout: None, - }, - }; - match sets - .iter_mut() - .find(|set| compatible(&set[0].need, &reader.need)) - { - Some(set) => set.push(reader), - None => sets.push(vec![reader]), - } - } - } - for (group, candidates) in memo.candidates.iter().enumerate() { - for candidate in candidates { - self.known - .entry((group, graph_signature(&candidate.root_node)?)) - .or_default() - .push(Rc::clone(&candidate.root_node)); - } - } - let mut layouts: Vec<(usize, usize, FlatDag)> = Vec::new(); - for set in &mut sets { - if spans_groups(set) { - assign_normalized_layouts(memo, set, &mut layouts)?; - } - } - Ok(sets) - } - - fn coverings(&self, set: &[Reader]) -> Result, Pass2RuleError> { - let tumbling_window_requirements: Option> = set - .iter() - .map(|reader| tumbling_window_requirement(&reader.need)) - .collect(); - let Some(requirements) = tumbling_window_requirements else { - let requirements = set - .iter() - .map(|reader| match &reader.need.family { - FieldDataType::Sketch(kind, _) => Some(kind.params().clone()), - _ => None, - }) - .collect(); - let first = &set[0].need.family; - return Ok(state_size_configurations(requirements) - .into_iter() - .map(|configuration| { - let family = match (first, configuration) { - (FieldDataType::Sketch(kind, grouping), Some(params)) => { - FieldDataType::Sketch( - SketchKind::new(*kind.algorithm(), params), - grouping.clone(), - ) - } - _ => first.clone(), - }; - StateCovering::Size { - accuracy_budget_parts: covered_parts(&family, set), - family, - } - }) - .collect()); - }; - Ok(tumbling_window_configurations(&requirements) - .into_iter() - .enumerate() - .map( - |(configuration_index, configuration)| StateCovering::TumblingWindows { - configuration_index, - accuracy_budget_parts: covered_parts(&configuration.family, set), - configuration, - }, - ) - .collect()) - } - - fn derive( - &mut self, - memo: &mut Memo, - reader: &Reader, - covering: &StateCovering, - ) -> Result { - let need = &reader.need; - let serves = match covering { - StateCovering::TumblingWindows { configuration, .. } => configuration - .serves(&tumbling_window_requirement(need).expect("tumbling window set")), - StateCovering::Size { family, .. } => serves_size(family, &need.family), - }; - if !serves { - return Ok(0); - } - let original = memo.candidates[reader.group][need.candidate].clone(); - let roots = [QueryRoot::Operator(Rc::clone(&original.root_node))]; - let result = match covering { - StateCovering::TumblingWindows { - configuration_index, - configuration, - .. - } => { - let Ok(count) = u32::try_from( - configuration.longest_window_ms / configuration.tumbling_window_width_ms, - ) else { - return Ok(0); - }; - if self.deduplicate_requests - && need - .normalized_layout - .is_some_and(|layout| !self.requests.insert((layout, *configuration_index))) - { - return Ok(0); - } - self.rewrite_count += 1; - with_tumbling_window_chain( - &roots, - need.merge.as_ref().unwrap(), - configuration.family.clone(), - configuration.tumbling_window_width_ms, - count, - ) - } - StateCovering::Size { family, .. } => { - self.rewrite_count += 1; - with_resized_build(&roots, &need.build, family.clone()) - } - }; - let boundaries = boundary_ptrs(memo, &original); - let roots = - result.map_err(|error| Pass2RuleError::StateConfiguration(error.to_string()))?; - let [QueryRoot::Operator(node)] = roots.as_slice() else { - unreachable!("operator rewrite") - }; - let node = with_child_nodes(&original.root_node, node, &boundaries, &mut HashMap::new()); - // Resizing affects internal state types, never public output fields. - if original.root_node.schema.fields != node.schema.fields - || original.root_node.schema.time_index != node.schema.time_index - || original.root_node.schema.unique_keys != node.schema.unique_keys - || (original.root_node.schema.closed && !node.schema.closed) - { - return Err(Pass2RuleError::StateConfiguration( - "state rewrite changed the reader output contract".into(), - )); - } - let previous = self - .known - .entry((reader.group, graph_signature(&node)?)) - .or_default(); - if previous.iter().any(|known| **known == *node) { - return Ok(0); - } - previous.push(Rc::clone(&node)); - let mut candidate = original; - candidate.root_node = node; - // The rewritten build is sized for the covering's shares, at least - // the reader's own. - candidate.accuracy_budget_parts = match covering { - StateCovering::TumblingWindows { - accuracy_budget_parts, - .. - } - | StateCovering::Size { - accuracy_budget_parts, - .. - } => (*accuracy_budget_parts).max(candidate.accuracy_budget_parts), - }; - memo.candidates[reader.group].push(candidate); - Ok(1) - } -} - -/// Numbers the normalized tumbling-window layouts of `set`'s readers, when it shares -/// tumbling windows: different original layouts can produce the same requested -/// reader/configuration, so each is normalized once and its rewrite made once, -/// instead of once for every existing tumbling-window width. Numbers continue across -/// sets (`layouts`), so a configuration index of one set never matches -/// another's. -fn assign_normalized_layouts( - memo: &Memo, - set: &mut [Reader], - layouts: &mut Vec<(usize, usize, FlatDag)>, -) -> Result<(), Pass2RuleError> { - let Some(requirements) = set - .iter() - .map(|reader| tumbling_window_requirement(&reader.need)) - .collect::>>() - else { - return Ok(()); - }; - let mut layout_keys: HashMap<(usize, String), Vec> = HashMap::new(); - for (reader, requirement) in set.iter_mut().zip(&requirements) { - let Ok(count) = u32::try_from(requirement.window_ms) else { - // Normalization must not narrow the legal long-window domain. - continue; - }; - let need = &reader.need; - let original = &memo.candidates[reader.group][need.candidate]; - let roots = [QueryRoot::Operator(Rc::clone(&original.root_node))]; - let normalized = with_tumbling_window_chain( - &roots, - need.merge.as_ref().unwrap(), - need.family.clone(), - 1, - count, - ) - .map_err(|error| Pass2RuleError::StateConfiguration(error.to_string()))?; - let flat = flatten(&normalized).0; - let signature = serde_json::to_string(&flat) - .map_err(|error| Pass2RuleError::StateConfiguration(error.to_string()))?; - let indices = layout_keys.entry((reader.group, signature)).or_default(); - let existing = indices.iter().copied().find(|&index| { - let (group, candidate, previous) = &layouts[index]; - let previous_candidate = &memo.candidates[*group][*candidate]; - previous == &flat - && previous_candidate.origin == original.origin - && previous_candidate.input_groups == original.input_groups - && previous_candidate.fused_groups == original.fused_groups - && previous_candidate.accuracy_budget_parts == original.accuracy_budget_parts - }); - reader.need.normalized_layout = Some(existing.unwrap_or_else(|| { - let index = layouts.len(); - layouts.push((reader.group, need.candidate, flat)); - indices.push(index); - index - })); - } - Ok(()) -} - -fn graph_signature(node: &Rc) -> Result { - serde_json::to_string(&flatten(&[QueryRoot::Operator(Rc::clone(node))]).0) - .map_err(|error| Pass2RuleError::StateConfiguration(error.to_string())) -} - -fn tumbling_window_requirement(reader: &StateNeed) -> Option { - let ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms, - tumbling_window_count, - .. - } = reader.build.asap()? - else { - return None; - }; - let ASAPOp::SummaryMerge { - newest_tumbling_windows, - .. - } = reader.merge.as_ref()?.asap()? - else { - return None; - }; - Some(TumblingWindowRequirement { - tumbling_window_width_ms: *tumbling_window_width_ms, - window_ms: tumbling_window_width_ms.checked_mul(u64::from( - newest_tumbling_windows.unwrap_or(*tumbling_window_count), - ))?, - family: reader.family.clone(), - }) -} - -/// Whether a build of `family` serves a reader of `reader`: the size both -/// share is `family` itself. -fn serves_size(family: &FieldDataType, reader: &FieldDataType) -> bool { - shared_size(family, reader).is_some_and(|joined| same_value(&joined, family)) -} - -/// The shares of the budget a build of `family` is sized for: the most of -/// any reader in `set` it serves (it is at least as large as that reader's -/// own build, sized for those shares); 1 when it serves none. -fn covered_parts(family: &FieldDataType, set: &[Reader]) -> usize { - set.iter() - .filter(|reader| serves_size(family, &reader.need.family)) - .map(|reader| reader.need.accuracy_budget_parts) - .max() - .unwrap_or(1) -} - -fn compatible(left: &StateNeed, right: &StateNeed) -> bool { - if shared_size(&left.family, &right.family).is_none() { - return false; - } - let left_inputs = left.build.children(); - let right_inputs = right.build.children(); - if left_inputs.len() != right_inputs.len() - || !left_inputs - .iter() - .zip(right_inputs) - .all(|(a, b)| Rc::ptr_eq(a, b) || same_range_input(a, b)) - { - return false; - } - // Ignore only the fields this rule rewrites. Build guarantees are derived - // from these operators and are recomposed by the existing rewrite helpers. - let signature = |reader: &StateNeed| { - let mut operator = reader.build.operator.map_children(|_| ()); - match &mut operator { - Operator::ASAP(ASAPOp::SummaryAgg { family, .. }) => *family = left.family.clone(), - Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { - family, - tumbling_window_width_ms, - tumbling_window_count, - .. - }) => { - *family = left.family.clone(); - *tumbling_window_width_ms = 1; - *tumbling_window_count = 1; - } - _ => unreachable!(), - } - let mut schema = reader.build.schema.clone(); - for field in &mut schema.fields { - if same_value(&field.dtype, &reader.family) { - field.dtype = left.family.clone(); - } - } - serde_json::to_string(&( - operator, - schema, - reader.build.result_kind, - reader.build.timing, - reader.build.materialization, - )) - .ok() - }; - signature(left).is_some_and(|signature_left| Some(signature_left) == signature(right)) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::pass1::memo_population::try_populate_pass1; - use crate::pass1::per_computation_candidates::generate_per_computation_candidates; - use crate::stage0::generate_dag_of_logical_memo_groups; - use crate::test_support::{entry_with, lower_promql}; - use asap_types::ir::properties::AccuracyTarget; - use asap_types::workload::{QueryRecurrence, RepeatedDemand, RepetitionInterval}; - - // Request deduplication preserves the entire declared finite configuration - // set while avoiding repeated rewrites through every initial tumbling-window layout. - #[test] - fn equivalent_tumbling_window_layouts_expand_each_configuration_once() { - let target = AccuracyTarget::max_error(0.01).failure_probability(0.001); - let roots: Vec<_> = [0.95, 0.99] - .into_iter() - .map(|quantile| { - QueryRoot::Operator(lower_promql( - &format!("quantile_over_time({quantile}, latency[5m])"), - target.clone(), - )) - }) - .collect(); - let initial = generate_dag_of_logical_memo_groups(&roots); - let inventory = - generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&roots)) - .unwrap(); - let mut memo = try_populate_pass1(initial, &inventory).unwrap(); - let entries = vec![ - entry_with( - target, - QueryRecurrence::Repeated(RepeatedDemand::FixedInterval(RepetitionInterval(6))) - ); - 2 - ]; - crate::pass2::memo::window_candidates::add_local_window_composition(&mut memo, &entries); - let mut unoptimized = memo.clone(); - let optimized_rewrites = expand_configurations(&mut memo, true).unwrap(); - let exhaustive_rewrites = expand_configurations(&mut unoptimized, false).unwrap(); - assert!( - optimized_rewrites < exhaustive_rewrites, - "{optimized_rewrites} versus {exhaustive_rewrites}" - ); - let signatures = |memo: &Memo| -> Vec> { - memo.candidates - .iter() - .map(|candidates| { - candidates - .iter() - .map(|candidate| graph_signature(&candidate.root_node).unwrap()) - .collect() - }) - .collect() - }; - assert_eq!(signatures(&memo), signatures(&unoptimized)); - let widths: HashSet<_> = memo - .candidates - .iter() - .flatten() - .flat_map(|candidate| OperatorNode::reachable(&candidate.root_node)) - .filter_map(|node| match node.asap() { - Some(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms, - .. - }) => Some(*tumbling_window_width_ms), - _ => None, - }) - .collect(); - assert_eq!(widths, HashSet::from([1, 2, 3, 6])); - } - - /// The KLL sizes (`k`) of the builds `candidate` reads. - fn kll_sizes(candidate: &asap_types::stages::CandidateForMemoGroup) -> Vec { - OperatorNode::reachable(&candidate.root_node) - .iter() - .filter_map(|node| match node.asap() { - Some( - ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), - .. - } - | ASAPOp::SummaryTumblingWindowChain { - family: FieldDataType::Sketch(kind, _), - .. - }, - ) => match kind.params() { - asap_types::ir::schema::SketchParams::Kll { k, .. } => Some(*k), - _ => None, - }, - _ => None, - }) - .collect() - } - - // A rewrite to a size another reader's build was sized for carries that - // build's shares of the budget: in `q0.5 / q0.9` each operand has a KLL - // sized for the whole budget and one for half of it, and the copy of the - // whole-budget reader rewritten to the half-budget size is sized for two - // shares, not one. - #[test] - fn a_rewrite_to_a_shared_size_carries_its_parts() { - let target = AccuracyTarget::max_error(0.01).failure_probability(0.001); - let roots = vec![QueryRoot::Operator(lower_promql( - "quantile_over_time(0.5, latency[5m]) / quantile_over_time(0.9, latency[5m])", - target.clone(), - ))]; - let initial = generate_dag_of_logical_memo_groups(&roots); - let inventory = - generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&roots)) - .unwrap(); - let mut memo = try_populate_pass1(initial, &inventory).unwrap(); - // Pass 1's parts per KLL size. - let mut parts_of_size: HashMap = HashMap::new(); - for candidate in memo.candidates.iter().flatten() { - for k in kll_sizes(candidate) { - let parts = parts_of_size.entry(k).or_insert(0); - *parts = (*parts).max(candidate.accuracy_budget_parts); - } - } - assert!( - parts_of_size.values().any(|&parts| parts == 2), - "{parts_of_size:?}" - ); - let before = memo.candidates.iter().map(Vec::len).sum::(); - expand_configurations(&mut memo, true).unwrap(); - assert!(memo.candidates.iter().map(Vec::len).sum::() > before); - for candidate in memo.candidates.iter().flatten() { - for k in kll_sizes(candidate) { - if let Some(&parts) = parts_of_size.get(&k) { - assert!( - candidate.accuracy_budget_parts >= parts, - "k {k}: parts {} below {parts}", - candidate.accuracy_budget_parts - ); - } - } - } - } -} diff --git a/crates/logical-asap-optimization/src/pass2/memo/rules/shared_producer.rs b/crates/logical-asap-optimization/src/pass2/memo/rules/shared_producer.rs deleted file mode 100644 index ea01f1956..000000000 --- a/crates/logical-asap-optimization/src/pass2/memo/rules/shared_producer.rs +++ /dev/null @@ -1,62 +0,0 @@ -//! The shared-producer rule: readers that need one computation at different -//! values (a top-k `k`, a sketch size, a window and step) share one producer -//! built at a need that covers them, and each reader derives its own result -//! from it as one more candidate; its own candidates stay, so plan selection -//! decides. Identical expressions are the trivial case, shared by memo group -//! identity, and add no candidates here. -use crate::pass2::memo::state_groups::Pass2RuleError; -use asap_types::stages::Memo; - -/// One reader of a kind of shared producer: its memo group and its need. -pub(in crate::pass2::memo) struct Reader { - pub group: usize, - pub need: N, -} - -/// A kind of producer several readers can share. -pub(in crate::pass2::memo) trait SharedProducer { - /// What one reader requires of the producer. - type Need; - /// A producer's need that serves several readers at once. - type Covering; - /// The memo's readers of this kind, in sets whose members can all read - /// one producer; in memo order. - fn sets(&mut self, memo: &Memo) -> Result>>, Pass2RuleError>; - /// The coverings to offer the readers of `set`. - fn coverings(&self, set: &[Reader]) -> Result, Pass2RuleError>; - /// Adds `reader`'s derivation from the producer at `covering`, if that - /// producer serves it; the number of candidates added. - fn derive( - &mut self, - memo: &mut Memo, - reader: &Reader, - covering: &Self::Covering, - ) -> Result; -} - -/// Whether `set`'s readers belong to two or more groups; a set inside one -/// group shares nothing. -pub(in crate::pass2::memo) fn spans_groups(set: &[Reader]) -> bool { - set.iter().any(|reader| reader.group != set[0].group) -} - -/// Gives every reader of a set spanning two or more groups its derivation -/// from each of the set's coverings; the number of candidates added. -pub(in crate::pass2::memo) fn add_shared_producer_candidates( - memo: &mut Memo, - producer: &mut P, -) -> Result { - let mut added = 0; - for set in producer.sets(memo)? { - if !spans_groups(&set) { - continue; - } - let coverings = producer.coverings(&set)?; - for reader in &set { - for covering in &coverings { - added += producer.derive(memo, reader, covering)?; - } - } - } - Ok(added) -} diff --git a/crates/logical-asap-optimization/src/pass2/memo/rules/summary_inputs.rs b/crates/logical-asap-optimization/src/pass2/memo/rules/summary_inputs.rs index e2bb0e84f..956141fbf 100644 --- a/crates/logical-asap-optimization/src/pass2/memo/rules/summary_inputs.rs +++ b/crates/logical-asap-optimization/src/pass2/memo/rules/summary_inputs.rs @@ -1,100 +1,12 @@ //! Expose private value projections and filters below summary builds as canonical inputs. //! This preserves their full expressions and schema, including opaque series //! identity, rather than assuming that a projection is an identity operation. -use super::reuse::{append_producer, append_reader, eligible, replace}; -use crate::pass2::memo::state_groups::{boundary_ptrs, nodes_within, Pass2RuleError}; -use asap_types::ir::{ASAPOp, NonASAPOp, OperatorNode}; -use asap_types::stages::{Memo, OriginOfCandidateForMemoGroup}; -use std::collections::{HashMap, HashSet}; -use std::rc::Rc; -pub(crate) fn factor_summary_inputs(memo: &mut Memo) -> Result<(), Pass2RuleError> { - let originals: Vec<_> = memo - .candidates - .iter() - .enumerate() - .flat_map(|(owner, candidates)| { - candidates - .iter() - .filter(|candidate| eligible(candidate)) - .map(move |candidate| (owner, candidate.clone())) - }) - .collect(); - for (owner, original) in originals { - let boundaries = boundary_ptrs(memo, &original); - let mut inputs = Vec::new(); - for node in nodes_within(&original.root_node, &boundaries) { - if matches!( - node.asap(), - Some(ASAPOp::SummaryAgg { .. } | ASAPOp::SummaryTumblingWindowChain { .. }) - ) { - for child in node.children() { - if !boundaries.contains(&Rc::as_ptr(child)) { - inputs.push(Rc::clone(child)); - } - } - } - } - let queries = memo.groups[owner].queries.clone(); - let mut replacements = HashMap::new(); - for input in inputs { - canonical_preprocessing(memo, &input, &boundaries, &queries, &mut replacements)?; - } - if !replacements.is_empty() { - let reader = replace(&original.root_node, &replacements); - append_reader(memo, owner, &original, reader, original.origin.clone())?; - } - } - Ok(()) -} +use crate::pass2::memo::rules::Pass2RuleError; +use asap_types::stages::Memo; -fn canonical_preprocessing( - memo: &mut Memo, - node: &Rc, - boundaries: &HashSet<*const OperatorNode>, - queries: &[usize], - replacements: &mut HashMap<*const OperatorNode, Rc>, -) -> Result { - if replacements.contains_key(&Rc::as_ptr(node)) { - return Ok(true); - } - if boundaries.contains(&Rc::as_ptr(node)) { - // Logical TimeRange groups can remain distinct when their inputs - // differ structurally. The existing state-sharing rule nevertheless - // permits identical range descriptors over the same canonical source. - if let Some(group) = memo.groups.iter().find(|group| { - crate::pass2::memo::state_groups::same_range_input(&group.logical_node, node) - }) { - replacements.insert(Rc::as_ptr(node), Rc::clone(&group.logical_node)); - } - return Ok(true); - } - if !matches!( - node.non_asap(), - Some(NonASAPOp::Project { .. } | NonASAPOp::Filter { .. }) - ) { - return Ok(false); - } - // Predicates/projections can reference subqueries. Every dependency must - // be canonical, not only the row-stream child. - for child in node.children() { - if !canonical_preprocessing(memo, child, boundaries, queries, replacements)? { - return Ok(false); - } - } - let canonical = replace(node, replacements); - let group = append_producer( - memo, - canonical, - queries, - OriginOfCandidateForMemoGroup::SummaryInput, - // These operators introduce no approximate state. Their identity does - // not depend on how a downstream consumer divides its error budget. - 1, - )?; - replacements.insert( - Rc::as_ptr(node), - Rc::clone(&memo.groups[group.0 as usize].logical_node), - ); - Ok(true) +/// Rule 3, summary inputs: a value projection or filter below a summary build as its own input group. +pub(crate) fn factor_summary_inputs(memo: &mut Memo) -> Result<(), Pass2RuleError> { + let _ = memo; + todo!("#1207 PR 4: rule 3 over groups") } diff --git a/crates/logical-asap-optimization/src/pass2/memo/rules/summary_results.rs b/crates/logical-asap-optimization/src/pass2/memo/rules/summary_results.rs deleted file mode 100644 index 49d84f991..000000000 --- a/crates/logical-asap-optimization/src/pass2/memo/rules/summary_results.rs +++ /dev/null @@ -1,82 +0,0 @@ -//! Factor composite implementations at summary readouts. Each readout can then -//! choose a state configuration independently, without enumerating combinations -//! of all private states in the enclosing query. -use super::reuse::{append_producer, append_reader, eligible, replace}; -use crate::pass2::memo::state_groups::{ - boundary_ptrs, direct_input_builds, nodes_within, Pass2RuleError, -}; -use asap_types::ir::{ASAPOp, OperatorNode}; -use asap_types::stages::{Memo, OriginOfCandidateForMemoGroup}; -use std::collections::{HashMap, HashSet}; -use std::rc::Rc; - -pub(crate) fn factor_summary_results(memo: &mut Memo) -> Result { - let originals: Vec<_> = memo - .candidates - .iter() - .enumerate() - .flat_map(|(group, candidates)| { - candidates - .iter() - .filter(|candidate| eligible(candidate)) - .map(move |candidate| (group, candidate.clone())) - }) - .collect(); - let mut added = 0; - for (owner, original) in originals { - let builds = direct_input_builds(memo, &original); - if builds.len() < 2 { - continue; - } - let build_pointers: HashSet<_> = - builds.iter().map(|(build, _)| Rc::as_ptr(build)).collect(); - let boundaries = boundary_ptrs(memo, &original); - let mut readouts = Vec::new(); - for node in nodes_within(&original.root_node, &boundaries) { - if !Rc::ptr_eq(&node, &original.root_node) - && matches!( - node.asap(), - Some(ASAPOp::SummaryEstimate { .. } | ASAPOp::FinalizeExactAccumulator { .. }) - ) - { - let private_builds: HashSet<_> = OperatorNode::reachable(&node) - .iter() - .map(Rc::as_ptr) - .filter(|pointer| build_pointers.contains(pointer)) - .collect(); - if private_builds.len() == 1 { - readouts.push(Rc::clone(&node)); - } - } - } - if readouts.is_empty() { - continue; - } - let queries = memo.groups[owner].queries.clone(); - let mut replacements = HashMap::new(); - for readout in readouts { - let group = append_producer( - memo, - Rc::clone(&readout), - &queries, - OriginOfCandidateForMemoGroup::SummaryResult, - original.accuracy_budget_parts, - )?; - // The original reader's allocation was used to generate these - // parameters; lifting the boundary must not reset that provenance. - replacements.insert( - Rc::as_ptr(&readout), - Rc::clone(&memo.groups[group.0 as usize].logical_node), - ); - } - let reader = replace(&original.root_node, &replacements); - added += usize::from(append_reader( - memo, - owner, - &original, - reader, - OriginOfCandidateForMemoGroup::SummaryResult, - )?); - } - Ok(added) -} diff --git a/crates/logical-asap-optimization/src/pass2/memo/rules/topk.rs b/crates/logical-asap-optimization/src/pass2/memo/rules/topk.rs index 9593a317e..e3b1b22dc 100644 --- a/crates/logical-asap-optimization/src/pass2/memo/rules/topk.rs +++ b/crates/logical-asap-optimization/src/pass2/memo/rules/topk.rs @@ -1,267 +1,11 @@ //! Exact ranking reuse with explicit exact-input groups. No global selection //! flags are required: only exact implementations can supply these inputs. -use super::reuse::{append_producer, append_reader, eligible, replace}; -use super::shared_producer::{add_shared_producer_candidates, Reader, SharedProducer}; -use crate::pass2::asap_aware_cse::topk_reuse_candidates; -use crate::pass2::memo::state_groups::{boundary_ptrs, nodes_within, Pass2RuleError}; -use asap_types::ir::operator::{AggIntent, Reduction}; -use asap_types::ir::properties::AccuracyGuarantee; -use asap_types::ir::schema::FieldDataType; -use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode, QueryRoot}; -use asap_types::stages::MemoGroupId; -use asap_types::stages::{Memo, OriginOfCandidateForMemoGroup}; -use std::collections::HashMap; -use std::rc::Rc; -pub(crate) fn add_topk_candidates(memo: &mut Memo) -> Result { - add_shared_producer_candidates( - memo, - &mut TopKRankings { - exact_groups: HashMap::new(), - }, - ) -} - -/// Exact top-k readers of one input and partition with different `k`: the -/// covering need is the ranking with the largest `k`, `Limit(k_max)` over -/// `Sort` of an exact input ([`topk_reuse_candidates`]); each reader derives -/// `Limit(k) ▸ Project` from it. -struct TopKRankings { - /// Each input group's exact-only group ([`exact_input_group`]). - exact_groups: HashMap>, -} - -/// One input's top-k readers' shared ranking: `Limit(k_max)` over `Sort`, -/// and the readers' groups, whose queries the producer serves. -struct SharedRanking { - ranking: Rc, - reader_groups: Vec, -} - -impl SharedProducer for TopKRankings { - /// The ranking of the reader's input and partition, at the largest `k` - /// of its set. - type Need = Rc; - type Covering = SharedRanking; - - fn sets(&mut self, memo: &Memo) -> Result>>>, Pass2RuleError> { - let mut owners = HashMap::new(); - let mut roots = Vec::new(); - for (group, entry) in memo.groups.iter().enumerate() { - let [input] = entry.logical_node.children()[..] else { - continue; - }; - let exact = Rc::new( - (**input) - .clone() - .with_guarantee(Some(AccuracyGuarantee::exact( - "exact input candidate group", - ))), - ); - let Ok(copy) = entry.logical_node.with_new_children(|_| Rc::clone(&exact)) else { - continue; - }; - let copy = Rc::new(copy); - owners.insert(Rc::as_ptr(©), group); - roots.push(QueryRoot::Operator(copy)); - } - Ok(topk_reuse_candidates(&roots) - .into_iter() - .map(|reuse| { - reuse - .members - .iter() - .filter_map(|node| { - Some(Reader { - group: *owners.get(&Rc::as_ptr(node))?, - need: Rc::clone(&reuse.ranking), - }) - }) - .collect() - }) - .collect()) - } - - fn coverings( - &self, - set: &[Reader>], - ) -> Result, Pass2RuleError> { - // The largest `k` holds every smaller top-k, so it serves the set. - Ok(vec![SharedRanking { - ranking: Rc::clone(&set[0].need), - reader_groups: set.iter().map(|reader| reader.group).collect(), - }]) - } - - fn derive( - &mut self, - memo: &mut Memo, - reader: &Reader>, - covering: &SharedRanking, - ) -> Result { - // Interning the ranking is idempotent, so every reader of the set - // reads the producer the first one interned. - let Some(input_group) = memo.groups[covering.reader_groups[0]] - .child_groups - .first() - .copied() - else { - return Ok(0); - }; - let Some(exact_input) = exact_input_group(memo, input_group, &mut self.exact_groups)? - else { - return Ok(0); - }; - let input = Rc::clone(&memo.groups[exact_input.0 as usize].logical_node); - let queries: Vec<_> = covering - .reader_groups - .iter() - .flat_map(|&group| memo.groups[group].queries.iter().copied()) - .collect::>() - .into_iter() - .collect(); - let sort = covering.ranking.children()[0] - .with_new_children(|_| Rc::clone(&input)) - .map_err(|error| Pass2RuleError::ReuseRule(error.to_string()))?; - let sort_group = append_producer( - memo, - Rc::new(sort), - &queries, - OriginOfCandidateForMemoGroup::TopKReuse, - 1, - )?; - let sort = Rc::clone(&memo.groups[sort_group.0 as usize].logical_node); - let ranking = covering - .ranking - .with_new_children(|_| Rc::clone(&sort)) - .map_err(|error| Pass2RuleError::ReuseRule(error.to_string()))?; - let ranking_group = append_producer( - memo, - Rc::new(ranking), - &queries, - OriginOfCandidateForMemoGroup::TopKReuse, - 1, - )?; - let ranking = Rc::clone(&memo.groups[ranking_group.0 as usize].logical_node); - let owner = reader.group; - let logical = Rc::clone(&memo.groups[owner].logical_node); - let Some(NonASAPOp::Aggregate { - reduction: Reduction::Reduce(keys), - measures, - .. - }) = logical.non_asap() - else { - return Ok(0); - }; - let [AggIntent::TopK { k, .. }] = measures.as_slice() else { - return Ok(0); - }; - let Some(original) = memo.candidates[owner].iter().find(|candidate| eligible(candidate) && matches!(candidate.root_node.non_asap(), Some(NonASAPOp::Aggregate {measures, ..}) if matches!(measures.as_slice(), [AggIntent::TopK {..}]))).cloned() else { return Ok(0) }; - let limited = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Limit { - n: Some(*k), - offset: 0, - partition_by: keys.clone(), - child: Rc::clone(&ranking), - })) - .map_err(|error| Pass2RuleError::ReuseRule(error.to_string()))?; - let columns = asap_types::ir::schema::topk::ranked_row_projection( - &ranking, - &Reduction::Reduce(keys.clone()), - &logical.schema.fields[keys.len()].name, - ) - .map_err(|error| Pass2RuleError::ReuseRule(error.to_string()))?; - let reader = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Project { - cols: columns, - qualifier: None, - child: limited, - })) - .map_err(|error| Pass2RuleError::ReuseRule(error.to_string()))?; - Ok(usize::from(append_reader( - memo, - owner, - &original, - reader, - OriginOfCandidateForMemoGroup::TopKReuse, - )?)) - } -} +use crate::pass2::memo::rules::Pass2RuleError; +use asap_types::stages::Memo; -fn exact_input_group( - memo: &mut Memo, - group: MemoGroupId, - known: &mut HashMap>, -) -> Result, Pass2RuleError> { - if let Some(result) = known.get(&group) { - return Ok(*result); - } - // An in-progress marker also prevents malformed cyclic inventories recursing. - known.insert(group, None); - let originals = memo.candidates[group.0 as usize].clone(); - let original_children: Vec<_> = originals - .iter() - .map(|candidate| candidate.input_groups.clone()) - .collect(); - let mut candidates = Vec::new(); - for mut candidate in originals { - if !eligible(&candidate) { - continue; - } - let boundaries = boundary_ptrs(memo, &candidate); - let mut exact = !nodes_within(&candidate.root_node, &boundaries).iter().any(|node| { - matches!(node.asap(), Some(ASAPOp::SummaryAgg {family, ..} | ASAPOp::SummaryTumblingWindowChain {family, ..}) if !matches!(family, FieldDataType::ExactAggregate(..))) - }); - if !exact { - continue; - } - let mut replacements = HashMap::new(); - let mut children = Vec::new(); - for child in &candidate.input_groups { - let Some(exact_child) = exact_input_group(memo, *child, known)? else { - exact = false; - break; - }; - replacements.insert( - Rc::as_ptr(&memo.groups[child.0 as usize].logical_node), - Rc::clone(&memo.groups[exact_child.0 as usize].logical_node), - ); - children.push(exact_child); - } - if !exact { - continue; - } - candidate.root_node = replace(&candidate.root_node, &replacements); - candidate.input_groups = children; - candidates.push(candidate); - } - if candidates.is_empty() { - return Ok(None); - } - if candidates.len() == original_children.len() - && candidates - .iter() - .zip(&original_children) - .all(|(candidate, children)| candidate.input_groups == *children) - { - // An already exact-only group remains the shared source for ordinary - // consumers too; cloning it would unnecessarily split scan/state reuse. - known.insert(group, Some(group)); - return Ok(Some(group)); - } - let node = Rc::new((*candidates[0].root_node).clone().with_guarantee(Some( - AccuracyGuarantee::exact("only exact implementations and exact inputs"), - ))); - let queries = memo.groups[group.0 as usize].queries.clone(); - let id = append_producer( - memo, - node, - &queries, - OriginOfCandidateForMemoGroup::TopKReuse, - 1, - )?; - for candidate in &mut candidates { - candidate.fused_groups = Vec::new(); - } - memo.candidates[id.0 as usize] = candidates; - known.insert(group, Some(id)); - Ok(Some(id)) +/// Rule 4, top-k reuse: a candidate of each top-k group reading a larger-`k` group, with an exact-input requirement. +pub(crate) fn add_topk_candidates(memo: &mut Memo) -> Result<(), Pass2RuleError> { + let _ = memo; + todo!("#1207 PR 4: rule 4 over groups") } diff --git a/crates/logical-asap-optimization/src/pass2/memo/shared_configurations.rs b/crates/logical-asap-optimization/src/pass2/memo/shared_configurations.rs deleted file mode 100644 index 9eb61a2db..000000000 --- a/crates/logical-asap-optimization/src/pass2/memo/shared_configurations.rs +++ /dev/null @@ -1,326 +0,0 @@ -//! Finite state configurations shared by memo rules and candidate selection. -use crate::stage0::cse::same_value; -use asap_accuracy::estimators::size_for_all_readers; -use asap_types::ir::schema::{ - sketch::{SketchKind, SketchParams}, - FieldDataType, -}; -use asap_types::workload::greatest_common_divisor; -use std::collections::BTreeSet; - -/// Every tumbling-window width that divides one of `grids` (zero grids -/// ignored), in ascending order. -pub(crate) fn tumbling_window_width_domain(grids: impl IntoIterator) -> Vec { - let grids: BTreeSet = grids.into_iter().filter(|&grid| grid > 0).collect(); - let mut widths = BTreeSet::new(); - for grid in grids { - widths.extend(tumbling_window_width_divisors(grid)); - } - widths.into_iter().collect() -} - -#[derive(Debug, Clone)] -pub(crate) struct TumblingWindowRequirement { - pub tumbling_window_width_ms: u64, - pub window_ms: u64, - pub family: FieldDataType, -} -#[derive(Debug, Clone, PartialEq)] -pub(crate) struct TumblingWindowConfiguration { - pub tumbling_window_width_ms: u64, - pub longest_window_ms: u64, - pub family: FieldDataType, -} -impl TumblingWindowConfiguration { - pub fn serves(&self, reader: &TumblingWindowRequirement) -> bool { - self.tumbling_window_width_ms > 0 - && reader - .tumbling_window_width_ms - .is_multiple_of(self.tumbling_window_width_ms) - && reader - .window_ms - .is_multiple_of(self.tumbling_window_width_ms) - && self - .longest_window_ms - .is_multiple_of(self.tumbling_window_width_ms) - && reader.window_ms <= self.longest_window_ms - && shared_size(&self.family, &reader.family) - .is_some_and(|joined| same_value(&joined, &self.family)) - } -} -/// Every positive integer-millisecond divisor of a grid, in ascending order. -/// The time representation fixes the search resolution; no representative-width -/// sampling or arbitrary candidate cap changes this domain. -pub(crate) fn tumbling_window_width_divisors(grid_ms: u64) -> Vec { - let mut widths = BTreeSet::new(); - let mut divisor = 1; - while divisor <= grid_ms / divisor { - if grid_ms.is_multiple_of(divisor) { - widths.insert(divisor); - widths.insert(grid_ms / divisor); - } - divisor += 1; - } - widths.into_iter().collect() -} - -/// Complete divisor grids, distinct required horizons, and joined state sizes. -/// This enumerates configurations of one compatible producer, not workload plans. -pub(crate) fn tumbling_window_configurations( - readers: &[TumblingWindowRequirement], -) -> Vec { - if readers - .iter() - .any(|r| r.tumbling_window_width_ms == 0 || r.window_ms == 0) - { - return Vec::new(); - } - let grids = - tumbling_window_width_domain(readers.iter().map(|reader| { - greatest_common_divisor(reader.tumbling_window_width_ms, reader.window_ms) - })); - let mut configurations = Vec::new(); - for &tumbling_window_width_ms in &grids { - let eligible: Vec<&TumblingWindowRequirement> = readers - .iter() - .filter(|r| { - r.tumbling_window_width_ms - .is_multiple_of(tumbling_window_width_ms) - && r.window_ms.is_multiple_of(tumbling_window_width_ms) - }) - .collect(); - let windows: BTreeSet = eligible.iter().map(|r| r.window_ms).collect(); - // The sizes: the readers' own, closed under the shared size. - let mut sizes: Vec = Vec::new(); - for reader in &eligible { - if !sizes.iter().any(|known| same_value(known, &reader.family)) { - sizes.push(reader.family.clone()); - } - } - let mut i = 0; - while i < sizes.len() { - for j in 0..i { - if let Some(joined) = shared_size(&sizes[i], &sizes[j]) { - if !sizes.iter().any(|known| same_value(known, &joined)) { - sizes.push(joined); - } - } - } - i += 1; - } - for &longest_window_ms in &windows { - for family in &sizes { - let configuration = TumblingWindowConfiguration { - tumbling_window_width_ms, - longest_window_ms, - family: family.clone(), - }; - let admitted: Vec<&&TumblingWindowRequirement> = eligible - .iter() - .filter(|r| configuration.serves(r)) - .collect(); - let window = admitted.iter().map(|r| r.window_ms).max(); - let size = admitted.iter().try_fold(None::, |acc, r| { - Some(Some(match acc { - None => r.family.clone(), - Some(acc) => shared_size(&acc, &r.family)?, - })) - }); - if window == Some(longest_window_ms) - && size.flatten().is_some_and(|size| same_value(&size, family)) - { - configurations.push(configuration); - } - } - } - } - configurations -} - -/// All distinct joined reader requirements, retaining recorded error contracts. -/// This is configuration generation only; sharing is represented by state groups. -pub(crate) fn state_size_configurations( - requirements: Vec>, -) -> Vec> { - if requirements.iter().any(Option::is_none) { - return vec![None]; - } - let requirements: Vec = requirements.into_iter().flatten().collect(); - let mut joins: Vec = Vec::new(); - for requirement in &requirements { - let mut grown = vec![requirement.clone()]; - for join in &joins { - if let Some(joined) = size_for_all_readers(&[join, requirement]) { - grown.push(joined); - } - } - for join in grown { - if !joins.iter().any(|known| same_size(known, &join)) { - joins.push(join); - } - } - } - let mut configurations = Vec::new(); - for configuration in joins { - let admitted: Vec<_> = requirements - .iter() - .filter(|required| { - size_for_all_readers(&[&configuration, required]) - .is_some_and(|joined| same_size(&joined, &configuration)) - }) - .collect(); - if size_for_all_readers(&admitted).is_some_and(|joined| same_size(&joined, &configuration)) - { - configurations.push(Some(configuration)); - } - } - configurations -} - -fn same_size(a: &SketchParams, b: &SketchParams) -> bool { - a == b && format!("{a:?}") == format!("{b:?}") -} - -pub(crate) fn shared_size(a: &FieldDataType, b: &FieldDataType) -> Option { - match (a, b) { - (FieldDataType::Sketch(a, grouping), FieldDataType::Sketch(b, grouping_b)) - if a.algorithm() == b.algorithm() && grouping == grouping_b => - { - let params = size_for_all_readers(&[a.params(), b.params()])?; - Some(FieldDataType::Sketch( - SketchKind::new(*a.algorithm(), params), - grouping.clone(), - )) - } - (FieldDataType::Sketch(..), _) | (_, FieldDataType::Sketch(..)) => None, - _ => (a == b).then(|| a.clone()), - } -} - -#[cfg(test)] -mod tests { - use super::*; - use asap_types::ir::schema::sketch::{GroupingStrategy, SketchAlgorithm, SketchParams}; - - fn kll(probability: Option) -> FieldDataType { - FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: 331, - failure_probability: probability, - }, - ), - GroupingStrategy::default(), - ) - } - - // State-size generation retains both local and shared capacities without - // depending on the removed implicit coupling-site representation. - #[test] - fn state_sizes_keep_each_distinct_reader_configuration() { - let small = SketchParams::Kll { - k: 100, - failure_probability: None, - }; - let large = SketchParams::Kll { - k: 200, - failure_probability: None, - }; - let configurations = state_size_configurations(vec![ - Some(small.clone()), - Some(large.clone()), - Some(small.clone()), - ]); - assert_eq!(configurations.len(), 2); - assert!(configurations.contains(&Some(small))); - assert!(configurations.contains(&Some(large))); - assert_eq!(state_size_configurations(vec![None, None]), vec![None]); - } - - // Equal capacity does not erase the stricter recorded failure probability. - #[test] - fn state_size_join_preserves_probability_metadata() { - let weak = SketchParams::Kll { - k: 331, - failure_probability: Some(0.01), - }; - let strict = SketchParams::Kll { - k: 331, - failure_probability: Some(0.001), - }; - let configurations = state_size_configurations(vec![Some(weak), Some(strict.clone())]); - assert!(configurations - .iter() - .flatten() - .any(|value| same_size(value, &strict))); - } - - // Equal physical capacities still carry distinct recorded error contracts; - // only the joined contract may serve the stricter probability requirement. - #[test] - fn tumbling_window_configuration_keeps_failure_probability_contract() { - let weaker = TumblingWindowConfiguration { - tumbling_window_width_ms: 1000, - longest_window_ms: 5000, - family: kll(None), - }; - let strict = TumblingWindowRequirement { - tumbling_window_width_ms: 1000, - window_ms: 5000, - family: kll(Some(0.001)), - }; - assert!(!weaker.serves(&strict)); - let configurations = tumbling_window_configurations(&[ - TumblingWindowRequirement { - family: kll(None), - ..strict.clone() - }, - strict.clone(), - ]); - assert!(configurations - .iter() - .any(|configuration| configuration.serves(&strict))); - assert!(configurations - .iter() - .all(|configuration| 1000_u64.is_multiple_of(configuration.tumbling_window_width_ms))); - } - - // Every integer-millisecond width that divides the grid is offered, not - // just the coarsest width or GCDs of the readers' original configurations. - #[test] - fn tumbling_window_width_domain_contains_every_legal_divisor() { - let reader = TumblingWindowRequirement { - tumbling_window_width_ms: 12, - window_ms: 60, - family: kll(None), - }; - let widths: BTreeSet<_> = tumbling_window_configurations(&[reader]) - .into_iter() - .map(|configuration| configuration.tumbling_window_width_ms) - .collect(); - assert_eq!(widths, BTreeSet::from([1, 2, 3, 4, 6, 12])); - } - - // Compare the divisor generator to direct enumeration, including zero. - #[test] - fn width_divisors_match_complete_integer_domain() { - for grid in 0..=256_u64 { - let expected: Vec<_> = (1..=grid) - .filter(|width| grid.is_multiple_of(*width)) - .collect(); - assert_eq!(tumbling_window_width_divisors(grid), expected); - } - } - - // Public finite enumeration rejects invalid grids instead of dividing by zero. - #[test] - fn invalid_tumbling_window_grid_has_no_configurations() { - assert!(tumbling_window_configurations(&[TumblingWindowRequirement { - tumbling_window_width_ms: 0, - window_ms: 5000, - family: kll(None) - }]) - .is_empty()); - } -} diff --git a/crates/logical-asap-optimization/src/pass2/memo/state_groups.rs b/crates/logical-asap-optimization/src/pass2/memo/state_groups.rs deleted file mode 100644 index e0c5bf94c..000000000 --- a/crates/logical-asap-optimization/src/pass2/memo/state_groups.rs +++ /dev/null @@ -1,1615 +0,0 @@ -//! Opt-in concrete summary-state groups for the candidate-connection planner. -//! -//! Simple summary builds and tumbling-window chains directly over canonical input groups -//! gain finite compatible sizes and tumbling-window horizons before exact state interning. -//! Concrete state identity includes capacity, guarantee, update/filter semantics, -//! and materialization. Supported private projections/filters gain explicit input groups. - -use std::collections::{BTreeMap, HashMap, HashSet}; -use std::rc::Rc; - -use asap_types::ir::{ASAPOp, NonASAPOp, OperatorNode}; - -use asap_types::stages::{CandidateForMemoGroup, Memo, OriginOfCandidateForMemoGroup}; -use asap_types::stages::{MemoGroup, MemoGroupId}; - -/// Why Pass 2's rules failed on a memo: its precondition check, a rule -/// (2–9), or the shared summary-state groups (10). -#[derive(Debug, thiserror::Error)] -pub enum Pass2RuleError { - #[error("memo group and candidate vectors have different lengths")] - MismatchedGroups, - #[error("memo contains an out-of-range group reference")] - InvalidGroupReference, - #[error("memo group capacity exceeds the group identifier range")] - TooManyGroups, - #[error("memo dependencies contain a cycle")] - CyclicDependencies, - #[error("explicit summary-state groups have already been introduced")] - AlreadyExpanded, - #[error("scan-label candidate expansion: {0}")] - ScanLabelWidening(String), - #[error("state configuration expansion: {0}")] - StateConfiguration(String), - #[error("producer reuse rule: {0}")] - ReuseRule(String), -} - -struct BuildReaders { - build: Rc, - children: Vec, - readers: Vec<(usize, usize, Rc)>, -} - -/// Rejects a memo the Pass 2 rules cannot extend: mismatched or -/// out-of-range group references, shared summary-state groups already -/// added, or cyclic dependencies. Pass 2 checks this before its rules. -pub(crate) fn check_unexpanded_memo(memo: &Memo) -> Result<(), Pass2RuleError> { - validate_references(memo)?; - if memo.groups.iter().any(|group| { - matches!( - group.logical_node.asap(), - Some(ASAPOp::SummaryAgg { .. } | ASAPOp::SummaryTumblingWindowChain { .. }) - ) - }) { - return Err(Pass2RuleError::AlreadyExpanded); - } - if memo.topological_order().is_none() { - return Err(Pass2RuleError::CyclicDependencies); - } - Ok(()) -} - -/// Pass 2's last step, after its rules: give each summary build that two or -/// more candidates make identically its own group, and add to each reader -/// group a candidate reading that group. Every existing candidate stays -/// (unsupported candidate shapes unchanged). Then rebuilds every group's -/// parents and the queries it may serve. On error, discard `memo`: it may -/// hold partial work. -pub(crate) fn add_shared_summary_state_groups(memo: &mut Memo) -> Result<(), Pass2RuleError> { - add_shared_summary_state_groups_with_index(memo, true) -} - -/// `indexed` looks builds up by fingerprint; otherwise by a linear scan -/// (the reference the index is tested against). -fn add_shared_summary_state_groups_with_index( - memo: &mut Memo, - indexed: bool, -) -> Result<(), Pass2RuleError> { - let mut builds: Vec = Vec::new(); - let mut build_index: HashMap, Vec> = HashMap::new(); - for (group, candidates) in memo.candidates.iter().enumerate() { - for (index, candidate) in candidates.iter().enumerate() { - for (build, children) in direct_input_builds(memo, candidate) { - let fingerprint = build_fingerprint(&build); - let bucket = build_index.entry(fingerprint).or_default(); - let matching = if indexed { - bucket - .iter() - .copied() - .find(|&index| same_concrete_build(&builds[index].build, &build)) - } else { - builds - .iter() - .position(|entry| same_concrete_build(&entry.build, &build)) - }; - if let Some(build_index) = matching { - builds[build_index].readers.push((group, index, build)); - } else { - bucket.push(builds.len()); - builds.push(BuildReaders { - build: Rc::clone(&build), - children, - readers: vec![(group, index, build)], - }); - } - } - } - } - let mut state_groups = Vec::new(); - let mut replacements: BTreeMap<(usize, usize), HashMap<*const OperatorNode, Rc>> = - BTreeMap::new(); - for build in builds { - // Repeated private builds within one result are reusable too. The - // reader list records concrete build occurrences, not just owners. - if build.readers.len() < 2 { - continue; - } - let state_group = MemoGroupId( - u32::try_from(memo.groups.len()).map_err(|_| Pass2RuleError::TooManyGroups)?, - ); - let mut queries: Vec<_> = build - .readers - .iter() - .flat_map(|(group, _, _)| memo.groups[*group].queries.iter().copied()) - .collect(); - queries.sort_unstable(); - queries.dedup(); - memo.groups.push(MemoGroup { - logical_node: Rc::clone(&build.build), - child_groups: build.children.clone(), - parents: Vec::new(), - queries, - }); - // The build is its readers' own build, so it is sized for their - // shares of the budget (equal builds have equal sizes). - let accuracy_budget_parts = build - .readers - .iter() - .map(|(group, index, _)| memo.candidates[*group][*index].accuracy_budget_parts) - .max() - .unwrap_or(1); - memo.candidates.push(vec![CandidateForMemoGroup { - fused_groups: Vec::new(), - root_node: Rc::clone(&build.build), - input_groups: build.children, - origin: OriginOfCandidateForMemoGroup::SharedBuild, - accuracy_budget_parts, - }]); - for (group, index, own_build) in build.readers { - replacements - .entry((group, index)) - .or_default() - .insert(Rc::as_ptr(&own_build), Rc::clone(&build.build)); - } - state_groups.push(state_group); - } - for ((group, index), replacements) in replacements { - // Factor every available producer boundary together. A producer used by - // only one selected reader is still legal; no state-subset enumeration - // is required. Retain the original entirely local implementation too. - let mut reader = memo.candidates[group][index].clone(); - // Only declared inputs and newly lifted states are boundaries. A - // factored result group's representative can equal its candidate root; - // treating every memo node as an input would create a self-alias here. - let boundaries = reader - .input_groups - .iter() - .chain(&state_groups) - .map(|id| (Rc::as_ptr(&memo.groups[id.0 as usize].logical_node), *id)) - .collect(); - reader.root_node = super::rules::reuse::replace(&reader.root_node, &replacements); - reader.input_groups = boundary_groups(&reader.root_node, &boundaries); - reader.origin = OriginOfCandidateForMemoGroup::SharedBuild; - memo.candidates[group].push(reader); - } - // Parents describe every possible group/candidate dependency, not a selected plan. - for group in &mut memo.groups { - group.parents.clear(); - } - for group in 0..memo.groups.len() { - let dependencies: HashSet<_> = memo.groups[group] - .child_groups - .iter() - .chain( - memo.candidates[group] - .iter() - .flat_map(|candidate| &candidate.input_groups), - ) - .copied() - .collect(); - for child in dependencies { - memo.groups[child.0 as usize] - .parents - .push(MemoGroupId(group as u32)); - } - } - if memo.topological_order().is_none() { - return Err(Pass2RuleError::CyclicDependencies); - } - // The chosen representative input may now serve additional queries through - // state reuse; propagate potential reach along the expanded dependency DAG. - for group in memo - .topological_order() - .ok_or(Pass2RuleError::CyclicDependencies)? - .into_iter() - .rev() - { - let queries = memo.groups[group.0 as usize].queries.clone(); - let children: HashSet<_> = memo.groups[group.0 as usize] - .child_groups - .iter() - .chain( - memo.candidates[group.0 as usize] - .iter() - .flat_map(|candidate| &candidate.input_groups), - ) - .copied() - .collect(); - for child in children { - memo.groups[child.0 as usize].queries.extend(&queries); - memo.groups[child.0 as usize].queries.sort_unstable(); - memo.groups[child.0 as usize].queries.dedup(); - } - } - Ok(()) -} - -fn validate_references(memo: &Memo) -> Result<(), Pass2RuleError> { - if memo.groups.len() != memo.candidates.len() { - return Err(Pass2RuleError::MismatchedGroups); - } - if memo.groups.len() > u32::MAX as usize { - return Err(Pass2RuleError::TooManyGroups); - } - for group in &memo.groups { - if group - .child_groups - .iter() - .chain(&group.parents) - .any(|id| id.0 as usize >= memo.groups.len()) - { - return Err(Pass2RuleError::InvalidGroupReference); - } - } - for candidate in memo.candidates.iter().flatten() { - if candidate - .input_groups - .iter() - .chain(&candidate.fused_groups) - .any(|id| id.0 as usize >= memo.groups.len()) - { - return Err(Pass2RuleError::InvalidGroupReference); - } - } - Ok(()) -} - -pub(super) fn simple_build( - memo: &Memo, - candidate: &CandidateForMemoGroup, -) -> Option<(Rc, Vec)> { - let mut builds = direct_input_builds(memo, candidate); - (builds.len() == 1).then(|| builds.remove(0)) -} - -pub(super) fn direct_input_builds( - memo: &Memo, - candidate: &CandidateForMemoGroup, -) -> Vec<(Rc, Vec)> { - if !matches!( - candidate.origin, - OriginOfCandidateForMemoGroup::Pass1 - | OriginOfCandidateForMemoGroup::TumblingWindows - | OriginOfCandidateForMemoGroup::WidenedScans - | OriginOfCandidateForMemoGroup::Rollup - | OriginOfCandidateForMemoGroup::SummaryResult - ) || candidate.accuracy_budget_parts == 0 - { - return Vec::new(); - } - let boundaries: HashMap<_, _> = candidate - .input_groups - .iter() - .map(|&group| { - ( - Rc::as_ptr(&memo.groups[group.0 as usize].logical_node), - group, - ) - }) - .collect(); - let mut stack = vec![Rc::clone(&candidate.root_node)]; - let mut seen = HashSet::new(); - let mut builds = Vec::new(); - while let Some(node) = stack.pop() { - if boundaries.contains_key(&Rc::as_ptr(&node)) || !seen.insert(Rc::as_ptr(&node)) { - continue; - } - if !Rc::ptr_eq(&node, &candidate.root_node) - && matches!( - node.asap(), - Some(ASAPOp::SummaryAgg { .. } | ASAPOp::SummaryTumblingWindowChain { .. }) - ) - { - let children: Option> = node - .children() - .iter() - .map(|child| boundaries.get(&Rc::as_ptr(child)).copied()) - .collect(); - if let Some(mut children) = children { - children.dedup(); - builds.push((Rc::clone(&node), children)); - } - } - stack.extend(node.children().into_iter().cloned()); - } - builds -} - -pub(super) fn boundary_groups( - node: &Rc, - boundaries: &HashMap<*const OperatorNode, MemoGroupId>, -) -> Vec { - let mut found = Vec::new(); - let mut stack = vec![Rc::clone(node)]; - let mut seen = HashSet::new(); - while let Some(node) = stack.pop() { - if !seen.insert(Rc::as_ptr(&node)) { - continue; - } - if let Some(&group) = boundaries.get(&Rc::as_ptr(&node)) { - if !found.contains(&group) { - found.push(group); - } - } else { - stack.extend(node.children().into_iter().rev().cloned()); - } - } - found -} - -/// The nodes of `candidate`'s input groups, where its own nodes end. -pub(super) fn boundary_ptrs( - memo: &Memo, - candidate: &CandidateForMemoGroup, -) -> HashSet<*const OperatorNode> { - candidate - .input_groups - .iter() - .map(|group| Rc::as_ptr(&memo.groups[group.0 as usize].logical_node)) - .collect() -} - -/// `root` and the nodes beneath it down to `boundaries` (excluded), each -/// once, in depth-first order (last child first). -pub(super) fn nodes_within( - root: &Rc, - boundaries: &HashSet<*const OperatorNode>, -) -> Vec> { - let mut pending = vec![Rc::clone(root)]; - let mut seen = HashSet::new(); - let mut nodes = Vec::new(); - while let Some(node) = pending.pop() { - if boundaries.contains(&Rc::as_ptr(&node)) || !seen.insert(Rc::as_ptr(&node)) { - continue; - } - pending.extend(node.children().into_iter().cloned()); - nodes.push(node); - } - nodes -} - -// Hashing narrows comparisons only; full strict equality below remains the -// authority, including probability metadata and floating-point corner cases. -#[derive(Hash, PartialEq, Eq)] -struct BuildFingerprint { - local: String, - inputs: Vec<(usize, Option)>, -} - -fn local_fingerprint(node: &OperatorNode) -> Option { - serde_json::to_string(&( - node.operator.map_children(|_| ()), - &node.schema, - node.result_kind, - &node.guarantee, - node.timing, - node.materialization, - )) - .ok() -} - -fn build_fingerprint(node: &OperatorNode) -> Option { - let inputs = node - .children() - .into_iter() - .map(|input| { - match input.non_asap() { - Some(NonASAPOp::TimeRange { child, .. }) => { - // Exactly the existing same_range_input contract: identical - // local range semantics over the same canonical source. - Some((Rc::as_ptr(child) as usize, Some(local_fingerprint(input)?))) - } - _ => Some((Rc::as_ptr(input) as usize, None)), - } - }) - .collect::>>()?; - Some(BuildFingerprint { - local: local_fingerprint(node)?, - inputs, - }) -} - -// Immediate inputs are canonical memo nodes. Compare local fields without -// recursively walking their potentially shared DAGs; serialization additionally -// distinguishes signed-zero parameters, matching conservative CSE semantics. -fn same_concrete_build(left: &OperatorNode, right: &OperatorNode) -> bool { - let left_inputs = left.children(); - let right_inputs = right.children(); - left_inputs.len() == right_inputs.len() - && left_inputs - .iter() - .zip(right_inputs) - .all(|(left, right)| Rc::ptr_eq(left, right) || same_range_input(left, right)) - && same_local_fields(left, right) -} - -// TimeRange lacks a unique row key and therefore is not generally memo-interned. -// Equal ranges over the same canonical input nevertheless feed identical data -// into a summary build, as the existing ASAP-aware summary reuse rule allows. -pub(super) fn same_range_input(left: &OperatorNode, right: &OperatorNode) -> bool { - let ( - Some(NonASAPOp::TimeRange { - child: left_child, .. - }), - Some(NonASAPOp::TimeRange { - child: right_child, .. - }), - ) = (left.non_asap(), right.non_asap()) - else { - return false; - }; - Rc::ptr_eq(left_child, right_child) && same_local_fields(left, right) -} - -fn same_local_fields(left: &OperatorNode, right: &OperatorNode) -> bool { - let signature = |node: &OperatorNode| { - ( - node.operator.map_children(|_| ()), - node.result_kind, - node.schema.clone(), - node.guarantee.clone(), - node.timing, - node.materialization, - ) - }; - let left = signature(left); - let right = signature(right); - left == right - && matches!((serde_json::to_string(&left), serde_json::to_string(&right)), (Ok(left), Ok(right)) if left == right) -} - -#[cfg(test)] -fn replace_node( - node: &Rc, - old: &Rc, - new: &Rc, -) -> Rc { - super::rules::reuse::replace(node, &HashMap::from([(Rc::as_ptr(old), Rc::clone(new))])) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::pass1::memo_population::try_populate_pass1; - use crate::pass1::per_computation_candidates::generate_per_computation_candidates; - use crate::stage0::generate_dag_of_logical_memo_groups; - use asap_types::ir::properties::AccuracyTarget; - use asap_types::ir::QueryRoot; - - /// An expanded memo with what its shared summary-state step added. - struct Expanded { - memo: Memo, - /// The groups the shared summary-state step added. - state_groups: Vec, - /// The candidates it added to reader groups. - added_reader_candidates: usize, - } - - /// Pass 2 after local window composition, over the complete tumbling-window-width - /// domain, as the Pass 2 entry runs it. - fn expand(memo: Memo) -> Result { - let memo = crate::test_support::sharing_rules_and_shared_state_groups(memo)?; - Ok(expanded(memo)) - } - - fn expanded(memo: Memo) -> Expanded { - let state_groups = crate::test_support::shared_summary_state_groups(&memo); - // Before this step no candidate is a shared build; after it, each - // state group has one and each added reader candidate is one. - let shared_builds = memo - .candidates - .iter() - .flatten() - .filter(|candidate| candidate.origin == OriginOfCandidateForMemoGroup::SharedBuild) - .count(); - Expanded { - added_reader_candidates: shared_builds - state_groups.len(), - state_groups, - memo, - } - } - - // Hash indexing is only an acceleration: independently run the original - // linear state lookup and compare every generated candidate and state. - #[test] - fn indexed_state_interning_preserves_linear_catalog() { - let memo = with_local_tumbling_windows( - workload(&[ - ("quantile_over_time(0.95, latency[5m])", 0.01), - ("quantile_over_time(0.99, latency[15m])", 0.02), - ("quantile_over_time(0.9, latency{job=\"other\"}[5m])", 0.01), - ]), - &[6, 6, 6], - ); - let mut memo = memo; - crate::pass2::logical_memo_candidates::apply_sharing_rules(&mut memo).unwrap(); - let mut linear = memo.clone(); - let mut indexed = memo; - add_shared_summary_state_groups_with_index(&mut indexed, true).unwrap(); - add_shared_summary_state_groups_with_index(&mut linear, false).unwrap(); - let (indexed, linear) = (expanded(indexed), expanded(linear)); - assert_eq!(indexed.state_groups, linear.state_groups); - assert_eq!(indexed.memo.groups.len(), linear.memo.groups.len()); - for (left, right) in indexed.memo.candidates.iter().zip(&linear.memo.candidates) { - let signatures = |candidates: &[CandidateForMemoGroup]| { - candidates - .iter() - .map(|candidate| { - ( - candidate.input_groups.clone(), - candidate.accuracy_budget_parts, - serde_json::to_string( - &asap_types::ir::flat::flatten(&[QueryRoot::Operator(Rc::clone( - &candidate.root_node, - ))]) - .0, - ) - .unwrap(), - ) - }) - .collect::>() - }; - assert_eq!(signatures(left), signatures(right)); - } - for (left, right) in indexed.memo.groups.iter().zip(&linear.memo.groups) { - assert_eq!(left.child_groups, right.child_groups); - assert_eq!(left.parents, right.parents); - assert_eq!(left.queries, right.queries); - } - } - - fn workload(queries: &[(&str, f64)]) -> Memo { - workload_with_identity(queries, false) - } - - fn workload_with_identity(queries: &[(&str, f64)], identity: bool) -> Memo { - let roots: Vec<_> = queries - .iter() - .map(|(query, error)| { - let root = crate::test_support::lower_promql( - query, - AccuracyTarget::max_error(*error).failure_probability(0.001), - ); - let root = if identity { - asap_types::ir::schema::schema_support::with_promql_series_identity(&root) - .unwrap() - } else { - root - }; - QueryRoot::Operator(root) - }) - .collect(); - let initial = generate_dag_of_logical_memo_groups(&roots); - let inventory = - generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&roots)) - .unwrap(); - try_populate_pass1(initial, &inventory).unwrap() - } - - // Equal-sized quantiles share one canonical state while local choices remain. - #[test] - fn quantile_readers_share_concrete_state_and_retain_local_candidates() { - let original = workload(&[ - ("quantile_over_time(0.95, latency[5m])", 0.01), - ("quantile_over_time(0.99, latency[5m])", 0.01), - ]); - let original_count = original.groups.len(); - let expanded = expand(original.clone()).unwrap(); - assert!(!expanded.state_groups.is_empty()); - assert!(expanded.added_reader_candidates >= 2); - let order = expanded.memo.topological_order().unwrap(); - for state in &expanded.state_groups { - let group = &expanded.memo.groups[state.0 as usize]; - assert_eq!(group.queries, vec![0, 1]); - for input in &group.child_groups { - assert_eq!(expanded.memo.groups[input.0 as usize].queries, vec![0, 1]); - assert!(group.logical_node.children().iter().any(|node| Rc::ptr_eq( - node, - &expanded.memo.groups[input.0 as usize].logical_node - ))); - } - assert_eq!(group.parents.len(), 2); - assert!(!asap_types::stages::root_groups(&expanded.memo.query_roots).contains(state)); - for &parent in &group.parents { - assert!( - order.iter().position(|id| id == state).unwrap() - < order.iter().position(|id| id == &parent).unwrap() - ); - let reader = expanded.memo.candidates[parent.0 as usize] - .iter() - .find(|candidate| candidate.input_groups.contains(state)) - .unwrap(); - assert!(OperatorNode::reachable(&reader.root_node) - .iter() - .any(|node| Rc::ptr_eq(node, &group.logical_node))); - assert_eq!(reader.input_groups, vec![*state]); - } - } - for group in 0..original_count { - for (before, after) in original.candidates[group] - .iter() - .zip(&expanded.memo.candidates[group]) - { - assert!(Rc::ptr_eq(&before.root_node, &after.root_node)); - assert_eq!(before.input_groups, after.input_groups); - } - } - assert!(matches!( - expand(expanded.memo), - Err(Pass2RuleError::AlreadyExpanded) - )); - } - - // A stricter concrete configuration can serve both readers; local sizes remain. - #[test] - fn distinct_sizes_gain_compatible_shared_configurations() { - let original = workload(&[ - ("quantile_over_time(0.95, latency[5m])", 0.01), - ("quantile_over_time(0.99, latency[5m])", 0.02), - ]); - let expanded = expand(original.clone()).unwrap(); - assert!(!expanded.state_groups.is_empty()); - for (group, candidates) in original.candidates.iter().enumerate() { - for candidate in candidates { - assert!(expanded.memo.candidates[group] - .iter() - .any(|after| Rc::ptr_eq(&candidate.root_node, &after.root_node))); - } - } - assert!(expanded - .state_groups - .iter() - .any(|id| expanded.memo.groups[id.0 as usize].queries == vec![0, 1])); - } - - // Resizing never makes different source predicates compatible. - #[test] - fn distinct_filters_do_not_share_concrete_builds() { - let original = workload(&[ - ("quantile_over_time(0.95, latency{job=\"a\"}[5m])", 0.01), - ("quantile_over_time(0.99, latency{job=\"b\"}[5m])", 0.02), - ]); - let count = original.groups.len(); - let expanded = expand(original).unwrap(); - assert!(expanded.state_groups.is_empty()); - assert_eq!(expanded.memo.groups.len(), count); - } - - fn with_local_tumbling_windows(mut memo: Memo, grids: &[u64]) -> Memo { - use crate::pass2::asap_aware_cse::{ - tumbling_window_candidates, window_composition_pass2_candidates, - }; - - let boundaries: HashMap<_, _> = memo - .groups - .iter() - .enumerate() - .map(|(id, group)| (Rc::as_ptr(&group.logical_node), MemoGroupId(id as u32))) - .collect(); - for group in 0..memo.groups.len() { - let grid = grids[memo.groups[group].queries[0]]; - for candidate in memo.candidates[group].clone() { - let roots = vec![QueryRoot::Operator(Rc::clone(&candidate.root_node))]; - let tumbling_windows = tumbling_window_candidates(&roots, &[Some(grid)]); - if tumbling_windows.len() != 1 { - continue; - } - let rewritten = window_composition_pass2_candidates(&roots, &tumbling_windows) - .unwrap() - .remove(0) - .roots - .unwrap(); - let QueryRoot::Operator(node) = &rewritten[0] else { - unreachable!() - }; - // Tumbling window rewriting recomposes guarantees; restore the original - // canonical scan boundary rather than making it a private input. - let input = tumbling_windows[0].build.children()[0].children()[0].clone(); - let chain = OperatorNode::reachable(node) - .into_iter() - .find(|node| { - matches!(node.asap(), Some(ASAPOp::SummaryTumblingWindowChain { .. })) - }) - .unwrap(); - let node = replace_node(node, chain.children()[0], &input); - let mut tumbling_window = candidate; - tumbling_window.root_node = Rc::clone(&node); - tumbling_window.input_groups = boundary_groups(&node, &boundaries); - tumbling_window.origin = OriginOfCandidateForMemoGroup::TumblingWindows; - memo.candidates[group].push(tumbling_window); - } - } - memo - } - - // Different label projections can consume one wider tumbling-window state through - // typed reader projections; no original scan needs to contain the union. - #[test] - fn disjoint_scan_labels_share_a_union_tumbling_window_state_with_typed_readers() { - let original = with_local_tumbling_windows( - workload(&[ - ("avg by (job) (quantile_over_time(0.95, latency[5m]))", 0.01), - ( - "avg by (region) (quantile_over_time(0.99, latency[5m]))", - 0.01, - ), - ]), - &[60_000, 60_000], - ); - assert!(!original.groups.iter().any(|group| matches!( - group.logical_node.non_asap(), - Some(NonASAPOp::Scan { .. }) - ) && group - .logical_node - .schema - .fields - .iter() - .any(|field| field.name == "job") - && group - .logical_node - .schema - .fields - .iter() - .any(|field| field.name == "region"))); - let local_nodes: Vec<_> = original - .candidates - .iter() - .map(|candidates| { - candidates - .iter() - .map(|candidate| Rc::clone(&candidate.root_node)) - .collect::>() - }) - .collect(); - let expanded = expand(original).unwrap(); - let shared = expanded - .state_groups - .iter() - .find(|id| { - let group = &expanded.memo.groups[id.0 as usize]; - matches!( - group.logical_node.asap(), - Some(ASAPOp::SummaryTumblingWindowChain { .. }) - ) && group.queries == vec![0, 1] - && ["job", "region"].iter().all(|name| { - group - .logical_node - .schema - .fields - .iter() - .any(|field| field.name == *name) - }) - }) - .expect("one state over union labels"); - let state = &expanded.memo.groups[shared.0 as usize]; - assert!(state.parents.len() >= 2); - for parent in &state.parents { - let reader = expanded.memo.candidates[parent.0 as usize] - .iter() - .find(|candidate| candidate.input_groups.contains(shared)) - .unwrap(); - assert_eq!( - reader.root_node.schema.fields, - expanded.memo.groups[parent.0 as usize] - .logical_node - .schema - .fields - ); - assert!(OperatorNode::reachable(&reader.root_node) - .iter() - .any(|node| matches!(node.non_asap(), Some(NonASAPOp::Project { .. })))); - reader.root_node.validate_structure().unwrap(); - } - for (group, locals) in local_nodes.into_iter().enumerate() { - assert!(locals - .iter() - .zip(&expanded.memo.candidates[group]) - .all(|(before, after)| Rc::ptr_eq(before, &after.root_node))); - } - assert!(expanded.memo.topological_order().is_some()); - assert!(state - .child_groups - .iter() - .all(|child| expanded.memo.groups[child.0 as usize].queries == vec![0, 1])); - } - - // Widening shifts identity's position but must retain its full typed field - // and remap the source's unique key rather than treating it as a plain label. - #[test] - fn widened_tumbling_window_inputs_preserve_series_identity_and_keys() { - let memo = with_local_tumbling_windows( - workload_with_identity( - &[ - ("avg by (job) (quantile_over_time(0.95, latency[5m]))", 0.01), - ( - "avg by (region) (quantile_over_time(0.99, latency[5m]))", - 0.01, - ), - ], - true, - ), - &[60_000, 60_000], - ); - let expanded = expand(memo).unwrap(); - let state = expanded - .state_groups - .iter() - .map(|id| &expanded.memo.groups[id.0 as usize]) - .find(|group| { - matches!( - group.logical_node.asap(), - Some(ASAPOp::SummaryTumblingWindowChain { .. }) - ) && group.queries == vec![0, 1] - && ["job", "region"].iter().all(|name| { - group - .logical_node - .schema - .fields - .iter() - .any(|field| field.name == *name) - }) - }) - .expect("identity-carrying union tumbling-window state"); - let input = state.logical_node.children()[0]; - assert!(input.schema.has_promql_series_identity()); - let identity = input - .schema - .column_id(asap_types::ir::schema::PROMQL_SERIES_IDENTITY) - .unwrap(); - let time = input.schema.time_index.unwrap(); - assert!(input - .schema - .unique_keys - .iter() - .any(|key| key.contains(&identity) && key.contains(&time))); - for parent in &state.parents { - let reader = expanded.memo.candidates[parent.0 as usize] - .iter() - .find(|candidate| { - candidate.input_groups.contains(&MemoGroupId( - expanded - .memo - .groups - .iter() - .position(|group| Rc::ptr_eq(&group.logical_node, &state.logical_node)) - .unwrap() as u32, - )) - }) - .unwrap(); - assert_eq!( - reader.root_node.schema, - expanded.memo.groups[parent.0 as usize].logical_node.schema - ); - } - } - - /// The groups `group`'s candidates read, directly or through the - /// candidates of the groups they read. - fn groups_read_by(memo: &Memo, group: usize) -> HashSet { - let mut read = HashSet::new(); - let mut pending = vec![group]; - while let Some(group) = pending.pop() { - for input in memo.candidates[group] - .iter() - .flat_map(|candidate| &candidate.input_groups) - { - if read.insert(input.0 as usize) { - pending.push(input.0 as usize); - } - } - } - read - } - - // A scan (or time range) that already has every label of its metric is - // not widened: widening recomposes guarantees, and an unwidened root - // that differs from the original only in its guarantee annotation must - // not become an identity `Project` over its own group. - #[test] - fn scan_label_widening_never_reads_its_own_group() { - for queries in [ - [ - ("sum by (job, region) (m)", 0.01), - ("sum by (job) (m)", 0.01), - ], - [ - ("sum by (job, region) (rate(m[5m]))", 0.01), - ("sum by (job) (rate(m[5m]))", 0.01), - ], - ] { - let mut memo = workload_with_identity(&queries, true); - crate::pass2::logical_memo_candidates::apply_sharing_rules(&mut memo).unwrap(); - assert!( - memo.candidates.iter().flatten().any( - |candidate| candidate.origin == OriginOfCandidateForMemoGroup::WidenedScans - ), - "{queries:?}: the narrower scan is widened" - ); - for group in 0..memo.groups.len() { - assert!( - !groups_read_by(&memo, group).contains(&group), - "{queries:?}: group {group} reads itself: {:?}", - memo.candidates[group] - .iter() - .map(|candidate| (&candidate.origin, &candidate.input_groups)) - .collect::>() - ); - } - let expanded = expand(workload_with_identity(&queries, true)).unwrap(); - for group in 0..expanded.memo.groups.len() { - assert!(!groups_read_by(&expanded.memo, group).contains(&group)); - } - } - } - - // A candidate reading its own group is a cycle the memo rejects. - #[test] - fn a_group_reading_itself_is_a_cyclic_memo() { - let mut memo = workload(&[("sum by (job) (m)", 0.01)]); - assert!(check_unexpanded_memo(&memo).is_ok()); - let mut reader = memo.candidates[0][0].clone(); - reader.input_groups = vec![MemoGroupId(0)]; - memo.candidates[0].push(reader); - assert_eq!(memo.topological_order(), None); - assert!(matches!( - check_unexpanded_memo(&memo), - Err(Pass2RuleError::CyclicDependencies) - )); - } - - // Label widening must not erase source predicates in order to create reuse. - #[test] - fn scan_label_widening_keeps_different_filters_separate() { - let original = with_local_tumbling_windows( - workload(&[ - ( - "avg by (job) (quantile_over_time(0.95, latency{env=\"a\"}[5m]))", - 0.01, - ), - ( - "avg by (region) (quantile_over_time(0.99, latency{env=\"b\"}[5m]))", - 0.01, - ), - ]), - &[60_000, 60_000], - ); - let expanded = expand(original).unwrap(); - assert!(expanded - .state_groups - .iter() - .all(|id| expanded.memo.groups[id.0 as usize].queries.len() == 1)); - } - - // Real tumbling window rewrites expose one identical chain to distinct quantile readers. - #[test] - fn identical_local_tumbling_window_chains_share_state_without_removing_local_choices() { - let original = with_local_tumbling_windows( - workload(&[ - ("quantile_over_time(0.95, latency[5m])", 0.01), - ("quantile_over_time(0.99, latency[5m])", 0.01), - ]), - &[60_000, 60_000], - ); - let local_tumbling_windows = original - .candidates - .iter() - .flatten() - .filter(|candidate| candidate.origin == OriginOfCandidateForMemoGroup::TumblingWindows) - .count(); - assert!(local_tumbling_windows >= 2); - let expanded = expand(original.clone()).unwrap(); - let chains: Vec<_> = expanded - .state_groups - .iter() - .filter(|id| { - matches!( - expanded.memo.groups[id.0 as usize].logical_node.asap(), - Some(ASAPOp::SummaryTumblingWindowChain { .. }) - ) - }) - .collect(); - assert!(!chains.is_empty()); - for id in chains { - let group = &expanded.memo.groups[id.0 as usize]; - assert_eq!(group.queries, vec![0, 1]); - assert_eq!(group.parents.len(), 2); - assert!(group.child_groups.iter().all(|input| group - .logical_node - .children() - .iter() - .any(|node| Rc::ptr_eq( - node, - &expanded.memo.groups[input.0 as usize].logical_node - )))); - } - for (group, candidates) in original.candidates.iter().enumerate() { - for original in candidates { - assert!(expanded.memo.candidates[group] - .iter() - .any(|candidate| Rc::ptr_eq(&original.root_node, &candidate.root_node))); - } - } - assert!(matches!( - expand(expanded.memo), - Err(Pass2RuleError::AlreadyExpanded) - )); - } - - // A finer grid and longer horizon serves a shorter reader through newest_tumbling_windows. - #[test] - fn different_tumbling_window_grids_and_horizons_gain_explicit_newest_tumbling_window_readers() { - let original = with_local_tumbling_windows( - workload(&[ - ("quantile_over_time(0.95, latency[5m])", 0.01), - ("quantile_over_time(0.99, latency[15m])", 0.02), - ]), - &[60_000, 30_000], - ); - let expanded = expand(original.clone()).unwrap(); - let state = expanded - .state_groups - .iter() - .find(|id| { - matches!( - expanded.memo.groups[id.0 as usize].logical_node.asap(), - Some(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms: 30_000, - tumbling_window_count: 30, - .. - }) - ) - }) - .expect("shared fine grid covering fifteen minutes"); - assert_eq!(expanded.memo.groups[state.0 as usize].queries, vec![0, 1]); - assert!(expanded.memo.candidates.iter().flatten().any(|reader| { - reader.input_groups.contains(state) - && OperatorNode::reachable(&reader.root_node) - .iter() - .any(|node| { - matches!( - node.asap(), - Some(ASAPOp::SummaryMerge { - newest_tumbling_windows: Some(10), - .. - }) - ) - }) - })); - for (group, candidates) in original.candidates.iter().enumerate() { - assert!(candidates - .iter() - .all(|before| expanded.memo.candidates[group] - .iter() - .any(|after| Rc::ptr_eq(&before.root_node, &after.root_node)))); - } - assert!(expanded.memo.topological_order().is_some()); - } - - // Rollup readers preserve original outputs while referencing the finer - // query's finalized values through an ordinary child-group dependency: - // the finer query's own result group, whose result is those values. - #[test] - fn exact_rollup_exposes_shared_finalized_values() { - let original = workload(&[ - ("sum by (job) (m)", 0.01), - ("sum by (job, instance) (m)", 0.01), - ]); - let expanded = expand(original).unwrap(); - let memo = &expanded.memo; - let fine = memo.query_roots[1].as_operator().unwrap().0 as usize; - assert!(memo.candidates[fine].iter().all(|candidate| matches!( - candidate.root_node.asap(), - Some(ASAPOp::FinalizeExactAccumulator { .. }) - ))); - assert!(memo - .candidates - .iter() - .enumerate() - .filter(|(group, _)| *group != fine) - .flat_map(|(_, candidates)| candidates) - .any( - |candidate| candidate.origin == OriginOfCandidateForMemoGroup::Rollup - && candidate.input_groups.contains(&MemoGroupId(fine as u32)) - )); - assert!(memo.topological_order().is_some()); - } - - // Sketch rollup exposes the resized finer state through an ordinary group - // dependency, while each output retains its own grouping and quantile. - #[test] - fn sketch_rollup_exposes_shared_finer_state() { - let original = workload(&[ - ("quantile(0.9, m)", 0.01), - ("quantile by (job) (0.9, m)", 0.02), - ]); - let expanded = expand(original).unwrap(); - assert!(expanded - .memo - .candidates - .iter() - .flatten() - .any( - |candidate| candidate.origin == OriginOfCandidateForMemoGroup::SharedBuild - && OperatorNode::reachable(&candidate.root_node) - .iter() - .any(|node| matches!( - node.asap(), - Some(ASAPOp::SummaryMerge { - group_by: Some(_), - .. - }) - )) - )); - assert!(expanded.memo.topological_order().is_some()); - } - - // Different k outputs share one sorted ranking, backed by a recursively - // exact candidate group rather than an unenforced exact-input annotation. - #[test] - fn topk_outputs_share_explicit_exact_ranking() { - let original = workload(&[ - ("topk by (job) (3, sum_over_time(m[1m]))", 0.01), - ("topk by (job) (5, sum_over_time(m[1m]))", 0.01), - ]); - let expanded = expand(original).unwrap(); - assert!(expanded.memo.groups.iter().any(|group| matches!( - group.logical_node.non_asap(), - Some(NonASAPOp::Limit { n: Some(5), .. }) - ) && group.parents.len() >= 2)); - assert!(expanded - .memo - .candidates - .iter() - .flatten() - .any(|candidate| candidate.origin == OriginOfCandidateForMemoGroup::TopKReuse)); - assert!(expanded.memo.topological_order().is_some()); - } - - // Widened source-group candidates make shared scans available even to - // ordinary operators that do not build an ASAP summary. - #[test] - fn general_source_groups_offer_typed_union_scan_readers() { - let original = workload(&[ - ("topk by (job) (3, m)", 0.01), - ("topk by (region) (5, m)", 0.01), - ]); - let expanded = expand(original.clone()).unwrap(); - let union = expanded - .memo - .groups - .iter() - .enumerate() - .find(|(_, group)| { - matches!(group.logical_node.non_asap(), Some(NonASAPOp::Scan { .. })) - && ["job", "region"].iter().all(|name| { - group - .logical_node - .schema - .fields - .iter() - .any(|field| field.name == *name) - }) - }) - .map(|(id, _)| MemoGroupId(id as u32)) - .expect("union scan"); - let readers = original - .groups - .iter() - .enumerate() - .filter(|(_, group)| { - matches!(group.logical_node.non_asap(), Some(NonASAPOp::Scan { .. })) - }) - .filter(|(group, _)| { - expanded.memo.candidates[*group].iter().any(|candidate| { - candidate.origin == OriginOfCandidateForMemoGroup::WidenedScans - && candidate.input_groups.contains(&union) - }) - }) - .count(); - assert!(readers >= 2); - } - - // A composite average can share only its sum state with another query while - // retaining the count state locally; reader budget allocation is preserved. - #[test] - fn multiple_private_states_can_share_only_the_common_producer() { - let mut original = workload(&[("avg(m)", 0.01), ("sum(m)", 0.01)]); - let average = original.query_roots[0].as_operator().unwrap().0 as usize; - assert!(original.candidates[average] - .iter() - .any(|candidate| direct_input_builds(&original, candidate).len() >= 2)); - for candidates in &mut original.candidates { - for candidate in candidates { - candidate.accuracy_budget_parts = 2; - } - } - let expanded = expand(original.clone()).unwrap(); - assert!(!expanded.state_groups.is_empty()); - let reader = expanded.memo.candidates[average] - .iter() - .find(|candidate| { - candidate.origin == OriginOfCandidateForMemoGroup::SharedBuild - && candidate - .input_groups - .iter() - .any(|group| expanded.state_groups.contains(group)) - }) - .expect("average has shared state input"); - assert_eq!(reader.accuracy_budget_parts, 2); - // The shared build is the readers' own, sized for two shares too. - for state in &expanded.state_groups { - assert_eq!( - expanded.memo.candidates[state.0 as usize][0].accuracy_budget_parts, - 2 - ); - } - assert!( - reader - .input_groups - .iter() - .any(|id| expanded.state_groups.contains(id) - && expanded.memo.groups[id.0 as usize].queries == vec![0, 1]), - "sum state is reusable by both queries" - ); - assert!( - reader - .input_groups - .iter() - .any(|id| expanded.state_groups.contains(id) - && expanded.memo.groups[id.0 as usize].queries == vec![0]), - "count state belongs only to the average query" - ); - assert!(original.candidates[average] - .iter() - .all(|before| expanded.memo.candidates[average] - .iter() - .any(|after| Rc::ptr_eq(&before.root_node, &after.root_node)))); - } - - // A global SQL-window quantile can consume newest tumbling windows of a longer, - // finer-grouped chain through the existing regrouping rule. - #[test] - fn grouped_window_rollup_exposes_one_finer_chain() { - use crate::test_support::{agg, scan_from, time_range}; - use asap_types::ir::operator::{AggIntent, Source}; - use asap_types::ir::schema::{DataType, Field, Schema}; - let scan = || { - scan_from( - Source::Table { - table_ref: "metrics".into(), - }, - Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("city", DataType::Utf8, true), - Field::plain("value", DataType::Float64, true), - ], - 0, - vec![], - ), - ) - }; - let quantile = |q| AggIntent::Quantile { - col: Some(2), - q, - accuracy: AccuracyTarget::max_error(0.01), - }; - let roots = vec![ - QueryRoot::Operator(agg( - vec![], - quantile(0.5), - time_range(std::time::Duration::from_secs(300), scan()), - )), - QueryRoot::Operator(agg( - vec![1], - quantile(0.9), - time_range(std::time::Duration::from_secs(1200), scan()), - )), - ]; - let initial = generate_dag_of_logical_memo_groups(&roots); - let generated = - generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&roots)) - .unwrap(); - let memo = with_local_tumbling_windows( - try_populate_pass1(initial, &generated).unwrap(), - &[10_000, 10_000], - ); - let expanded = expand(memo).unwrap(); - assert!(expanded - .memo - .candidates - .iter() - .flatten() - .any( - |candidate| candidate.origin == OriginOfCandidateForMemoGroup::SharedBuild - && OperatorNode::reachable(&candidate.root_node) - .iter() - .any(|node| matches!( - node.asap(), - Some(ASAPOp::SummaryMerge { - group_by: Some(_), - .. - }) - )) - && candidate.input_groups.iter().any(|group| matches!( - expanded.memo.groups[group.0 as usize].logical_node.asap(), - Some(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_count: 120, - .. - }) - )) - )); - } - - // Private identity projections retain their full typed contract, but no - // longer hide compatible builds from canonical state sharing. - #[test] - fn private_projection_inputs_gain_canonical_shared_states() { - check_private_inputs(false); - } - - // Key-only HAVING uses private Filter nodes below summary builds; equal - // filters can share without dropping their predicates. - #[test] - fn private_filter_inputs_gain_canonical_shared_states() { - check_private_inputs(true); - } - - fn check_private_inputs(filter: bool) { - use asap_types::ir::scalar::ScalarValue; - use asap_types::ir::{Operator, Predicate, ProjectItem, ScalarExpr}; - let mut memo = workload(&[ - ("quantile_over_time(0.95, latency[5m])", 0.01), - ("quantile_over_time(0.99, latency[5m])", 0.01), - ]); - let mut injected = 0; - for owner in 0..memo.candidates.len() { - let originals = memo.candidates[owner].clone(); - for mut candidate in originals { - let builds = direct_input_builds(&memo, &candidate); - if builds.is_empty() { - continue; - } - let mut replacements = HashMap::new(); - for (build, _) in builds { - let input = build.children()[0]; - let project = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Project { - cols: input - .schema - .fields - .iter() - .enumerate() - .map(|(column, field)| ProjectItem { - alias: Some(field.name.clone()), - expr: ScalarExpr::Column(column), - }) - .collect(), - qualifier: None, - child: Rc::clone(input), - })) - .unwrap(); - let project = if filter { - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Filter { - pred: Predicate(ScalarExpr::Literal(ScalarValue::Boolean(true))), - child: project, - })) - .unwrap() - } else { - project - }; - replacements.insert(Rc::as_ptr(input), project); - } - candidate.root_node = - crate::pass2::memo::rules::reuse::replace(&candidate.root_node, &replacements); - assert!(direct_input_builds(&memo, &candidate).is_empty()); - memo.candidates[owner].push(candidate); - injected += 1; - } - } - assert!(injected >= 2); - let expanded = expand(memo).unwrap(); - assert!(expanded.state_groups.iter().any(|state| { - let group = &expanded.memo.groups[state.0 as usize]; - group.queries == vec![0, 1] - && group.child_groups.iter().any(|input| { - expanded.memo.candidates[input.0 as usize] - .iter() - .any(|candidate| { - candidate.origin == OriginOfCandidateForMemoGroup::SummaryInput - }) - }) - })); - assert!(expanded.memo.topological_order().is_some()); - } - - // Real identity-aware frequency candidates retain UnivMon inputs and their - // error metadata when private preprocessing is made an explicit boundary. - #[test] - fn univmon_identity_inputs_remain_admissible_after_factoring() { - use asap_types::ir::schema::{sketch::SketchAlgorithm, FieldDataType}; - let mut memo = workload_with_identity(&[("count_over_time(latency[5m])", 0.01)], true); - let is_univmon = |node: &OperatorNode| matches!(node.asap(), Some(ASAPOp::SummaryAgg {family: FieldDataType::Sketch(kind, _), ..}) if kind.algorithm() == &SketchAlgorithm::UnivMon); - let before: Vec<_> = memo - .candidates - .iter() - .flatten() - .flat_map(|candidate| OperatorNode::reachable(&candidate.root_node)) - .filter(|node| is_univmon(node)) - .collect(); - assert!(!before.is_empty(), "real generation must offer UnivMon"); - crate::pass2::memo::rules::summary_inputs::factor_summary_inputs(&mut memo).unwrap(); - assert!(memo - .candidates - .iter() - .flatten() - .any(|candidate| direct_input_builds(&memo, candidate) - .iter() - .any(|(node, _)| is_univmon(node)))); - for original in before { - assert!( - memo.candidates - .iter() - .flatten() - .flat_map(|candidate| OperatorNode::reachable(&candidate.root_node)) - .any(|node| Rc::ptr_eq(&original, &node)), - "local generated candidates remain intact" - ); - } - } - - // Reader deduplication must retain distinct recorded probability contracts - // even though SketchParams structural equality intentionally ignores them. - #[test] - fn same_capacity_readers_retain_distinct_failure_probability_contracts() { - use asap_types::ir::schema::{ - sketch::{SketchAlgorithm, SketchKind, SketchParams}, - FieldDataType, - }; - let mut memo = workload(&[("quantile_over_time(0.95, latency[5m])", 0.01)]); - let (owner, mut base, build) = memo.candidates.iter().enumerate().find_map(|(owner, candidates)| candidates.iter().find_map(|candidate| { - let (build, _) = simple_build(&memo, candidate)?; - matches!(build.asap(), Some(ASAPOp::SummaryAgg {family: FieldDataType::Sketch(kind, _), ..}) if kind.algorithm() == &SketchAlgorithm::Kll).then(|| (owner, candidate.clone(), build)) - })).unwrap(); - let Some(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, grouping), - .. - }) = build.asap() - else { - unreachable!() - }; - let SketchParams::Kll { k, .. } = kind.params() else { - unreachable!() - }; - let boundaries: HashSet<_> = base - .input_groups - .iter() - .map(|id| Rc::as_ptr(&memo.groups[id.0 as usize].logical_node)) - .collect(); - fn without_private_guarantees( - node: &Rc, - boundaries: &HashSet<*const OperatorNode>, - ) -> Rc { - if boundaries.contains(&Rc::as_ptr(node)) { - return Rc::clone(node); - } - Rc::new(OperatorNode { - operator: node - .operator - .map_children(|child| without_private_guarantees(child, boundaries)), - guarantee: None, - ..(**node).clone() - }) - } - let rewritten = |probability| { - let family = FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: *k, - failure_probability: Some(probability), - }, - ), - grouping.clone(), - ); - let roots = crate::pass2::asap_aware_cse::with_resized_build( - &[QueryRoot::Operator(Rc::clone(&base.root_node))], - &build, - family, - ) - .unwrap(); - let QueryRoot::Operator(root) = &roots[0] else { - unreachable!() - }; - let root = - crate::pass2::memo::rules::reuse::restore_inputs(root, &memo, &base.input_groups); - without_private_guarantees(&root, &boundaries) - }; - let weak = rewritten(0.001); - let strict = rewritten(0.0001); - assert_eq!( - *weak, *strict, - "ordinary equality omits the probability metadata" - ); - base.root_node = weak; - memo.candidates[owner] = vec![base.clone()]; - assert!(crate::pass2::memo::rules::reuse::append_reader( - &mut memo, - owner, - &base, - strict, - OriginOfCandidateForMemoGroup::Rollup - ) - .unwrap()); - assert_eq!(memo.candidates[owner].len(), 2); - } - - // Equal private producers can be shared within one composite result; they - // do not need to originate from different memo owner groups. - #[test] - fn repeated_states_within_one_owner_share_a_producer() { - let mut memo = workload(&[("quantile_over_time(0.95, latency[5m])", 0.01)]); - let (owner, mut candidate) = memo - .candidates - .iter() - .enumerate() - .find_map(|(group, candidates)| { - candidates - .iter() - .find(|candidate| simple_build(&memo, candidate).is_some()) - .map(|candidate| (group, candidate.clone())) - }) - .unwrap(); - let boundaries: HashSet<_> = candidate - .input_groups - .iter() - .map(|id| Rc::as_ptr(&memo.groups[id.0 as usize].logical_node)) - .collect(); - fn duplicate( - node: &Rc, - boundaries: &HashSet<*const OperatorNode>, - ) -> Rc { - if boundaries.contains(&Rc::as_ptr(node)) { - return Rc::clone(node); - } - Rc::new(OperatorNode { - operator: node - .operator - .map_children(|child| duplicate(child, boundaries)), - ..(**node).clone() - }) - } - let second = duplicate(&candidate.root_node, &boundaries); - let combined = - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Concat { - children: vec![Rc::clone(&candidate.root_node), second], - })) - .unwrap(); - candidate.root_node = Rc::clone(&combined); - memo.groups[owner].logical_node = combined; - memo.candidates[owner] = vec![candidate]; - let expanded = expand(memo).unwrap(); - assert_eq!(expanded.state_groups.len(), 1); - let state = expanded.state_groups[0]; - assert!(expanded.memo.groups[state.0 as usize] - .parents - .contains(&MemoGroupId(owner as u32))); - assert!(expanded.memo.candidates[owner] - .iter() - .any( - |candidate| candidate.origin == OriginOfCandidateForMemoGroup::SharedBuild - && candidate.input_groups.contains(&state) - )); - } - - // Public expansion rejects malformed IDs before indexing dependency arrays. - #[test] - fn malformed_references_fail_before_dependency_traversal() { - let mut memo = workload(&[("quantile_over_time(0.95, latency[5m])", 0.01)]); - memo.groups[0].child_groups.push(MemoGroupId(u32::MAX)); - assert!(matches!( - expand(memo), - Err(Pass2RuleError::InvalidGroupReference) - )); - } -} diff --git a/crates/logical-asap-optimization/src/pass2/memo/window_candidates.rs b/crates/logical-asap-optimization/src/pass2/memo/window_candidates.rs index 468de54d5..62594119f 100644 --- a/crates/logical-asap-optimization/src/pass2/memo/window_candidates.rs +++ b/crates/logical-asap-optimization/src/pass2/memo/window_candidates.rs @@ -1,315 +1,18 @@ -//! Local tumbling-window candidates generated from compatible reader grids. Selected -//! connections, not the union of potential consumers, determine eligibility. -#[cfg(test)] -use super::shared_configurations::tumbling_window_width_divisors; -use super::shared_configurations::tumbling_window_width_domain; -use crate::pass2::asap_aware_cse::{ - tumbling_window_candidates, window_composition_pass2_candidates, -}; -use asap_types::ir::flat::flatten; -use asap_types::ir::{OperatorNode, QueryRoot}; -use asap_types::stages::{CandidateForMemoGroup, Memo, OriginOfCandidateForMemoGroup}; -use asap_types::stages::{MemoGroup, MemoGroupId as GroupId}; -use asap_types::workload::QueryWorkloadEntry; -#[cfg(test)] -use std::collections::BTreeSet; -use std::collections::{HashMap, HashSet}; -use std::rc::Rc; +//! Rule 1, local window composition: tumbling-window layouts $\omega_v$ as +//! candidates of a summary state group (#1160). -/// Offer each supported reader-grid configuration, for every legal -/// tumbling-window width. Stage 2 checks the actual chosen consumers, -/// including ingestion alignment and range evaluation steps. -pub fn add_local_window_composition(memo: &mut Memo, entries: &[QueryWorkloadEntry]) { - // Flatten each existing/generated graph once. Repeatedly flattening the - // growing candidate list makes a complete tumbling-window-width domain quadratic. - let mut known: HashMap<(usize, String), Vec> = HashMap::new(); - for (group, candidates) in memo.candidates.iter().enumerate() { - for (index, candidate) in candidates.iter().enumerate() { - if let Ok(signature) = candidate_signature(candidate) { - known.entry((group, signature)).or_default().push(index); - } - } - } - for (group, candidate) in local_tumbling_window_candidates(memo, entries) { - let Ok(signature) = candidate_signature(&candidate) else { - memo.candidates[group].push(candidate); - continue; - }; - let bucket = known.entry((group, signature)).or_default(); - if !bucket.iter().any(|&index| { - let previous = &memo.candidates[group][index]; - previous.origin == candidate.origin - && previous.input_groups == candidate.input_groups - && previous.fused_groups == candidate.fused_groups - && previous.accuracy_budget_parts == candidate.accuracy_budget_parts - && previous.root_node == candidate.root_node - }) { - bucket.push(memo.candidates[group].len()); - memo.candidates[group].push(candidate); - } - } -} +use crate::pass2::memo::rules::Pass2RuleError; +use asap_types::stages::Memo; +use asap_types::workload::QueryWorkloadEntry; -/// Expand legal tumbling-window widths while preserving canonical memo inputs. -fn local_tumbling_window_candidates( - memo: &Memo, +/// Rule 1: for each summary state group, a candidate per tumbling width $w$ +/// from the gcd closure of the ranges and steps of the queries that reach it, +/// offered only when $w$ divides the group's query range. `entries[i]` is +/// query `i`'s workload entry. +pub(crate) fn add_local_window_composition( + memo: &mut Memo, entries: &[QueryWorkloadEntry], -) -> Vec<(usize, CandidateForMemoGroup)> { - let group_of = group_of_node(&memo.groups); - let mut own: Vec<(usize, CandidateForMemoGroup)> = Vec::new(); - for (g, group) in memo.groups.iter().enumerate() { - let grids = tumbling_window_width_domain( - group - .queries - .iter() - .filter_map(|&query| entries.get(query)?.evaluation_grid_ms()), - ); - for grid in grids { - for candidate in &memo.candidates[g] { - if !matches!( - candidate.origin, - OriginOfCandidateForMemoGroup::Pass1 - | OriginOfCandidateForMemoGroup::SharedBuild - ) { - continue; - } - let canonical_scans = OperatorNode::reachable(&candidate.root_node) - .into_iter() - .filter(|node| { - matches!( - node.non_asap(), - Some(asap_types::ir::NonASAPOp::Scan { .. }) - ) && group_of.contains_key(&Rc::as_ptr(node)) - }) - .collect::>(); - let roots = [QueryRoot::Operator(Rc::clone(&candidate.root_node))]; - let tumbling_windows = tumbling_window_candidates(&roots, &[Some(grid)]); - if tumbling_windows.is_empty() { - continue; - } - let Ok(pass2_candidates) = - window_composition_pass2_candidates(&roots, &tumbling_windows) - else { - continue; - }; - for pass2_candidate in pass2_candidates { - let Ok(rewritten) = pass2_candidate.roots else { - continue; - }; - let [QueryRoot::Operator(node)] = &rewritten[..] else { - continue; - }; - // Guarantee recomposition can clone unchanged source scans. - // Restore only an unambiguous original scan with the same full - // source/schema/execution contract; scans have intrinsic exact - // semantics. Otherwise the tumbling window's canonical input disappears. - let mut restored = HashMap::new(); - for rewritten in OperatorNode::reachable(node) { - let mut matches = canonical_scans.iter().filter(|original| { - original.operator == rewritten.operator - && original.schema == rewritten.schema - && original.result_kind == rewritten.result_kind - && original.timing == rewritten.timing - && original.materialization == rewritten.materialization - }); - if let Some(original) = matches.next() { - if matches.next().is_none() { - restored.insert(Rc::as_ptr(&rewritten), original); - } - } - } - let node = restore_scan_boundaries(node, &restored, &mut HashMap::new()); - own.push(( - g, - rewritten_candidate( - &group_of, - GroupId(g as u32), - candidate, - node, - OriginOfCandidateForMemoGroup::TumblingWindows, - ), - )); - } - } - } - } - own -} - -fn restore_scan_boundaries( - node: &Rc, - restored: &HashMap<*const OperatorNode, &Rc>, - done: &mut HashMap<*const OperatorNode, Rc>, -) -> Rc { - if let Some(scan) = restored.get(&Rc::as_ptr(node)) { - return Rc::clone(scan); - } - if let Some(previous) = done.get(&Rc::as_ptr(node)) { - return Rc::clone(previous); - } - let mut changed = false; - let operator = node.operator.map_children(|child| { - let rebuilt = restore_scan_boundaries(child, restored, done); - changed |= !Rc::ptr_eq(child, &rebuilt); - rebuilt - }); - let rebuilt = if changed { - Rc::new( - OperatorNode::with_schema(operator, node.schema.clone()) - .with_guarantee(node.guarantee.clone()) - .with_timing(node.timing) - .with_materialization(node.materialization), - ) - } else { - Rc::clone(node) - }; - done.insert(Rc::as_ptr(node), Rc::clone(&rebuilt)); - rebuilt -} - -// Union of complete divisor domains covers individual readers and every -// compatible subset: common widths belong to each reader's divisor domain. -// Nongridded potential consumers do not veto a repeated reader's tumbling windows. -#[cfg(test)] -fn compatible_reader_grids(group: &MemoGroup, entries: &[QueryWorkloadEntry]) -> Vec { - super::shared_configurations::tumbling_window_width_domain( - group - .queries - .iter() - .filter_map(|&query| entries.get(query)?.evaluation_grid_ms()), - ) -} - -fn group_of_node(groups: &[MemoGroup]) -> HashMap<*const OperatorNode, usize> { - groups - .iter() - .enumerate() - .map(|(g, group)| (Rc::as_ptr(&group.logical_node), g)) - .collect() -} - -/// `from`, a candidate of group `owner`, with root `node` and `origin`. -fn rewritten_candidate( - group_of: &HashMap<*const OperatorNode, usize>, - owner: GroupId, - from: &CandidateForMemoGroup, - node: Rc, - origin: OriginOfCandidateForMemoGroup, -) -> CandidateForMemoGroup { - let implements: Vec = std::iter::once(owner) - .chain(from.fused_groups.iter().copied()) - .collect(); - let reached = |node: &Rc| child_groups(group_of, node, &implements); - let children = if reached(&from.root_node) == from.input_groups { - reached(&node) - } else { - from.input_groups.clone() - }; - CandidateForMemoGroup { - fused_groups: from.fused_groups.clone(), - root_node: node, - input_groups: children, - origin, - accuracy_budget_parts: from.accuracy_budget_parts, - } -} - -fn child_groups( - group_of: &HashMap<*const OperatorNode, usize>, - node: &Rc, - implements: &[GroupId], -) -> Vec { - fn walk( - node: &Rc, - group_of: &HashMap<*const OperatorNode, usize>, - implements: &[GroupId], - seen: &mut HashSet<*const OperatorNode>, - out: &mut Vec, - ) { - for child in node.children() { - if !seen.insert(Rc::as_ptr(child)) { - continue; - } - match group_of.get(&Rc::as_ptr(child)) { - Some(&g) if !implements.contains(&GroupId(g as u32)) => { - if !out.contains(&GroupId(g as u32)) { - out.push(GroupId(g as u32)); - } - } - _ => walk(child, group_of, implements, seen, out), - } - } - } - let mut out = Vec::new(); - walk(node, group_of, implements, &mut HashSet::new(), &mut out); - out -} - -fn candidate_signature(candidate: &CandidateForMemoGroup) -> Result { - serde_json::to_string(&flatten(&[QueryRoot::Operator(Rc::clone(&candidate.root_node))]).0) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::pass1::memo_population::try_populate_pass1; - use crate::pass1::per_computation_candidates::generate_per_computation_candidates; - use crate::stage0::generate_dag_of_logical_memo_groups; - use crate::test_support::{entry_with, lower_promql}; - use asap_types::ir::properties::AccuracyTarget; - use asap_types::workload::{QueryRecurrence, RepeatedDemand, RepetitionInterval}; - - fn repeated(interval: u32) -> QueryRecurrence { - QueryRecurrence::Repeated(RepeatedDemand::FixedInterval(RepetitionInterval(interval))) - } - - fn fixture(recurrences: &[QueryRecurrence]) -> (Memo, Vec) { - let target = AccuracyTarget::max_error(0.05).failure_probability(0.1); - let root = lower_promql("quantile_over_time(0.95, latency[15m])", target.clone()); - let roots: Vec<_> = recurrences - .iter() - .map(|_| QueryRoot::Operator(Rc::clone(&root))) - .collect(); - let initial = generate_dag_of_logical_memo_groups(&roots); - let generated = - generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&roots)) - .unwrap(); - let memo = try_populate_pass1(initial, &generated).unwrap(); - let entries = recurrences - .iter() - .map(|recurrence| entry_with(target.clone(), recurrence.clone())) - .collect(); - (memo, entries) - } - - // A nongridded potential reader cannot eliminate another reader's tumbling windows. - #[test] - fn nongridded_potential_reader_preserves_local_tumbling_window_candidates() { - let (memo, entries) = fixture(&[repeated(60_000), QueryRecurrence::Unknown]); - let candidates = local_tumbling_window_candidates(&memo, &entries); - assert!(!candidates.is_empty()); - assert!(candidates.iter().all( - |(_, candidate)| candidate.origin == OriginOfCandidateForMemoGroup::TumblingWindows - )); - } - - // Individual grids and subset GCDs allow separate or shared physical choices. - #[test] - fn reader_grids_include_subsets_instead_of_only_full_union() { - let (memo, entries) = fixture(&[ - repeated(120_000), - repeated(180_000), - QueryRecurrence::Unknown, - ]); - let group = &memo.groups[memo.query_roots[0].as_operator().unwrap().0 as usize]; - assert_eq!( - compatible_reader_grids(group, &entries), - tumbling_window_width_divisors(120_000) - .into_iter() - .chain(tumbling_window_width_divisors(180_000)) - .collect::>() - .into_iter() - .collect::>() - ); - } +) -> Result<(), Pass2RuleError> { + let _ = (memo, entries); + todo!("#1207 PR 4: rule 1, tumbling layouts as candidates of the state group") } diff --git a/crates/logical-asap-optimization/src/stage0.rs b/crates/logical-asap-optimization/src/stage0.rs index c26a9dc2e..5764cfeaf 100644 --- a/crates/logical-asap-optimization/src/stage0.rs +++ b/crates/logical-asap-optimization/src/stage0.rs @@ -1,228 +1,16 @@ -//! Stage 0: the memo groups of the workload's query roots -//! ([`DagOfLogicalMemoGroups`]), by hash-consing identical logical -//! sub-expressions (the `cse` module). Stage 1 starts from these groups. +//! Stage 0: the memo groups of the workload's query roots (a [`Memo`] +//! without candidates). Each node of the queries' DAGs becomes the group of +//! its expression, found or added by [`Memo::find_or_add_group`], so +//! identical sub-expressions are one group. Stage 1 starts from these groups. -/// Structural common-subexpression elimination (hash-consing) over the -/// logical DAG. -pub(crate) mod cse; - -use std::collections::{HashMap, HashSet}; -use std::rc::Rc; - -use crate::stage0::cse::share_common_sub_dags; -use asap_types::ir::{OperatorNode, QueryRoot}; -use asap_types::stages::{DagOfLogicalMemoGroups, MemoGroup, MemoGroupId}; +use asap_types::ir::{QueryRoot, SchemaDerivationError}; +use asap_types::stages::Memo; /// Stage 0: the memo groups of the workload's query roots, query `i` -/// being `roots[i]`, by hash-consing identical logical sub-expressions -/// (`share_common_sub_dags`). -pub fn generate_dag_of_logical_memo_groups(roots: &[QueryRoot]) -> DagOfLogicalMemoGroups { - let (groups, group_of) = hash_consed(roots); - let group = |node: &Rc| group_of[&Rc::as_ptr(node)]; - let query_roots = roots - .iter() - .map(|root| match root { - QueryRoot::Operator(node) => QueryRoot::Operator(group(node)), - QueryRoot::Scalar(expr) => { - QueryRoot::Scalar(expr.map_operator_refs(&mut |node| group(node))) - } - }) - .collect(); - DagOfLogicalMemoGroups::new(groups, query_roots, group_of) -} - -/// Each original node (of `roots`), by address, to its group. -type GroupOf = HashMap<*const OperatorNode, MemoGroupId>; - -/// The groups of `roots`, children before parents, and each original node's -/// group. A scalar query reaches the groups of the operator nodes its -/// expression reads. -fn hash_consed(roots: &[QueryRoot]) -> (Vec, GroupOf) { - let originals: Vec<(usize, Rc)> = roots - .iter() - .enumerate() - .flat_map(|(query, root)| { - operator_roots(root) - .into_iter() - .map(move |node| (query, node)) - }) - .collect(); - let interned = share_common_sub_dags(originals.clone()); - // Number the interned nodes children first. - let mut groups: Vec = Vec::new(); - let mut id_of: HashMap<*const OperatorNode, MemoGroupId> = HashMap::new(); - for (_, root) in &interned { - number(root, &mut groups, &mut id_of); - } - for (query, root) in &interned { - let g = id_of[&Rc::as_ptr(root)]; - let mut reached = HashSet::new(); - reach(&groups, g, &mut reached); - for reached in reached { - let queries = &mut groups[reached.0 as usize].queries; - if !queries.contains(query) { - queries.push(*query); - } - } - } - for group in &mut groups { - group.queries.sort_unstable(); - } - // Interning rebuilds every node over its children in order, so the two - // DAGs are walked side by side. - let mut group_of = GroupOf::new(); - for ((_, original), (_, root)) in originals.iter().zip(&interned) { - pair(original, root, &id_of, &mut group_of); - } - (groups, group_of) -} - -/// The operator nodes a query root reads. -fn operator_roots(root: &QueryRoot) -> Vec> { - root.operator_refs().into_iter().cloned().collect() -} - -/// Number `node` (interned) and its descendants, children first. -fn number( - node: &Rc, - groups: &mut Vec, - id_of: &mut HashMap<*const OperatorNode, MemoGroupId>, -) -> MemoGroupId { - if let Some(&id) = id_of.get(&Rc::as_ptr(node)) { - return id; - } - let children: Vec = node - .children() - .into_iter() - .map(|child| number(child, groups, id_of)) - .collect(); - let id = MemoGroupId(groups.len() as u32); - for &child in &children { - let parents = &mut groups[child.0 as usize].parents; - if !parents.contains(&id) { - parents.push(id); - } - } - groups.push(MemoGroup { - logical_node: Rc::clone(node), - child_groups: children, - parents: Vec::new(), - queries: Vec::new(), - }); - id_of.insert(Rc::as_ptr(node), id); - id -} - -/// `g` and every group below it. -fn reach(groups: &[MemoGroup], g: MemoGroupId, reached: &mut HashSet) { - if reached.insert(g) { - for &child in &groups[g.0 as usize].child_groups { - reach(groups, child, reached); - } - } -} - -/// Map `original` and its descendants to the groups of `interned`, its -/// interned copy. -fn pair( - original: &Rc, - interned: &Rc, - id_of: &HashMap<*const OperatorNode, MemoGroupId>, - group_of: &mut GroupOf, -) { - if group_of - .insert(Rc::as_ptr(original), id_of[&Rc::as_ptr(interned)]) - .is_some() - { - return; - } - for (child, interned_child) in original.children().into_iter().zip(interned.children()) { - pair(child, interned_child, id_of, group_of); - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::test_support::lower_promql; - use asap_types::ir::operator::operator_properties::BinaryOpKind; - use asap_types::ir::properties::AccuracyTarget; - use asap_types::ir::scalar::expr_ir::ArithmeticOpKind; - use asap_types::ir::{NonASAPOp, ScalarExpr}; - use asap_types::stages::queries_rooted_at; - - fn query(text: &str) -> QueryRoot { - QueryRoot::Operator(lower_promql(text, AccuracyTarget::max_error(0.01))) - } - - /// Logical interning shares equivalent results without needing Pass 1 choices. - #[test] - fn repeated_results_share_groups_but_quantile_levels_do_not() { - let roots = vec![ - query("sum by (job) (m)"), - query("sum by (job) (m)"), - query("quantile_over_time(0.95, m[5m])"), - query("quantile_over_time(0.99, m[5m])"), - ]; - let memo = generate_dag_of_logical_memo_groups(&roots); - let root_group = |q: usize| *memo.query_roots[q].as_operator().unwrap(); - assert_eq!(root_group(0), root_group(1)); - assert_ne!(root_group(2), root_group(3)); - for (index, group) in memo.groups.iter().enumerate() { - assert!(group - .child_groups - .iter() - .all(|child| (child.0 as usize) < index)); - for child in &group.child_groups { - assert!(memo.groups[child.0 as usize] - .parents - .contains(&MemoGroupId(index as u32))); - } - } - } - - /// A per-series quantile has no provable unique key, but it is - /// deterministic: both operands of `G2 / G2` read the same rows, so the - /// two equal sub-expressions are one group read twice. - #[test] - fn equal_keyless_deterministic_results_share_one_group() { - let roots = vec![QueryRoot::Operator(lower_promql( - "avg by (job) (quantile_over_time(0.5, data[5m]) / quantile_over_time(0.5, data[5m]) + quantile_over_time(0.9, data[5m]))", - AccuracyTarget::max_error(0.01).failure_probability(0.001), - ))]; - let memo = generate_dag_of_logical_memo_groups(&roots); - let division = memo - .groups - .iter() - .find(|group| { - matches!(group.logical_node.non_asap(), Some(NonASAPOp::BinaryOp { operator, .. }) - if operator.kind == BinaryOpKind::Arithmetic(ArithmeticOpKind::Div)) - }) - .expect("a `G2 / G3` group"); - assert_eq!( - division.child_groups[0], division.child_groups[1], - "both operands of the division are one group" - ); - assert_eq!(memo.groups.len(), 7); - } - - /// Scalar references are roots of their query; literals need no invented node. - #[test] - fn scalar_and_literal_roots_keep_workload_positions() { - let QueryRoot::Operator(node) = query("sum by (job) (m)") else { - unreachable!() - }; - let roots = vec![ - QueryRoot::Scalar(ScalarExpr::literal_f64(1.0)), - QueryRoot::Scalar(ScalarExpr::PromqlScalarFromVector(node.clone())), - QueryRoot::Operator(node.clone()), - ]; - let memo = generate_dag_of_logical_memo_groups(&roots); - let id = memo.group_for_node(&node).unwrap(); - assert_eq!(queries_rooted_at(&memo.query_roots, id), vec![1, 2]); - assert_eq!(memo.groups[id.0 as usize].reached_by_queries(), [1, 2]); - assert!(generate_dag_of_logical_memo_groups(&roots[..1]) - .groups - .is_empty()); - } +/// being `roots[i]`, every group without candidates. Fails with the first +/// node that breaks the node rules over its children's groups +/// ([`Memo::find_or_add_group`]). +pub fn memo_of_queries(roots: &[QueryRoot]) -> Result { + let _ = roots; + todo!("#1207 PR 2: Stage 0 over Memo::find_or_add_group") } diff --git a/crates/logical-asap-optimization/src/stage0/cse.rs b/crates/logical-asap-optimization/src/stage0/cse.rs deleted file mode 100644 index 6f209a4e6..000000000 --- a/crates/logical-asap-optimization/src/stage0/cse.rs +++ /dev/null @@ -1,895 +0,0 @@ -//! Structural common-subexpression elimination over the unified operator IR: -//! bottom-up hash-consing of [`OperatorNode`] DAGs across a workload's roots. -//! -//! CSE only runs on already-bound, already-canonicalized plans — structural -//! matching is meaningless before canonicalization has converged -//! semantically-equivalent queries onto one shape. [`share_common_sub_dags`] -//! is the single entry point, run once per workload batch (a batch of one -//! still deduplicates a query's own repeated sub-DAGs, see below). -//! -//! ## Algorithm: classic hash-consing / value-numbering -//! -//! Bottom-up: every child is interned before its parent, so two parents whose -//! children were independently deduplicated down to the same `Rc`s are -//! structurally identical iff their own fields also match, without re-walking -//! the sub-DAGs. "Child" means everything [`OperatorNode::children`] returns: -//! the operator inputs *and* the operator nodes a scalar expression reads -//! (`PromqlScalarFromVector`, `ScalarSubquery`, `Exists`, `InSubquery`), so a -//! vector read by `scalar(v)` in two queries is shared like any other input. -//! The scalar expressions themselves stay opaque data on their owning node. -//! -//! ## Correctness: hash is a filter, `PartialEq` is the decision -//! -//! This is the one non-negotiable rule. A **false positive** here — two -//! sub-DAGs wrongly judged shareable — is a wrong query answer, not a missed -//! optimization: two different queries would read each other's data. -//! [`structural_hash`] (SipHash over a canonical serialization, no -//! collision-freedom guarantee) may only narrow the candidate set within one -//! bucket; the typed equality check on that bucket ([`same_node`]) is what -//! actually decides sharing, every time, no exceptions for "the hash probably -//! didn't collide." Equality is intentionally conservative: it recognizes -//! *exact* structural matches only, never "a stricter-accuracy summary could -//! also answer a looser request" (that subsumption question belongs to the -//! ASAP matcher, not here). -//! -//! ## Legality -//! -//! Structural equality is necessary but not sufficient. A non-ASAP node is -//! only ever *returned* as a match for another when every execution over the -//! same input rows returns the same output rows -//! ([`NonASAPOp::is_shareable_when_equal`]). Equal nodes are -//! interned children first, so two equal nodes read one input, and one copy -//! returns each reader the rows it would have computed alone. A unique key -//! is not needed: equal per-series quantiles without a provable key are one -//! node. An operator that picks rows by input order alone (an unordered -//! `LIMIT`) is always inserted fresh, because two copies may return -//! different rows. -//! An ASAP node (summary state and its evaluations) has no such gate: equal -//! operator, schema and guarantee make it shareable, exactly as post-ASAP -//! sharing decided before this IR. -//! -//! ## Single-query CSE falls out for free -//! -//! A repeated sub-expression within *one* query (the same grouped aggregate on -//! both `BinaryOp` branches) is deduplicated by the same bottom-up interning — -//! a workload of size one still interns bottom-up within that one DAG. - -use std::collections::hash_map::DefaultHasher; -use std::collections::HashMap; -use std::hash::{Hash, Hasher}; -use std::rc::Rc; - -use asap_types::ir::operator::node::{Operator, OperatorNode}; -use asap_types::ir::NonASAPOp; - -/// [`structural_hash`]'s memoization cache: an already-hashed node's `Rc` -/// pointer to its hash. A fresh cache is always correct; what matters is -/// letting it persist across every node of one bottom-up pass rather than -/// starting a new one per call. The caller must keep every cached node alive -/// for the cache's lifetime, or a reused address would alias a stale entry. -pub(crate) type NodeToStructuralHashCache = HashMap<*const OperatorNode, u64>; - -/// The operator with every child (operator inputs and the operator nodes -/// referenced from its scalar expressions alike) replaced by `()`. What -/// remains is the node's own data: variant tag, scalar expressions, -/// parameters. -fn own_fields(node: &OperatorNode) -> Operator<()> { - node.operator.map_children(|_| ()) -} - -/// Coarse structural hash used only to bucket [`InternTable::intern`]'s -/// candidate search — never the sharing decision ([`same_node`] is). -/// -/// `OperatorNode` carries `f64`s (`ScalarValue::Float64`, quantile targets, -/// `AccuracyGuarantee` bounds, …), so it cannot derive `std::hash::Hash`. The -/// hash is SipHash over two parts: -/// -/// 1. the canonical JSON of [`own_fields`] plus `result_kind`, `schema`, -/// `guarantee`, `timing` and `materialization` — every field `PartialEq` compares except the -/// children. A scalar expression is serialized as data with each operator -/// node it reads replaced by `()`, so a reference to an -/// interned sub-DAG contributes nothing of its own here; -/// 2. for every child in [`OperatorNode::children`] order (operator inputs, -/// then scalar-referenced nodes), the child's own `structural_hash`, -/// memoized in `cache` by `Rc` pointer identity. -/// -/// Part 2 is what makes equal sub-DAGs hash equal whether they are reached -/// through an operator input or through a `scalar(v)`, and what keeps the -/// pass linear: a node is generally a DAG, and re-serializing a shared -/// descendant once per parent would cost `O(sub-DAG)` per node instead of -/// `O(1)` beyond the children's already-known hashes. A non-finite `f64` -/// serializes as `null`, merely widening one (still equality-checked) bucket. -pub(crate) fn structural_hash(node: &OperatorNode, cache: &mut NodeToStructuralHashCache) -> u64 { - fn child_hash(child: &Rc, cache: &mut NodeToStructuralHashCache) -> u64 { - let ptr = Rc::as_ptr(child); - if let Some(&h) = cache.get(&ptr) { - return h; - } - let h = structural_hash(child, cache); - cache.insert(ptr, h); - h - } - - let mut hasher = DefaultHasher::new(); - let own = ( - own_fields(node), - node.result_kind, - &node.schema, - &node.guarantee, - node.timing, - node.materialization, - ); - serde_json::to_string(&own) - .unwrap_or_default() - .hash(&mut hasher); - for child in node.children() { - child_hash(child, cache).hash(&mut hasher); - } - hasher.finish() -} - -/// Numeric `PartialEq` alone conflates signed zeros. The serialized check is -/// additional evidence, never a replacement for typed equality (JSON maps -/// non-finite floats to `null`). Used for the guarantee, whose bounds are -/// floats a shared node must preserve bit-for-bit, and for summary families, -/// whose `SketchParams` equality omits the recorded error and probability. -pub(crate) fn same_value(left: &T, right: &T) -> bool { - left == right - && match (serde_json::to_string(left), serde_json::to_string(right)) { - (Ok(left), Ok(right)) => left == right, - _ => false, - } -} - -/// Cache of child-pair comparisons already decided by [`same_node`], keyed by -/// pointer pair. Only interned (table-owned, hence alive) nodes are keys. -type NodePairToIsSameNodeCache = HashMap<(*const OperatorNode, *const OperatorNode), bool>; - -/// The sharing decision: typed equality of two nodes. -/// -/// `OperatorNode`'s derived `PartialEq` would recurse into children by value -/// even when both sides hold the same `Rc` (`OperatorNode` is not `Eq`, so -/// `Rc` gets no pointer shortcut), expanding a shared diamond once per path. -/// Children are therefore compared by pointer first; only when the pointers -/// differ (an equal child that was not legal to share) are the values -/// compared, memoized per pair so a diamond is still walked once. -fn same_node( - left: &OperatorNode, - right: &OperatorNode, - cache: &mut NodePairToIsSameNodeCache, -) -> bool { - let (lc, rc) = (left.children(), right.children()); - if lc.len() != rc.len() { - return false; - } - let children_equal = lc.iter().zip(&rc).all(|(a, b)| { - if Rc::ptr_eq(a, b) { - return true; - } - let key = (Rc::as_ptr(a), Rc::as_ptr(b)); - if let Some(&eq) = cache.get(&key) { - return eq; - } - let eq = same_node(a, b, cache); - cache.insert(key, eq); - eq - }); - children_equal - && left.result_kind == right.result_kind - && left.schema == right.schema - && left.timing == right.timing - && left.materialization == right.materialization - && same_value(&left.guarantee, &right.guarantee) - && same_value(&own_fields(left), &own_fields(right)) -} - -/// Bottom-up hash-consing table: structurally-equal, sharing-legal nodes -/// collapse onto one `Rc`. -/// -/// `buckets` is keyed by [`structural_hash`] — a coarse candidate filter -/// only. Every entry within one bucket is a full node kept around for the -/// [`same_node`] comparison that actually decides a match; a hash collision -/// between structurally different nodes just means a harmless linear scan of -/// a few extra candidates. -struct InternTable { - buckets: HashMap>>, - /// Persisted for the table's whole lifetime so hashing is `O(1)` per node - /// beyond its children; every cached node is owned by `buckets`. - structural_hash_cache: NodeToStructuralHashCache, - same_node_cache: NodePairToIsSameNodeCache, -} - -impl InternTable { - fn new() -> Self { - Self { - buckets: HashMap::new(), - structural_hash_cache: HashMap::new(), - same_node_cache: HashMap::new(), - } - } - - /// Intern one node whose children are already interned: look it up by - /// [`structural_hash`], confirm with [`same_node`], and — only when - /// sharing is legal (module doc, "Legality") — return the existing `Rc` - /// instead of allocating a new one. - fn intern(&mut self, node: OperatorNode) -> Rc { - let hash = structural_hash(&node, &mut self.structural_hash_cache); - // A node that is not legal to share is never *returned* as a match - // for something else; it still occupies a fresh slot in the bucket - // (harmless: later scans require legality of the new node too). - let reusable = node.is_asap() - || node - .non_asap() - .is_some_and(NonASAPOp::is_shareable_when_equal); - let bucket = self.buckets.entry(hash).or_default(); - if reusable { - if let Some(existing) = bucket - .iter() - .find(|candidate| same_node(candidate, &node, &mut self.same_node_cache)) - { - return Rc::clone(existing); - } - } - let rc = Rc::new(node); - bucket.push(Rc::clone(&rc)); - rc - } -} - -/// Input pointer → (input `Rc`, interned result). The input `Rc` is retained -/// so its address cannot be freed and reused by a fresh allocation while the -/// cache still maps it. -type NodeToInternedNodeCache = HashMap<*const OperatorNode, (Rc, Rc)>; - -/// Intern `node`'s children (recursively), then `node` itself. The rebuilt -/// node keeps `node`'s retained schema, result kind, guarantee, timing and -/// materialization: -/// every child is replaced by an equal node, so each derived property stays -/// valid, and the result is `PartialEq`-equal to the input. -fn intern_bottom_up( - table: &mut InternTable, - visited: &mut NodeToInternedNodeCache, - node: &Rc, -) -> Rc { - if let Some((_, interned)) = visited.get(&Rc::as_ptr(node)) { - return Rc::clone(interned); - } - let operator = node - .operator - .map_children(|child| intern_bottom_up(table, visited, child)); - let rebuilt = OperatorNode::with_schema(operator, node.schema.clone()) - .with_guarantee(node.guarantee.clone()) - .with_timing(node.timing) - .with_materialization(node.materialization); - let interned = table.intern(rebuilt); - visited.insert(Rc::as_ptr(node), (Rc::clone(node), Rc::clone(&interned))); - interned -} - -/// Share structurally-identical, sharing-legal sub-DAGs across a workload's -/// roots (or within one root). Every root's *value* is unchanged -/// (`PartialEq`-equal to its input) — only its internal `Rc` structure may -/// now alias another root's, or another part of its own DAG. A node already -/// reached through two paths is visited once. -/// -/// `Id` is caller-chosen — a workload entry's key, an index, a query name. -pub fn share_common_sub_dags( - roots: Vec<(Id, Rc)>, -) -> Vec<(Id, Rc)> { - let mut table = InternTable::new(); - let mut visited = NodeToInternedNodeCache::new(); - roots - .into_iter() - .map(|(id, root)| (id, intern_bottom_up(&mut table, &mut visited, &root))) - .collect() -} - -#[cfg(test)] -mod tests { - use super::*; - use asap_types::ir::operator::agg_intent::AggIntent; - use asap_types::ir::operator::asap::ASAPOp; - use asap_types::ir::operator::operator_properties::{ - BinaryOpKind, GroupKeys, Reduction, Source, - }; - use asap_types::ir::properties::guarantee::AccuracyGuarantee; - use asap_types::ir::scalar::expr_ir::{ColumnRef, CompareOpKind}; - use asap_types::ir::schema::sketch::{ - GroupingStrategy, SketchAlgorithm, SketchKind, SketchParams, SummaryUpdate, - }; - use asap_types::ir::schema::{DataType, Field, FieldDataType, Schema}; - use asap_types::ir::ScalarExpr; - use asap_types::ir::{BinaryOperator, NonASAPOp}; - - use asap_types::ir::properties::AccuracyTarget; - - fn node(op: NonASAPOp) -> Rc { - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(op)).unwrap() - } - - /// `[ts, service, value, latency]`, no unique key. - fn scan() -> Rc { - node(NonASAPOp::Scan { - source: Source::TimeSeries { metric: "m".into() }, - predicates: vec![], - schema: Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("service", DataType::Utf8, false), - Field::plain("value", DataType::Float64, false), - Field::plain("latency", DataType::Float64, false), - ], - 0, - vec![], - ), - }) - } - - fn quantile_agg(by: Vec, col: Option, q: f64) -> Rc { - node(NonASAPOp::Aggregate { - reduction: Reduction::by(by), - measures: vec![AggIntent::Quantile { - col, - q, - accuracy: AccuracyTarget::exact(), - }], - output_names: vec![], - filters: vec![], - having: None, - child: scan(), - }) - } - - fn compare(lhs: Rc, rhs: Rc) -> Rc { - node(NonASAPOp::BinaryOp { - operator: BinaryOperator { - checked_relative_division: false, - checked_finite_division: false, - kind: BinaryOpKind::Compare(CompareOpKind::Eq), - vector_match: None, - }, - return_bool: false, - lhs, - rhs, - }) - } - - fn two_roots(a: Rc, b: Rc) -> (Rc, Rc) { - let shared = share_common_sub_dags(vec![("a", a), ("b", b)]); - let [(_, ra), (_, rb)] = shared.as_slice() else { - panic!("expected 2 roots"); - }; - (Rc::clone(ra), Rc::clone(rb)) - } - - #[test] - fn distinct_column_quantiles_do_not_merge() { - // Grouped (unique key present) so only the differing `col` blocks it. - let (ra, rb) = two_roots( - quantile_agg(vec![1], Some(2), 0.5), - quantile_agg(vec![1], Some(3), 0.5), - ); - assert!(!Rc::ptr_eq(&ra, &rb)); - assert_ne!(ra, rb); - } - - // Two aggregates that differ only in one measure's `FILTER` predicate - // compute different values, so structural sharing keeps them apart. - #[test] - fn filtered_and_unfiltered_aggregates_do_not_merge() { - use asap_types::ir::scalar::ScalarValue; - use asap_types::ir::Predicate; - let unfiltered = quantile_agg(vec![1], Some(2), 0.5); - let filtered = node(NonASAPOp::Aggregate { - reduction: Reduction::by(vec![1]), - measures: vec![AggIntent::Quantile { - col: Some(2), - q: 0.5, - accuracy: AccuracyTarget::exact(), - }], - output_names: vec![], - filters: vec![Some(Predicate(ScalarExpr::Compare { - left: Box::new(ScalarExpr::Column(3)), - op: CompareOpKind::Gt, - right: Box::new(ScalarExpr::Literal(ScalarValue::Float64(1.0))), - semantics: asap_types::ir::ExprSemantics::Sql, - }))], - having: None, - child: scan(), - }); - let (ra, rb) = two_roots(unfiltered, filtered); - assert!( - !Rc::ptr_eq(&ra, &rb), - "a filtered measure must not be shared" - ); - assert_ne!(ra, rb); - } - - #[test] - fn identical_global_scalar_aggregates_share_their_single_result() { - let a = quantile_agg(vec![], Some(2), 0.9); - let b = quantile_agg(vec![], Some(2), 0.9); - assert_eq!(a, b, "fixture sanity: structurally equal"); - assert!( - !a.schema.has_unique_key(), - "fixture sanity: a global aggregate has no provable unique key" - ); - let (ra, rb) = two_roots(a, b); - assert!( - Rc::ptr_eq(&ra, &rb), - "a global scalar aggregate has one stable result per evaluation" - ); - } - - /// A global scalar result can be shared only with the same computation. - #[test] - fn distinct_global_quantiles_do_not_merge() { - let (a, b) = two_roots( - quantile_agg(vec![], Some(2), 0.5), - quantile_agg(vec![], Some(2), 0.9), - ); - assert!(!Rc::ptr_eq(&a, &b)); - let (a, b) = two_roots( - quantile_agg(vec![], Some(2), 0.5), - quantile_agg(vec![], Some(3), 0.5), - ); - assert!(!Rc::ptr_eq(&a, &b)); - } - - /// Two equal scans of a keyless table (SQL rows) are one node: each - /// reader reads every row the scan outputs. - #[test] - fn equal_scans_without_a_unique_key_are_shared() { - let table_scan = || { - node(NonASAPOp::Scan { - source: Source::Table { - table_ref: "metrics".into(), - }, - predicates: vec![], - schema: Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("city", DataType::Utf8, true), - Field::plain("value", DataType::Float64, true), - ], - 0, - vec![], - ), - }) - }; - let a = table_scan(); - assert!(!a.schema.has_unique_key(), "fixture sanity: keyless rows"); - let (ra, rb) = two_roots(a, table_scan()); - assert!(Rc::ptr_eq(&ra, &rb)); - } - - /// Grouped quantiles differing only in accuracy keep two roots over one shared scan. - #[test] - fn differing_accuracy_is_never_merged_but_the_scan_child_is_shared() { - let shared_scan = scan(); - let quantile = |epsilon: f64| { - node(NonASAPOp::Aggregate { - reduction: Reduction::by(vec![1]), - measures: vec![AggIntent::Quantile { - col: Some(2), - q: 0.99, - accuracy: AccuracyTarget::max_error(epsilon), - }], - output_names: vec![], - filters: vec![], - having: None, - child: Rc::clone(&shared_scan), - }) - }; - let a = quantile(0.01); - assert!(a.schema.has_unique_key(), "fixture sanity: grouped"); - let (ra, rb) = two_roots(a, quantile(0.05)); - assert!(!Rc::ptr_eq(&ra, &rb)); - assert_ne!(ra, rb); - let ( - Some(NonASAPOp::Aggregate { child: child_a, .. }), - Some(NonASAPOp::Aggregate { child: child_b, .. }), - ) = (ra.non_asap(), rb.non_asap()) - else { - panic!("expected Aggregate roots"); - }; - assert!(Rc::ptr_eq(child_a, child_b)); - } - - #[test] - fn median_and_explicit_half_percentile_merge() { - // Two spellings that lower to the identical grouped `Quantile { q: 0.5 }`. - let (m, p) = two_roots( - quantile_agg(vec![1], Some(2), 0.5), - quantile_agg(vec![1], Some(2), 0.5), - ); - assert!(Rc::ptr_eq(&m, &p)); - } - - #[test] - fn single_query_shares_its_own_repeated_sub_dag() { - // One root with the same grouped aggregate on both branches, built as - // two separately-allocated sub-DAGs (no sharing yet). - let root = compare( - quantile_agg(vec![1], Some(2), 0.5), - quantile_agg(vec![1], Some(2), 0.5), - ); - let shared = share_common_sub_dags(vec![("q", root)]); - let [(_, root)] = shared.as_slice() else { - panic!("expected 1 root"); - }; - let Some(NonASAPOp::BinaryOp { lhs, rhs, .. }) = root.non_asap() else { - panic!("expected BinaryOp root, got {root:?}"); - }; - assert!(Rc::ptr_eq(lhs, rhs)); - } - - #[test] - fn shared_root_value_is_unchanged() { - let a = quantile_agg(vec![1], Some(2), 0.5) - .as_ref() - .clone() - .with_guarantee(Some(AccuracyGuarantee::exact("fixture"))); - let before = Rc::new(a); - let (ra, _) = two_roots(Rc::clone(&before), Rc::clone(&before)); - assert_eq!(ra.as_ref(), before.as_ref()); - assert!( - ra.guarantee.is_some(), - "retained properties survive the rebuild" - ); - } - - // ── scalar-referenced sub-DAGs ────────────────────────────────────── - - /// `vector(scalar(sum by (service) (up)))`. - fn scalar_of_vector() -> Rc { - let sum_up = node(NonASAPOp::Aggregate { - reduction: Reduction::by(vec![1]), - measures: vec![AggIntent::Sum { col: Some(2) }], - output_names: vec![], - filters: vec![], - having: None, - child: scan(), - }); - assert!(sum_up.schema.has_unique_key(), "fixture sanity"); - node(NonASAPOp::PromqlVectorFromScalar( - ScalarExpr::PromqlScalarFromVector(sum_up), - )) - } - - fn bridged_vector(root: &Rc) -> &Rc { - match root.non_asap() { - Some(NonASAPOp::PromqlVectorFromScalar(ScalarExpr::PromqlScalarFromVector(v))) => v, - other => panic!("expected vector(scalar(v)), got {other:?}"), - } - } - - #[test] - fn scalar_referenced_vector_is_shared_across_queries() { - let (ra, rb) = two_roots(scalar_of_vector(), scalar_of_vector()); - assert!( - Rc::ptr_eq(bridged_vector(&ra), bridged_vector(&rb)), - "the vector read by scalar(v) is a child and must be interned" - ); - assert!( - Rc::ptr_eq(&ra, &rb), - "the scalar bridge has no unique key but returns the same rows" - ); - } - - #[test] - fn structural_hash_sees_through_a_scalar_reference() { - // Two equal bridges must hash equal whether or not their referenced - // vector is the same Rc — the reference contributes the vector's - // memoized hash, not its identity. - let a = scalar_of_vector(); - let b = scalar_of_vector(); - let mut cache = HashMap::new(); - assert_eq!( - structural_hash(&a, &mut cache), - structural_hash(&b, &mut cache) - ); - assert_eq!( - cache.len(), - 4, - "aggregate + scan cached once per root: {cache:?}" - ); - let other = node(NonASAPOp::PromqlVectorFromScalar( - ScalarExpr::PromqlScalarFromVector(quantile_agg(vec![1], Some(2), 0.5)), - )); - assert_ne!( - structural_hash(&a, &mut cache), - structural_hash(&other, &mut cache) - ); - } - - // ── ASAP nodes ────────────────────────────────────────────────────── - - fn summary_agg(alpha: f64, guarantee: Option) -> Rc { - let family = FieldDataType::Sketch( - SketchKind::new(SketchAlgorithm::DDSketch, SketchParams::DDSketch { alpha }), - GroupingStrategy::default(), - ); - let schema = Schema::lifted(vec![Field::new("state", family.clone(), false)], None); - assert!(!schema.has_unique_key(), "fixture sanity"); - Rc::new( - OperatorNode::with_schema( - Operator::ASAP(ASAPOp::SummaryAgg { - child: scan(), - family, - input: SummaryUpdate::column(ColumnRef::SampleValue), - reduction: Reduction::PerEntity, - grouping: GroupingStrategy::default(), - filter: None, - }), - schema, - ) - .with_guarantee(guarantee), - ) - } - - #[test] - fn asap_nodes_share_without_a_unique_key() { - let exact = || Some(AccuracyGuarantee::exact("fixture")); - let (ra, rb) = two_roots(summary_agg(0.01, exact()), summary_agg(0.01, exact())); - assert!(Rc::ptr_eq(&ra, &rb)); - assert!(ra.guarantee.is_some()); - } - - #[test] - fn asap_nodes_with_distinct_parameters_or_guarantees_are_not_shared() { - let exact = || Some(AccuracyGuarantee::exact("fixture")); - let (ra, rb) = two_roots(summary_agg(0.01, exact()), summary_agg(0.001, exact())); - assert!(!Rc::ptr_eq(&ra, &rb), "different sketch parameters"); - let (ra, rb) = two_roots(summary_agg(0.01, exact()), summary_agg(0.01, None)); - assert!( - !Rc::ptr_eq(&ra, &rb), - "an unknown guarantee never borrows an exact one" - ); - assert!(rb.guarantee.is_none()); - } - - #[test] - fn evaluations_share_their_producer_but_not_each_other() { - use asap_types::ir::schema::sketch::SketchStatistic; - let evaluation = |q: f64| { - Rc::new(OperatorNode::with_schema( - Operator::ASAP(ASAPOp::SummaryEstimate { - summary_input: summary_agg(0.01, None), - query: SketchStatistic::Quantile { q }, - }), - Schema::lifted( - vec![Field::plain("quantile", DataType::Float64, false)], - None, - ), - )) - }; - let (p95, p99) = two_roots(evaluation(0.95), evaluation(0.99)); - let producer = |n: &Rc| Rc::clone(n.children()[0]); - assert!(!Rc::ptr_eq(&p95, &p99)); - assert!(Rc::ptr_eq(&producer(&p95), &producer(&p99))); - } - - // ── structural_hash (DAG-aware memoization) ───────────────────────── - - #[test] - fn structural_hash_is_stable_across_cache_states() { - let agg = quantile_agg(vec![1], Some(2), 0.5); - let mut cold = HashMap::new(); - let mut warm = HashMap::new(); - structural_hash(&scan(), &mut warm); - assert_eq!( - structural_hash(&agg, &mut cold), - structural_hash(&agg, &mut warm), - "hash must be independent of unrelated cache state" - ); - } - - #[test] - fn structural_hash_of_an_internally_shared_dag_matches_the_unshared_equivalent() { - let agg = quantile_agg(vec![1], Some(2), 0.5); - let shared_root = compare(Rc::clone(&agg), Rc::clone(&agg)); - let unshared_root = compare( - quantile_agg(vec![1], Some(2), 0.5), - quantile_agg(vec![1], Some(2), 0.5), - ); - assert_eq!( - structural_hash(&shared_root, &mut HashMap::new()), - structural_hash(&unshared_root, &mut HashMap::new()), - ); - } - - #[test] - fn structural_hash_memoizes_a_shared_descendant_exactly_once() { - let agg = quantile_agg(vec![1], Some(2), 0.5); - let root = compare(Rc::clone(&agg), Rc::clone(&agg)); - let mut cache = HashMap::new(); - structural_hash(&root, &mut cache); - assert_eq!( - cache.len(), - 2, - "one entry per unique node in the shared branch (Aggregate + Scan): {cache:?}" - ); - } - - /// `DISTINCT` (every column) returns the same rows each time; a `Dedup` - /// on some columns keeps an unspecified row per key. - #[test] - fn dedup_shares_only_on_every_column() { - let dedup = |cols: Vec| { - node(NonASAPOp::Dedup { - cols, - child: scan(), - }) - }; - let (ra, rb) = two_roots(dedup(vec![]), dedup(vec![])); - assert!(Rc::ptr_eq(&ra, &rb)); - let (ra, rb) = two_roots(dedup(vec![1]), dedup(vec![1])); - assert!(!Rc::ptr_eq(&ra, &rb)); - } - - /// `LIMIT 10` without an order may return any 10 rows, so two copies may - /// differ; one shared copy would make both readers see the same rows. - #[test] - fn unordered_limits_are_not_shared() { - let limit = || { - node(NonASAPOp::Limit { - n: Some(10), - offset: 0, - partition_by: GroupKeys::default(), - child: scan(), - }) - }; - let (ra, rb) = two_roots(limit(), limit()); - assert!(!Rc::ptr_eq(&ra, &rb)); - } - - /// A `LIMIT` over a `Sort` with the same partitions keeps the first rows - /// of a fixed order; over a `Sort` partitioned differently, which rows - /// come first still depends on input order. - #[test] - fn limits_over_a_sort_with_their_partitions_are_shared() { - let limit = |sort_partitions: GroupKeys| { - node(NonASAPOp::Limit { - n: Some(10), - offset: 0, - partition_by: GroupKeys::default(), - child: node(NonASAPOp::Sort { - keys: vec![asap_types::ir::scalar::SortKey { - expr: ScalarExpr::column(2), - ascending: false, - nulls_first: false, - }], - partition_by: sort_partitions, - child: scan(), - }), - }) - }; - let (ra, rb) = two_roots(limit(GroupKeys::default()), limit(GroupKeys::default())); - assert!(Rc::ptr_eq(&ra, &rb)); - let (ra, rb) = two_roots(limit(GroupKeys::by(vec![1])), limit(GroupKeys::by(vec![1]))); - assert!(!Rc::ptr_eq(&ra, &rb)); - } - - /// `count without (service)` has no provable unique key, but it computes - /// the same rows from the same input, so equal copies are one node. - #[test] - fn without_grouping_is_shared() { - let without_agg = || { - node(NonASAPOp::Aggregate { - reduction: Reduction::Reduce(GroupKeys::without(vec![0])), - measures: vec![AggIntent::Count { - accuracy: AccuracyTarget::exact(), - }], - output_names: vec![], - filters: vec![], - having: None, - child: scan(), - }) - }; - let a = without_agg(); - assert!(!a.schema.has_unique_key()); - let (ra, rb) = two_roots(a, without_agg()); - assert!(Rc::ptr_eq(&ra, &rb)); - } - - #[test] - fn already_shared_nodes_are_visited_once() { - // A diamond already present in the input stays one node and is not - // re-interned per path. - let agg = quantile_agg(vec![1], Some(2), 0.5); - let root = compare(Rc::clone(&agg), Rc::clone(&agg)); - let shared = share_common_sub_dags(vec![("q", root)]); - let Some(NonASAPOp::BinaryOp { lhs, rhs, .. }) = shared[0].1.non_asap() else { - panic!("expected BinaryOp root"); - }; - assert!(Rc::ptr_eq(lhs, rhs)); - assert_eq!(OperatorNode::reachable(&shared[0].1).len(), 3); - } - - // Comparing a shareable node whose equal-but-unshareable children form a - // deep diamond must not expand the diamond once per path. The timeout is - // a coarse runaway guard, not a performance SLA. - #[test] - fn shared_diamond_does_not_expand_during_comparison() { - let (done, completion) = std::sync::mpsc::channel(); - let worker = std::thread::spawn(move || { - fn keyed_diamond() -> Rc { - // An unordered `Limit` at each of the 24 levels is never - // shared, so comparing the `Dedup` on top walks the diamond. - let mut current = scan(); - for _ in 0..24 { - let limited = node(NonASAPOp::Limit { - n: Some(10), - offset: 0, - partition_by: GroupKeys::default(), - child: current, - }); - current = compare(Rc::clone(&limited), limited); - } - node(NonASAPOp::Dedup { - cols: vec![], - child: current, - }) - } - let (ra, rb) = two_roots(keyed_diamond(), keyed_diamond()); - assert!(Rc::ptr_eq(&ra, &rb)); - done.send(()).unwrap(); - }); - completion - .recv_timeout(std::time::Duration::from_secs(5)) - .expect("comparison expanded the shared DAG"); - worker.join().unwrap(); - } - - /// A keyed (hence shareable) projection emitting the literal `value`. - fn keyed_literal(value: f64) -> Rc { - let keyed = node(NonASAPOp::Scan { - source: Source::TimeSeries { metric: "m".into() }, - predicates: vec![], - schema: Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("service", DataType::Utf8, false), - ], - 0, - vec![vec![1]], - ), - }); - node(NonASAPOp::Project { - cols: vec![ - asap_types::ir::ProjectItem { - alias: None, - expr: ScalarExpr::Column(1), - }, - asap_types::ir::ProjectItem { - alias: Some("v".into()), - expr: ScalarExpr::literal_f64(value), - }, - ], - qualifier: None, - child: keyed, - }) - } - - /// Sharing preserves IEEE signed zero, and JSON's `null` encoding of - /// non-finite floats never becomes the equality decision. - #[test] - fn signed_zero_and_nonfinite_values_remain_distinct() { - assert!( - keyed_literal(0.0).schema.has_unique_key(), - "fixture is shareable" - ); - for (a, b) in [ - (0.0, -0.0), - (-0.0, 0.0), - (f64::INFINITY, f64::NEG_INFINITY), - (f64::NAN, f64::NAN), - ] { - let (ra, rb) = two_roots(keyed_literal(a), keyed_literal(b)); - assert!(!Rc::ptr_eq(&ra, &rb), "{a} and {b} must not be shared"); - } - let (ra, rb) = two_roots(keyed_literal(f64::INFINITY), keyed_literal(f64::INFINITY)); - assert!(Rc::ptr_eq(&ra, &rb)); - } -} diff --git a/crates/logical-asap-optimization/src/test_support.rs b/crates/logical-asap-optimization/src/test_support.rs index 988cb4210..7eceb36ee 100644 --- a/crates/logical-asap-optimization/src/test_support.rs +++ b/crates/logical-asap-optimization/src/test_support.rs @@ -4,20 +4,11 @@ use std::rc::Rc; -// Stage 1 internals other crates' tests call: Pass 1's working form and its -// population step, single-candidate composition, Stage 0 hash-consing, and -// the Pass 2 rule entry points those tests build memos with. -pub use crate::pass1::memo_population::try_populate_pass1; +// Stage 1 internals other crates' tests call: Pass 1's working form, +// and single-candidate composition. pub use crate::pass1::per_computation_candidates::{ compose_logical_candidate, generate_per_computation_candidates, PerComputationCandidates, }; -pub use crate::pass2::asap_aware_cse::{ - rollup_candidates, rollup_pass2_candidates, tumbling_window_candidates, - window_composition_pass2_candidates, window_composition_shared_chain_candidates, - with_union_scan_labels, -}; -pub use crate::pass2::memo::window_candidates::add_local_window_composition; -pub use crate::stage0::cse::share_common_sub_dags; use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::OperatorNode; @@ -26,40 +17,6 @@ use asap_types::workload::{ QueryRequirements, QueryWorkload, }; -/// Pass 2 after local window composition, on a memo the test composed -/// itself (or not at all): rules 2–9, then -/// `add_shared_summary_state_groups`. -/// Production runs these through -/// [`generate_logical_asap_memo_candidates`](crate::pass2::logical_memo_candidates::generate_logical_asap_memo_candidates). -pub fn sharing_rules_and_shared_state_groups( - mut memo: asap_types::stages::Memo, -) -> Result { - crate::pass2::logical_memo_candidates::apply_sharing_rules(&mut memo)?; - crate::pass2::memo::state_groups::add_shared_summary_state_groups(&mut memo)?; - Ok(memo) -} - -/// The groups `add_shared_summary_state_groups` -/// added to an expanded memo, in order: each has one candidate, the -/// shared build (origin `SharedBuild`); no other group's candidates are -/// only that. -pub fn shared_summary_state_groups( - memo: &asap_types::stages::Memo, -) -> Vec { - memo.candidates - .iter() - .enumerate() - .filter(|(_, candidates)| { - matches!( - candidates.as_slice(), - [only] if only.origin - == asap_types::stages::OriginOfCandidateForMemoGroup::SharedBuild - ) - }) - .map(|(group, _)| asap_types::stages::MemoGroupId(group as u32)) - .collect() -} - /// The statistic classic HLL's failure probability at `precision` and /// `relative_error` over `source` is left as, for Stage 3 tests that bind /// it. The estimator keeps the format crate-private. @@ -108,12 +65,8 @@ use std::time::Duration; use asap_types::ir::operator::agg_intent::AggIntent; use asap_types::ir::operator::operator_properties::{GroupKeys, Reduction, Source}; -use asap_types::ir::properties::timing::{ - apply_materialization_timings, MaterializationAssignment, NodeToTimedNodeCache, -}; use asap_types::ir::schema::{ColumnId, DataType, Field, Schema}; use asap_types::ir::{NonASAPOp, Predicate, ScalarExpr, TimeRangeKind}; -use asap_types::physical::{Materialization, Medium}; /// A `TimeSeries("m")` scan over `[ts(0), value(1), labels...]`, time index 0, /// no unique key. @@ -220,27 +173,6 @@ pub fn time_range(range: Duration, child: Rc) -> Rc .unwrap() } -/// `root` timed under the default (every sub-DAG `NotMaterialized`) -/// assignment — the shape export and the post-ASAP validators consume. -pub fn timed(root: &Rc) -> Rc { - apply_materialization_timings( - root, - &MaterializationAssignment::default(), - &mut NodeToTimedNodeCache::new(), - ) - .expect("default materialization timings apply") -} - -/// `root` timed with every sub-DAG `MatAtIngestion` where the DAG allows it. -pub fn at_ingestion(root: &Rc) -> Rc { - apply_materialization_timings( - root, - &MaterializationAssignment::with_default(Materialization::MatAtIngestion(Medium::Memory)), - &mut NodeToTimedNodeCache::new(), - ) - .expect("at-ingestion materialization timings apply") -} - /// The Count-Min summary of a Stage 1 candidate of `query`. pub fn cms_summary(query: &str) -> OperatorNode { use crate::pass1::per_computation_candidates::compose_logical_candidate; @@ -251,7 +183,7 @@ pub fn cms_summary(query: &str) -> OperatorNode { let root = lower_promql(query, target); // Inspect each local computation choice under the built-in deployment. let per_computation = generate_per_computation_candidates( - &crate::stage0::generate_dag_of_logical_memo_groups(&[QueryRoot::Operator(root)]), + &crate::stage0::memo_of_queries(&[QueryRoot::Operator(root)]).unwrap(), ) .unwrap() .without_unsupported_summaries(None); @@ -277,7 +209,7 @@ pub fn cms_summary(query: &str) -> OperatorNode { Some(ASAPOp::SummaryAgg { family: FieldDataType::Sketch(kind, _), .. - }) if *kind.algorithm() == SketchAlgorithm::Cms + }) if *kind == SketchAlgorithm::Cms ) }) .map(|node| (*node).clone()) diff --git a/crates/physical-asap-optimization/Cargo.toml b/crates/physical-asap-optimization/Cargo.toml index 9bc25ba9d..dc4220f5e 100644 --- a/crates/physical-asap-optimization/Cargo.toml +++ b/crates/physical-asap-optimization/Cargo.toml @@ -6,29 +6,7 @@ edition = "2021" # #509 Stage 2, physical ASAP-aware optimization: materialization and # physical operator implementation of each Stage 1 candidate, and the # PhysicalASAPMemoCandidates Stage 3 selects from (`catalog`). Depends only on -# asap-types (the IR Stage 1 emits) and the shared accuracy model -# (asap-accuracy, guarantee recomposition of a selected plan), never on -# Stage 1's crate, plan selection or the executor. Tests build their inputs -# with Stage 1. +# asap-types, never on Stage 1's crate, plan selection or the executor. [dependencies] -asap-accuracy = { path = "../accuracy" } asap-types = { path = "../types" } -# The tumbling-window retention of a realized plan (`AssignmentRealizer::realize`). -asap-cost = { path = "../cost" } -rmp-serde = "1" -serde_json = "1" thiserror = "2" - -[features] -# Exposes `test_support` (single-candidate realization, operator -# implementation, tumbling window coverage, and `test_support::catalog`: catalog -# internals plan selection's unit tests exercise with its solver), for other -# crates' tests. Tests also enumerate whole workloads with the reference pricer. -test-support = ["reference-pricer"] -# `optimize_physical_asap`: the whole-workload enumeration behind the -# developer tools' reference pricing (`asap-devtools`). Planning does not use it. -reference-pricer = [] - -[dev-dependencies] -asap-types = { path = "../types", features = ["test-support"] } -asap-logical-asap-optimization = { path = "../logical-asap-optimization", features = ["test-support"] } diff --git a/crates/physical-asap-optimization/src/catalog/configurations.rs b/crates/physical-asap-optimization/src/catalog/configurations.rs deleted file mode 100644 index c5b8ade0a..000000000 --- a/crates/physical-asap-optimization/src/catalog/configurations.rs +++ /dev/null @@ -1,425 +0,0 @@ -//! Finite, unpriced execution configurations over typed immediate candidates. -//! -//! This does not enumerate consumer subsets. Current summary eligibility is a -//! conjunction of individual-reader restrictions and pairwise interval/phase -//! compatibility; selected-reader constraints encode those conditions. Complete -//! Stage 2 validation still checks operator legality and upstream timing. -//! Potential shared non-summary nodes also receive query materialization; Stage 2 -//! checks highest-shared eligibility on the selected graph, not all possible consumers. -use asap_types::stages::{ProducerCandidates, ProducerSetInterner}; -use std::collections::HashMap; - -use crate::summary_materialization_eligibility; -use asap_types::ir::operator::{agg_has_exact_per_entity_kernel, Reduction}; -use asap_types::ir::properties::ExecutionTiming; -use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode}; -use asap_types::physical::{Materialization, Medium}; -use asap_types::workload::{DataWorkload, QueryWorkloadEntry}; - -use super::inventory::CandidateInventory; -use asap_types::ir::QueryRoot; -use asap_types::stages::{ - output_references, MemoGroupId, OperatorNodeOfCandidateForMemoGroupId, - PhysicalASAPMemoCandidates, PhysicalCandidate, PhysicalCandidateId, PhysicalCandidateKey, - PruneReason, PrunedCandidate, ReaderConstraint, -}; - -/// [`PhysicalCandidate::may_keep_across_evaluations_per_sole_reader_query`] for `node` configured -/// with `materialization`: for each query in `entries`, whether -/// `summary_materialization_eligibility` with that query as the only reader -/// allows keeping it across evaluations. Empty unless `node` is a tumbling-window chain, -/// `materialization` is `MatAtQueryKeptAcrossEvaluations(Memory)` and the -/// deployment keeps query-time results. -pub fn may_keep_across_evaluations_per_sole_reader_query( - node: &OperatorNode, - materialization: Materialization, - entries: &[QueryWorkloadEntry], - data: &DataWorkload, - query_time_results_retained: bool, -) -> Vec { - if materialization != Materialization::MatAtQueryKeptAcrossEvaluations(Medium::Memory) - || !matches!(node.asap(), Some(ASAPOp::SummaryTumblingWindowChain { .. })) - || !query_time_results_retained - { - return Vec::new(); - } - (0..entries.len()) - .map(|reader| { - summary_materialization_eligibility( - node, - &[reader], - entries, - data, - query_time_results_retained, - ) - .is_ok_and(|eligibility| eligibility.may_keep_across_evaluations) - }) - .collect() -} - -#[derive(Debug, thiserror::Error)] -pub enum ConfigurationError { - #[error("invalid source candidate inventory: {0}")] - InvalidInventory(String), - #[error("candidate {candidate} already has timing or materialization assigned")] - AssignedExecution { candidate: usize }, -} - -/// Each operator node of `source` with every timing and materialization -/// Stage 2 allows, one candidate each. `query_roots` are the memo's; their -/// group references are `source`'s output references. -pub fn enumerate_candidate_configurations( - source: &CandidateInventory, - query_roots: &[QueryRoot], - entries: &[QueryWorkloadEntry], - data: &DataWorkload, - query_time_results_retained: bool, - raw_data_retained: bool, -) -> Result { - let invalid = |message: &str| ConfigurationError::InvalidInventory(message.into()); - source.validate().map_err(|error| invalid(&error))?; - let output_queries: Vec = output_references(query_roots) - .into_iter() - .map(|(query, _)| query) - .collect(); - let source_count = source.operator_nodes_of_candidates_for_memo_groups.len(); - if output_queries.len() != source.query_output_nodes.len() - || output_queries.iter().any(|&query| query >= entries.len()) - { - return Err(invalid("template or output/query mapping is incomplete")); - } - if source - .implementation_roots_of_group - .iter() - .flatten() - .any(|id| id.0 >= source_count) - { - return Err(invalid("group references missing source candidate")); - } - let mut query_outputs = vec![Vec::new(); entries.len()]; - let mut potential = vec![vec![false; entries.len()]; source_count]; - let mut may_produce_query_output = vec![false; source_count]; - for (output, &query) in output_queries.iter().enumerate() { - query_outputs[query].push(output); - let mut pending = source.query_output_nodes[output].clone(); - for root in &pending { - may_produce_query_output[root.0] = true; - } - while let Some(candidate) = pending.pop() { - if !potential[candidate.0][query] { - potential[candidate.0][query] = true; - pending.extend( - source.producer_nodes_per_input[candidate.0] - .iter() - .flat_map(|input| input.iter().copied()), - ); - } - } - } - let mut candidates = Vec::new(); - let mut reader_constraints = Vec::new(); - let mut pruned_candidates = Vec::new(); - let mut source_configurations = vec![Vec::new(); source_count]; - for source_id in 0..source_count { - let node = &source.operator_nodes_of_candidates_for_memo_groups[source_id].physical_node; - if node.timing.is_some() || node.materialization != Materialization::NotMaterialized { - return Err(ConfigurationError::AssignedExecution { - candidate: source_id, - }); - } - let summary = matches!( - node.asap(), - Some(ASAPOp::SummaryAgg { .. } | ASAPOp::SummaryTumblingWindowChain { .. }) - ); - let forced_query = matches!( - node.asap(), - Some( - ASAPOp::SummaryEstimate { .. } - | ASAPOp::EvaluatePopulation { .. } - | ASAPOp::MaintainPopulation { .. } - ) - ); - let queries = potential[source_id] - .iter() - .enumerate() - .filter_map(|(query, reached)| reached.then_some(query)) - .collect::>(); - let mut modes = vec![(Materialization::NotMaterialized, ExecutionTiming::QueryTime)]; - if summary { - modes.push(( - Materialization::MatAtQuery(Medium::Memory), - ExecutionTiming::QueryTime, - )); - if queries.iter().any(|&query| { - summary_mode_allowed( - node, - &[query], - Materialization::MatAtIngestion(Medium::Memory), - entries, - data, - query_time_results_retained, - ) - }) { - modes.push(( - Materialization::MatAtIngestion(Medium::Memory), - ExecutionTiming::IngestionTime, - )); - } - if matches!(node.asap(), Some(ASAPOp::SummaryTumblingWindowChain { .. })) - && queries.iter().any(|&query| { - summary_mode_allowed( - node, - &[query], - Materialization::MatAtQueryKeptAcrossEvaluations(Medium::Memory), - entries, - data, - query_time_results_retained, - ) - }) - { - modes.push(( - Materialization::MatAtQueryKeptAcrossEvaluations(Medium::Memory), - ExecutionTiming::QueryTime, - )); - } - } else if !forced_query { - modes.push(( - Materialization::NotMaterialized, - ExecutionTiming::IngestionTime, - )); - } - // Potential reach only generates the choice. Selected-reader cardinality - // and Stage 2's highest-shared rule determine actual eligibility. - if !summary && queries.len() >= 2 { - modes.push(( - Materialization::MatAtQuery(Medium::Memory), - ExecutionTiming::QueryTime, - )); - } - // Configurations the deployment cannot run are not generated; each is - // recorded as pruned. - let group = source.operator_nodes_of_candidates_for_memo_groups[source_id].group(); - let mut prune_modes = |reason, applies: &dyn Fn(ExecutionTiming) -> bool| { - modes.retain(|&(materialization, timing)| { - let pruned = applies(timing); - if pruned { - pruned_candidates.push(PrunedCandidate { - group, - candidate: PhysicalCandidateKey { - operator_node: OperatorNodeOfCandidateForMemoGroupId(source_id), - timing, - materialization, - }, - reason, - }); - } - !pruned - }); - }; - if !per_entity_aggregate_executable(node) { - prune_modes(PruneReason::PerEntityAggregateNotExecutable, &|_| true); - } - // Without raw data retained, only ingestion-time work reads raw rows. - if !raw_data_retained && matches!(node.operator, Operator::NonASAP(NonASAPOp::Scan { .. })) - { - prune_modes(PruneReason::RawDataNotRetained, &|timing| { - timing != ExecutionTiming::IngestionTime - }); - } - for (materialization, timing) in modes { - let id = PhysicalCandidateId(candidates.len()); - source_configurations[source_id].push(id); - candidates.push(PhysicalCandidate { - operator_node: OperatorNodeOfCandidateForMemoGroupId(source_id), - timing, - materialization, - producer_candidates_per_input: Vec::new(), - may_produce_query_output: may_produce_query_output[source_id] - && timing == ExecutionTiming::QueryTime, - may_keep_across_evaluations_per_sole_reader_query: - may_keep_across_evaluations_per_sole_reader_query( - node, - materialization, - entries, - data, - query_time_results_retained, - ), - }); - if materialization == Materialization::MatAtQuery(Medium::Memory) { - if !summary { - reader_constraints.push(ReaderConstraint::HighestSharedQueryMaterialization { - candidate: id, - query_outputs: query_outputs.clone(), - }); - } - reader_constraints.push(ReaderConstraint::MinimumQueryReaders { - candidate: id, - query_outputs: query_outputs.clone(), - minimum: 2, - }); - } - if summary { - let eligible = queries - .iter() - .copied() - .filter(|&query| { - summary_mode_allowed( - node, - &[query], - materialization, - entries, - data, - query_time_results_retained, - ) - }) - .collect::>(); - let outputs = eligible - .iter() - .flat_map(|&query| query_outputs[query].iter().copied()) - .collect(); - reader_constraints.push(ReaderConstraint::AllowedOutputs { - candidate: id, - outputs, - }); - // The current Stage 2 rules require common intervals/phases, - // whose conflicts are pairwise. No arbitrary set predicate is - // approximated by this encoding. - for (position, &left) in eligible.iter().enumerate() { - for &right in &eligible[position + 1..] { - if !summary_mode_allowed( - node, - &[left, right], - materialization, - entries, - data, - query_time_results_retained, - ) { - for &left_output in &query_outputs[left] { - for &right_output in &query_outputs[right] { - reader_constraints.push( - ReaderConstraint::IncompatibleOutputs { - candidate: id, - left: left_output, - right: right_output, - }, - ); - } - } - } - } - } - } - } - } - let mut producer_sets = ProducerSetInterner::default(); - // Compatibility depends on consumer timing and producer mode/timing only. - // Evaluate each source configuration once per timing, then reuse this exact - // index; reader eligibility remains expressed by separate graph constraints. - let mut timing_index = Vec::with_capacity(source_configurations.len()); - for configurations in &source_configurations { - let mut by_timing = [ProducerCandidates::default(), ProducerCandidates::default()]; - for (timing_index, timing) in [ExecutionTiming::QueryTime, ExecutionTiming::IngestionTime] - .into_iter() - .enumerate() - { - let mut compatible = Vec::new(); - for &producer in configurations { - if compatible_timing(timing, &candidates[producer.0]) { - compatible.push(producer); - } - } - by_timing[timing_index] = producer_sets.intern(compatible); - } - timing_index.push(by_timing); - } - let mut input_cache: HashMap<(ProducerCandidates, ExecutionTiming), ProducerCandidates> = - HashMap::new(); - for candidate in &mut candidates { - let mut requirements = Vec::new(); - for input in &source.producer_nodes_per_input[candidate.operator_node.0] { - let key = (input.clone(), candidate.timing); - let compatible_producers = if let Some(producer_candidates) = input_cache.get(&key) { - producer_candidates.clone() - } else { - let timing = match candidate.timing { - ExecutionTiming::QueryTime => 0, - ExecutionTiming::IngestionTime => 1, - }; - let mut compatible = Vec::new(); - for producer in input { - compatible.extend(timing_index[producer.0][timing].iter().copied()); - } - let producer_candidates = producer_sets.intern(compatible); - input_cache.insert(key, producer_candidates.clone()); - producer_candidates - }; - requirements.push(compatible_producers); - } - candidate.producer_candidates_per_input = requirements; - } - let mut result = PhysicalASAPMemoCandidates::new( - source.operator_nodes_of_candidates_for_memo_groups.clone(), - candidates, - query_roots.to_vec(), - reader_constraints, - ); - result.pruned_candidates = pruned_candidates; - super::validate_candidates(&result).map_err(|error| invalid(&error))?; - Ok(result) -} - -/// Whether the executor can run `node`: false only for an exact per-entity -/// `Aggregate` (a per-series range reduction over raw data samples) that -/// has more than one measure, a measure without an exact per-entity kernel -/// ([`agg_has_exact_per_entity_kernel`]), a measure filter or a `HAVING`. -/// The rule reads one operator node, so Stage 2 does not generate the node -/// rather than Stage 3 rejecting each plan that reads it. -fn per_entity_aggregate_executable(node: &OperatorNode) -> bool { - match &node.operator { - Operator::NonASAP(NonASAPOp::Aggregate { - reduction: Reduction::PerEntity, - measures, - filters, - having, - .. - }) => { - matches!(measures.as_slice(), [measure] if agg_has_exact_per_entity_kernel(measure)) - && filters.iter().all(Option::is_none) - && having.is_none() - } - _ => true, - } -} - -pub fn compatible_timing(consumer_timing: ExecutionTiming, producer: &PhysicalCandidate) -> bool { - consumer_timing == producer.timing - || (consumer_timing == ExecutionTiming::QueryTime - && matches!(producer.materialization, Materialization::MatAtIngestion(_))) -} - -fn summary_mode_allowed( - node: &OperatorNode, - readers: &[usize], - mode: Materialization, - entries: &[QueryWorkloadEntry], - data: &DataWorkload, - query_time_results_retained: bool, -) -> bool { - let Ok(eligibility) = summary_materialization_eligibility( - node, - readers, - entries, - data, - query_time_results_retained, - ) else { - return false; - }; - eligibility.range_compatibility.is_ok() - && match mode { - Materialization::NotMaterialized | Materialization::MatAtQuery(_) => true, - Materialization::MatAtIngestion(_) => eligibility.at_ingestion.is_ok(), - Materialization::MatAtQueryKeptAcrossEvaluations(_) => { - eligibility.may_keep_across_evaluations - } - } -} diff --git a/crates/physical-asap-optimization/src/catalog/factoring.rs b/crates/physical-asap-optimization/src/catalog/factoring.rs deleted file mode 100644 index 4ee9e4b68..000000000 --- a/crates/physical-asap-optimization/src/catalog/factoring.rs +++ /dev/null @@ -1,249 +0,0 @@ -//! Memoize identical producer interfaces and quotient equivalent terminal readers. -//! Internal computations are never merged. -use std::collections::HashMap; - -use asap_types::ir::OperatorNode; -use asap_types::stages::MemoGroupId; - -use asap_types::stages::{CandidateForMemoGroupId, PhysicalCandidateId}; -use asap_types::stages::{ProducerCandidates, ProducerSetInterner}; - -#[derive(Hash, PartialEq, Eq)] -pub struct MemoGroupOutputKey(Vec); - -impl MemoGroupOutputKey { - /// Include everything output_contract_satisfied reads from its expected - /// node. Byte-exact keys preserve parameters that PartialEq or JSON may - /// erase, and include closedness and every required uniqueness key. - pub fn new(group: MemoGroupId, expected: &OperatorNode) -> Result { - rmp_serde::to_vec(&(group.0, &expected.result_kind, &expected.schema)) - .map(Self) - .map_err(|error| error.to_string()) - } -} - -#[derive(Default)] -pub struct MemoGroupOutputToCompatibleProducersCache { - results: HashMap, - lists: ProducerSetInterner, - #[cfg(feature = "test-support")] - disabled: bool, - #[cfg(feature = "test-support")] - hits: std::cell::Cell, -} - -impl MemoGroupOutputToCompatibleProducersCache { - pub fn get(&self, key: &MemoGroupOutputKey) -> Option { - #[cfg(feature = "test-support")] - if self.disabled { - return None; - } - let found = self.results.get(key).cloned(); - #[cfg(feature = "test-support")] - if found.is_some() { - self.hits.set(self.hits.get() + 1); - } - found - } - - #[cfg(feature = "test-support")] - pub fn hits(&self) -> usize { - self.hits.get() - } - - #[cfg(feature = "test-support")] - pub fn disabled() -> Self { - Self { - disabled: true, - ..Self::default() - } - } - - pub fn remember( - &mut self, - key: MemoGroupOutputKey, - producer_candidates: Vec, - ) -> ProducerCandidates { - let producer_candidates = self.lists.intern(producer_candidates); - self.results.insert(key, producer_candidates.clone()); - producer_candidates - } -} - -/// Quotient only terminal unary summary readers. Internal consumers and merge -/// operators retain their original identities. A selection binds the -/// factored reader independently per output reference (one selected node -/// each), so every selection of the unfactored readers maps to one of the -/// factored reader, and each binding of it is an unfactored reader with the -/// same complete interface. -/// -/// These operators bypass ConcreteRows' witness-based row certification. -/// certified_price checks every supplied producer; template-based proposal -/// prices remain advisory. Execution and accuracy are recomputed from selected -/// producers, never from the representative template's child witness. -/// -/// A kept node's `candidates_for_memo_group` becomes the union, ascending, -/// of the candidates of the nodes factored into it, and its -/// `accuracy_budget_parts` the largest of theirs. -pub fn factor_terminal_readers( - inventory: &mut super::inventory::CandidateInventory, -) -> Result { - use super::inventory::CandidateInventoryError; - use asap_types::ir::ASAPOp; - let invalid = |error: String| CandidateInventoryError::InvalidMemo(error); - let count = inventory.operator_nodes_of_candidates_for_memo_groups.len(); - let mut internal = vec![false; count]; - for inputs in &inventory.producer_nodes_per_input { - for input in inputs { - for producer in input { - internal[producer.0] = true; - } - } - } - let mut outputs = vec![Vec::new(); count]; - for (output, root) in inventory.query_output_nodes.iter().enumerate() { - for id in root { - outputs[id.0].push(output); - } - } - let mut classes = HashMap::, usize>::new(); - let mut representative: Vec<_> = (0..count).collect(); - let mut eligible = vec![false; count]; - for id in 0..count { - let node = &inventory.operator_nodes_of_candidates_for_memo_groups[id]; - if internal[id] - || outputs[id].is_empty() - || inventory.producer_nodes_per_input[id].len() != 1 - { - continue; - } - let template = &node.physical_node; - if !matches!( - template.asap(), - Some(ASAPOp::SummaryEstimate { .. } | ASAPOp::FinalizeExactAccumulator { .. }) - ) { - continue; - } - let mut inputs = Vec::new(); - let operator = template.operator.map_children(|child| { - inputs.push(child.clone()); - 0u8 - }); - let [input] = inputs.as_slice() else { - continue; - }; - eligible[id] = true; - let key = rmp_serde::to_vec(&( - node.group().0, - &outputs[id], - operator, - &template.schema, - template.result_kind, - &template.guarantee, - template.timing, - template.materialization, - &input.schema, - input.result_kind, - &input.guarantee, - input.timing, - input.materialization, - )) - .map_err(|error| invalid(error.to_string()))?; - if let Some(&previous) = classes.get(&key) { - representative[id] = previous; - } else { - classes.insert(key, id); - } - } - let removed = representative - .iter() - .enumerate() - .filter(|(id, rep)| *id != **rep) - .count(); - if removed == 0 { - return Ok(0); - } - let mut unions = HashMap::>::new(); - let mut known = HashMap::>::new(); - let mut candidates_for_memo_group = HashMap::>::new(); - let mut accuracy_budget_parts = HashMap::::new(); - for (id, &rep) in representative.iter().enumerate() { - if eligible[id] { - let producer_candidates = unions.entry(rep).or_default(); - let known = known.entry(rep).or_default(); - for &producer_id in &inventory.producer_nodes_per_input[id][0] { - if known.insert(producer_id) { - producer_candidates.push(producer_id); - } - } - } - if id != rep { - let factored = &inventory.operator_nodes_of_candidates_for_memo_groups[id]; - candidates_for_memo_group - .entry(rep) - .or_default() - .extend(&factored.candidates_for_memo_group); - let parts = accuracy_budget_parts.entry(rep).or_insert(0); - *parts = (*parts).max(factored.accuracy_budget_parts); - } - } - let mut new_ids = vec![PhysicalCandidateId(0); count]; - let mut next = 0; - for (id, &rep) in representative.iter().enumerate() { - if id == rep { - new_ids[id] = PhysicalCandidateId(next); - next += 1; - } - } - for (id, &rep) in representative.iter().enumerate() { - new_ids[id] = new_ids[rep]; - } - let mut lists = ProducerSetInterner::default(); - let mut nodes = Vec::with_capacity(next); - let mut producer_nodes_per_input = Vec::with_capacity(next); - for (id, node) in inventory - .operator_nodes_of_candidates_for_memo_groups - .iter() - .enumerate() - { - if representative[id] != id { - continue; - } - let mut node = node.clone(); - if let Some(factored) = candidates_for_memo_group.remove(&id) { - node.candidates_for_memo_group.extend(factored); - node.candidates_for_memo_group.sort_unstable(); - node.candidates_for_memo_group.dedup(); - } - if let Some(parts) = accuracy_budget_parts.remove(&id) { - node.accuracy_budget_parts = node.accuracy_budget_parts.max(parts); - } - let mut inputs = inventory.producer_nodes_per_input[id].clone(); - if let Some(producer_candidates) = unions.remove(&id) { - inputs[0] = producer_candidates.into(); - } - for input in &mut inputs { - *input = lists.intern(input.iter().map(|id| new_ids[id.0]).collect::>()); - } - nodes.push(node); - producer_nodes_per_input.push(inputs); - } - let remap = |ids: &mut Vec| { - let mut seen = std::collections::HashSet::new(); - *ids = ids - .iter() - .map(|id| new_ids[id.0]) - .filter(|id| seen.insert(*id)) - .collect(); - }; - for root in &mut inventory.query_output_nodes { - remap(root); - } - for group in &mut inventory.implementation_roots_of_group { - remap(group); - } - inventory.operator_nodes_of_candidates_for_memo_groups = nodes; - inventory.producer_nodes_per_input = producer_nodes_per_input; - inventory.validate().map_err(invalid)?; - Ok(removed) -} diff --git a/crates/physical-asap-optimization/src/catalog/inventory.rs b/crates/physical-asap-optimization/src/catalog/inventory.rs deleted file mode 100644 index cc8daf559..000000000 --- a/crates/physical-asap-optimization/src/catalog/inventory.rs +++ /dev/null @@ -1,346 +0,0 @@ -//! Lower the logical candidate memo into typed immediate operator nodes and -//! the nodes that may produce each of their inputs. -//! -//! Consumer-dependent demand, composed accuracy, and physical materialization -//! are handled by later configuration and evaluation stages. Private exact -//! operators are lowered before decomposition; the resulting nodes still need -//! Stage 2 validation before execution. Nothing is priced. - -use std::collections::{HashMap, HashSet}; -use std::rc::Rc; - -use asap_types::ir::OperatorNode; -use asap_types::physical::Materialization; -use asap_types::stages::Memo; -use asap_types::stages::MemoGroupId; - -use super::factoring::{MemoGroupOutputKey, MemoGroupOutputToCompatibleProducersCache}; -use asap_types::stages::{ - CandidateForMemoGroupId, OperatorNodeOfCandidateForMemoGroup, PhysicalCandidateId, - ProducerCandidates, -}; - -/// The memo's implementation candidates lowered to operator nodes, before -/// timing and materialization. Node IDs index -/// `operator_nodes_of_candidates_for_memo_groups`; they become -/// [`asap_types::stages::OperatorNodeOfCandidateForMemoGroupId`]s of the -/// configured candidates. -#[derive(Debug)] -pub struct CandidateInventory { - /// One per operator node: each implementation root, then the private - /// nodes below it. - pub operator_nodes_of_candidates_for_memo_groups: Vec, - /// Per node, per input (in `map_children` order): the nodes that may - /// produce it (a private child, or the input group's compatible roots). - pub producer_nodes_per_input: Vec>, - /// Per output reference: the roots of its group whose output matches the - /// group's logical node. - pub query_output_nodes: Vec>, - /// Per group: its implementation roots, excluding each implementation's - /// private nodes. - pub implementation_roots_of_group: Vec>, -} - -impl CandidateInventory { - /// Every producer and query output names an existing node, once per list. - pub fn validate(&self) -> Result<(), String> { - let count = self.operator_nodes_of_candidates_for_memo_groups.len(); - if self.producer_nodes_per_input.len() != count { - return Err("producer lists are not indexed by node".into()); - } - for (node, inputs) in self.producer_nodes_per_input.iter().enumerate() { - for (input, producer_candidates) in inputs.iter().enumerate() { - super::validate_ids( - producer_candidates, - count, - &format!("candidate {node}, requirement {input}"), - )?; - } - } - for (output, nodes) in self.query_output_nodes.iter().enumerate() { - super::validate_ids(nodes, count, &format!("query root {output}"))?; - } - Ok(()) - } -} - -#[derive(Debug, thiserror::Error)] -pub enum CandidateInventoryError { - #[error("invalid candidate memo: {0}")] - InvalidMemo(String), - #[error("unsupported candidate {candidate} in group {group}: {reason}")] - Unsupported { - group: usize, - candidate: usize, - reason: &'static str, - }, - #[error( - "physical operator implementation for candidate {candidate} in group {group}: {reason}" - )] - PhysicalImplementation { - group: usize, - candidate: usize, - reason: String, - }, -} - -/// The inventory with identical terminal readers factored into one node -/// (whose `candidates_for_memo_group` lists every candidate it can implement). -pub fn build_candidate_inventory( - memo: &Memo, - output_groups: &[MemoGroupId], -) -> Result { - let mut inventory = build_with_compatible_producers_cache( - memo, - output_groups, - &mut MemoGroupOutputToCompatibleProducersCache::default(), - )?; - super::factoring::factor_terminal_readers(&mut inventory)?; - Ok(inventory) -} - -pub fn build_with_compatible_producers_cache( - memo: &Memo, - output_groups: &[MemoGroupId], - compatible_producers_cache: &mut MemoGroupOutputToCompatibleProducersCache, -) -> Result { - let invalid = |message: &str| CandidateInventoryError::InvalidMemo(message.into()); - if memo.groups.len() != memo.candidates.len() { - return Err(invalid("group and candidate vector lengths differ")); - } - if output_groups - .iter() - .any(|group| group.0 as usize >= memo.groups.len()) - { - return Err(invalid("query output references missing group")); - } - let mut inventory = CandidateInventory { - operator_nodes_of_candidates_for_memo_groups: Vec::new(), - producer_nodes_per_input: Vec::new(), - query_output_nodes: Vec::new(), - implementation_roots_of_group: vec![Vec::new(); memo.groups.len()], - }; - for (group, candidates) in memo.candidates.iter().enumerate() { - for (index, candidate) in candidates.iter().enumerate() { - let unsupported = |reason| CandidateInventoryError::Unsupported { - group, - candidate: index, - reason, - }; - let implemented: HashSet<_> = std::iter::once(MemoGroupId(group as u32)) - .chain(candidate.fused_groups.iter().copied()) - .collect(); - if implemented.len() != 1 + candidate.fused_groups.len() - || implemented - .iter() - .any(|id| id.0 as usize >= memo.groups.len()) - { - return Err(invalid( - "candidate names duplicate or missing absorbed groups", - )); - } - // Only the owner group is this candidate's output contract. Its fused - // groups describe absorbed computations inside its private subtree, - // not additional results it can produce. Private-node decomposition - // charges that work; independent consumers of an absorbed group - // still select one of that group's own candidates. Reverse support - // leaves unused absorbed groups inactive without exclusion rules. - if candidate.accuracy_budget_parts == 0 { - return Err(unsupported("zero accuracy budget parts")); - } - if candidate - .input_groups - .iter() - .any(|child| child.0 as usize >= memo.groups.len()) - { - return Err(invalid("candidate references missing input group")); - } - let inputs = candidate - .input_groups - .iter() - .map(|child| Rc::clone(&memo.groups[child.0 as usize].logical_node)) - .collect::>(); - if inputs - .iter() - .any(|input| Rc::ptr_eq(input, &candidate.root_node)) - { - return Err(unsupported( - "root aliases an input group without an operator", - )); - } - let implemented = crate::implement_candidate_operators(&candidate.root_node, &inputs) - .map_err(|error| CandidateInventoryError::PhysicalImplementation { - group, - candidate: index, - reason: error.to_string(), - })?; - let id = - PhysicalCandidateId(inventory.operator_nodes_of_candidates_for_memo_groups.len()); - inventory.implementation_roots_of_group[group].push(id); - inventory.operator_nodes_of_candidates_for_memo_groups.push( - OperatorNodeOfCandidateForMemoGroup { - candidates_for_memo_group: vec![CandidateForMemoGroupId { - group: MemoGroupId(group as u32), - index, - }], - is_implementation_root: true, - accuracy_budget_parts: candidate.accuracy_budget_parts, - physical_node: implemented, - }, - ); - inventory.producer_nodes_per_input.push(Vec::new()); - } - } - for (group, candidates) in memo.candidates.iter().enumerate() { - for (index, candidate) in candidates.iter().enumerate() { - let mut boundaries = HashMap::new(); - for &child in &candidate.input_groups { - let pointer = Rc::as_ptr(&memo.groups[child.0 as usize].logical_node); - if boundaries - .insert(pointer, child) - .is_some_and(|previous| previous != child) - { - return Err(invalid("one boundary node names different input groups")); - } - } - if boundaries.contains_key(&Rc::as_ptr(&candidate.root_node)) { - return Err(CandidateInventoryError::Unsupported { - group, - candidate: index, - reason: "root aliases an input group without an operator", - }); - } - let mut builder = ImmediateCandidateBuilder { - inventory: &mut inventory, - compatible_producers_cache: &mut *compatible_producers_cache, - boundaries, - private_nodes: HashMap::new(), - used_boundaries: HashSet::new(), - owner: CandidateForMemoGroupId { - group: MemoGroupId(group as u32), - index, - }, - owner_accuracy_budget_parts: candidate.accuracy_budget_parts, - }; - let root_id = builder.inventory.implementation_roots_of_group[group][index]; - let implemented = Rc::clone( - &builder - .inventory - .operator_nodes_of_candidates_for_memo_groups[root_id.0] - .physical_node, - ); - builder.add(&implemented, Some(root_id))?; - if candidate - .input_groups - .iter() - .any(|group| !builder.used_boundaries.contains(group)) - { - return Err(invalid( - "declared input group has no canonical node boundary", - )); - } - } - } - for group in output_groups { - let expected = &memo.groups[group.0 as usize].logical_node; - let mut compatible_candidates = Vec::new(); - for &id in &inventory.implementation_roots_of_group[group.0 as usize] { - let template = - &inventory.operator_nodes_of_candidates_for_memo_groups[id.0].physical_node; - if compatible_output(expected, template)? { - compatible_candidates.push(id); - } - } - inventory.query_output_nodes.push(compatible_candidates); - } - inventory.validate().map_err(|error| invalid(&error))?; - Ok(inventory) -} - -struct ImmediateCandidateBuilder<'a> { - inventory: &'a mut CandidateInventory, - compatible_producers_cache: &'a mut MemoGroupOutputToCompatibleProducersCache, - boundaries: HashMap<*const OperatorNode, MemoGroupId>, - private_nodes: HashMap<*const OperatorNode, PhysicalCandidateId>, - used_boundaries: HashSet, - /// The memo candidate being lowered; owner of every node it adds. - owner: CandidateForMemoGroupId, - /// The owner's `accuracy_budget_parts`, which its private nodes carry. - owner_accuracy_budget_parts: usize, -} - -impl ImmediateCandidateBuilder<'_> { - fn add( - &mut self, - node: &Rc, - reserved: Option, - ) -> Result { - if let Some(&existing) = self.private_nodes.get(&Rc::as_ptr(node)) { - return Ok(existing); - } - if node.timing.is_some() || node.materialization != Materialization::NotMaterialized { - return Err(CandidateInventoryError::Unsupported { - group: self.owner.group.0 as usize, - candidate: self.owner.index, - reason: "assigned execution timing or materialization", - }); - } - let mut requirements = Vec::new(); - for input in super::reconstruction::ordered_inputs(node) { - let producers = if let Some(&group) = self.boundaries.get(&Rc::as_ptr(&input)) { - self.used_boundaries.insert(group); - let key = MemoGroupOutputKey::new(group, &input) - .map_err(CandidateInventoryError::InvalidMemo)?; - if let Some(compatible) = self.compatible_producers_cache.get(&key) { - compatible - } else { - let mut compatible = Vec::new(); - for &candidate in - &self.inventory.implementation_roots_of_group[group.0 as usize] - { - let template = &self.inventory.operator_nodes_of_candidates_for_memo_groups - [candidate.0] - .physical_node; - if compatible_output(&input, template)? { - compatible.push(candidate); - } - } - self.compatible_producers_cache.remember(key, compatible) - } - } else { - vec![self.add(&input, None)?].into() - }; - requirements.push(producers); - } - let id = if let Some(id) = reserved { - self.inventory.producer_nodes_per_input[id.0] = requirements; - id - } else { - let id = PhysicalCandidateId( - self.inventory - .operator_nodes_of_candidates_for_memo_groups - .len(), - ); - self.inventory.producer_nodes_per_input.push(requirements); - self.inventory - .operator_nodes_of_candidates_for_memo_groups - .push(OperatorNodeOfCandidateForMemoGroup { - candidates_for_memo_group: vec![self.owner], - is_implementation_root: false, - accuracy_budget_parts: self.owner_accuracy_budget_parts, - physical_node: Rc::clone(node), - }); - id - }; - self.private_nodes.insert(Rc::as_ptr(node), id); - Ok(id) - } -} - -fn compatible_output( - expected: &OperatorNode, - producer: &OperatorNode, -) -> Result { - super::reconstruction::output_contract_satisfied(expected, producer) - .map_err(CandidateInventoryError::InvalidMemo) -} diff --git a/crates/physical-asap-optimization/src/catalog/mod.rs b/crates/physical-asap-optimization/src/catalog/mod.rs index eb3069ead..d278afef4 100644 --- a/crates/physical-asap-optimization/src/catalog/mod.rs +++ b/crates/physical-asap-optimization/src/catalog/mod.rs @@ -1,48 +1,32 @@ -//! Stage 2's physical candidates: implement each memo candidate's -//! operators, enumerate its timing and materialization, record which -//! candidates can produce each input, and prune what cannot be used +//! Stage 2's physical candidates: expand each logical candidate into one +//! physical candidate per valid $(\tau, \ell, \sigma)$, and each logical edge +//! into the physical edges between the two candidates' alternatives //! ([`generate_physical_asap_memo_candidates`]); and realize one selected -//! assignment as a physical plan (#1047). The output type, +//! DAG as a physical plan (#1047). The output type, //! `PhysicalASAPMemoCandidates`, lives in `asap_types::stages`; Stage 3 //! selects from it. -mod configurations; -mod factoring; -mod inventory; -mod pruning; mod realize; -mod reconstruction; -pub use configurations::ConfigurationError; -pub use inventory::CandidateInventoryError; pub use realize::Stage2AssignmentRealizer; -use asap_types::stages::{ - LogicalASAPMemoCandidates, PhysicalASAPMemoCandidates, PhysicalCandidateId, ReaderConstraint, -}; +use asap_types::stages::{LogicalASAPMemoCandidates, PhysicalASAPMemoCandidates}; use asap_types::workload::{DataWorkload, QueryWorkloadEntry}; /// Why [`generate_physical_asap_memo_candidates`] failed. #[derive(Debug, thiserror::Error)] pub enum ErrorGeneratingPhysicalASAPMemoCandidates { - /// Lowering the memo's candidates to operator nodes failed. - #[error(transparent)] - Inventory(#[from] CandidateInventoryError), - /// Enumerating timings and materializations failed. - #[error(transparent)] - Configuration(#[from] ConfigurationError), /// Structural pruning found the candidates malformed. #[error("{0}")] Pruning(String), } -/// Stage 2: implement every operator node of every candidate in `logical`'s -/// memo, enumerate the timings and materializations the data and deployment -/// allow (`query_time_results_retained`; no query-time raw scans unless -/// `raw_data_retained`; no exact per-entity aggregate the executor cannot -/// run), record which candidates may produce each input, and prune what -/// cannot be used. Every configuration not generated and every candidate -/// pruned is recorded in the output's `pruned_candidates` with its -/// `PruneReason`. Prices nothing. +/// Stage 2: expand every logical candidate of `logical` into its physical +/// alternatives (decision-tree levels 4–6: timing, lifetime, schedule) the +/// data and deployment allow (`query_time_results_retained`; no query-time +/// raw scans unless `raw_data_retained`), and every logical edge into its +/// physical edges with $\alpha^{\log}$ and $\alpha^{\mathrm{phys}}$. Every +/// alternative not generated is recorded in the output's +/// `pruned_candidates` with its `PruneReason`. Prices nothing. pub fn generate_physical_asap_memo_candidates( logical: &LogicalASAPMemoCandidates, entries: &[QueryWorkloadEntry], @@ -50,114 +34,12 @@ pub fn generate_physical_asap_memo_candidates( query_time_results_retained: bool, raw_data_retained: bool, ) -> Result { - let memo = &logical.memo; - let output_groups: Vec<_> = asap_types::stages::output_references(&memo.query_roots) - .into_iter() - .map(|(_, group)| group) - .collect(); - let inventory = inventory::build_candidate_inventory(memo, &output_groups)?; - let configured = configurations::enumerate_candidate_configurations( - &inventory, - &memo.query_roots, + let _ = ( + logical, entries, data, query_time_results_retained, raw_data_retained, - )?; - pruning::prune(configured).map_err(ErrorGeneratingPhysicalASAPMemoCandidates::Pruning) -} - -/// Each output reference's candidates ([`PhysicalASAPMemoCandidates::query_output_candidates`]). -fn query_output_candidates(physical: &PhysicalASAPMemoCandidates) -> Vec> { - (0..physical.output_reference_count()) - .map(|output| physical.query_output_candidates(output).collect()) - .collect() -} - -/// `ids` name existing candidates (of `count`), each once. -fn validate_ids(ids: &[PhysicalCandidateId], count: usize, context: &str) -> Result<(), String> { - let mut seen = std::collections::HashSet::new(); - for id in ids { - if id.0 >= count { - return Err(format!("{context} references missing candidate {}", id.0)); - } - if !seen.insert(*id) { - return Err(format!("{context} repeats candidate {}", id.0)); - } - } - Ok(()) -} - -/// Every producer, operator node and reader constraint of `physical` names an -/// existing candidate or output reference, and each query reader constraint -/// partitions the output references. -fn validate_candidates(physical: &PhysicalASAPMemoCandidates) -> Result<(), String> { - let count = physical.candidates.len(); - let outputs_count = physical.output_reference_count(); - let invalid = |message: &str| format!("invalid physical ASAP memo candidates: {message}"); - for constraint in &physical.reader_constraints { - let outputs = match constraint { - ReaderConstraint::MinimumQueryReaders { query_outputs, .. } - | ReaderConstraint::HighestSharedQueryMaterialization { query_outputs, .. } => { - let outputs: Vec<_> = query_outputs.iter().flatten().copied().collect(); - let unique: std::collections::HashSet<_> = outputs.iter().copied().collect(); - if outputs.len() != outputs_count || unique.len() != outputs.len() { - return Err(invalid( - "query reader buckets must partition every output exactly once", - )); - } - outputs - } - ReaderConstraint::AllowedOutputs { outputs, .. } => outputs.clone(), - ReaderConstraint::IncompatibleOutputs { left, right, .. } => vec![*left, *right], - }; - if constraint.candidate().0 >= count - || outputs.iter().any(|&output| output >= outputs_count) - { - return Err(invalid( - "reader constraint references missing candidate or output", - )); - } - } - for (index, candidate) in physical.candidates.iter().enumerate() { - if candidate.operator_node.0 >= physical.operator_nodes_of_candidates_for_memo_groups.len() - { - return Err(invalid(&format!( - "candidate {index} references missing operator node" - ))); - } - for (input, producer_candidates) in - candidate.producer_candidates_per_input.iter().enumerate() - { - validate_ids( - producer_candidates, - count, - &format!("candidate {index}, requirement {input}"), - ) - .map_err(|message| invalid(&message))?; - } - } - Ok(()) -} - -/// Catalog internals that plan selection's unit tests exercise together with -/// its solver. -#[cfg(feature = "test-support")] -pub mod test_support { - pub use super::configurations::{ - compatible_timing, enumerate_candidate_configurations, - may_keep_across_evaluations_per_sole_reader_query, - }; - pub use super::factoring::{ - factor_terminal_readers, MemoGroupOutputKey, MemoGroupOutputToCompatibleProducersCache, - }; - pub use super::inventory::{ - build_candidate_inventory, build_with_compatible_producers_cache, CandidateInventory, - }; - pub use super::pruning::usable_candidates; - pub use super::realize::{materialize_selected_candidates, restore_query_roots}; - pub use super::reconstruction::{ - ordered_inputs, output_contract_satisfied, paired_node_identities, - reconstruct_selected_candidates, ReconstructedCandidates, - }; + ); + todo!("#1207 PR 5: physical expansion and physical edges") } diff --git a/crates/physical-asap-optimization/src/catalog/pruning.rs b/crates/physical-asap-optimization/src/catalog/pruning.rs deleted file mode 100644 index 4cde4cb9e..000000000 --- a/crates/physical-asap-optimization/src/catalog/pruning.rs +++ /dev/null @@ -1,195 +0,0 @@ -//! Remove provably unusable configurations before constructing solver variables. -//! No costs or dominance assumptions are involved: finite input realizability, -//! potential output reach and necessary reader counts are sufficient proofs. -use asap_types::stages::{ - PhysicalASAPMemoCandidates, PhysicalCandidateId, PhysicalCandidateKey, PruneReason, - PrunedCandidate, ReaderConstraint, -}; -use asap_types::stages::{ProducerCandidates, ProducerSetInterner}; -use std::collections::HashMap; - -/// `configured` without the candidates `prune_reasons` rules out, its -/// IDs renumbered densely in their order; each removed one is appended to -/// its `pruned_candidates` with the reason. `configured` is already -/// validated: configuration enumeration ends by validating its output. -pub fn prune(configured: PhysicalASAPMemoCandidates) -> Result { - let count = configured.candidates.len(); - let reasons = prune_reasons(&configured); - let keep: Vec = reasons.iter().map(Option::is_none).collect(); - let mut pruned_candidates = configured.pruned_candidates.clone(); - for (candidate, reason) in configured.candidates.iter().zip(&reasons) { - if let Some(reason) = *reason { - pruned_candidates.push(PrunedCandidate { - group: configured.operator_nodes_of_candidates_for_memo_groups - [candidate.operator_node.0] - .group(), - candidate: PhysicalCandidateKey { - operator_node: candidate.operator_node, - timing: candidate.timing, - materialization: candidate.materialization, - }, - reason, - }); - } - } - let mut mapping = vec![None; count]; - let mut next = 0; - for (id, retained) in keep.iter().enumerate() { - if *retained { - mapping[id] = Some(PhysicalCandidateId(next)); - next += 1; - } - } - let mut producer_sets = ProducerSetInterner::default(); - let mut remapped: HashMap = HashMap::new(); - let candidates = configured - .candidates - .into_iter() - .enumerate() - .filter_map(|(id, mut candidate)| { - if !keep[id] { - return None; - } - for input in &mut candidate.producer_candidates_per_input { - *input = remapped - .entry(input.clone()) - .or_insert_with(|| { - producer_sets.intern( - input - .iter() - .filter_map(|id| mapping[id.0]) - .collect::>(), - ) - }) - .clone(); - } - Some(candidate) - }) - .collect(); - let reader_constraints = configured - .reader_constraints - .into_iter() - .filter_map(|mut constraint| { - let id = constraint.candidate_mut(); - *id = mapping[id.0]?; - Some(constraint) - }) - .collect(); - // Operator nodes are kept whole: `operator_node` indexes them unchanged. - let mut pruned = PhysicalASAPMemoCandidates::new( - configured.operator_nodes_of_candidates_for_memo_groups, - candidates, - configured.query_roots, - reader_constraints, - ); - pruned.pruned_candidates = pruned_candidates; - super::validate_candidates(&pruned)?; - Ok(pruned) -} - -/// Which candidates of `physical` [`prune`] keeps. -#[cfg(feature = "test-support")] -pub fn usable_candidates(physical: &PhysicalASAPMemoCandidates) -> Vec { - prune_reasons(physical) - .iter() - .map(Option::is_none) - .collect() -} - -/// Why each candidate of `physical` cannot be used, `None` for a usable -/// one: the reason of the first round that ruled it out, and within a -/// round the first of: an input with no buildable producer, no query output -/// reached, too few query readers. -fn prune_reasons(physical: &PhysicalASAPMemoCandidates) -> Vec> { - let count = physical.candidates.len(); - let query_output_candidates = super::query_output_candidates(physical); - let mut reasons: Vec> = vec![None; count]; - loop { - let enabled: Vec = reasons.iter().map(Option::is_none).collect(); - // Least fixed point: a cycle without a finite implementation is not a - // producer. A cycle with an independent escape candidate remains usable. - let mut realizable = vec![false; count]; - loop { - let mut changed = false; - for (id, candidate) in physical.candidates.iter().enumerate() { - if enabled[id] - && !realizable[id] - && candidate - .producer_candidates_per_input - .iter() - .all(|input| input.iter().any(|id| realizable[id.0])) - { - realizable[id] = true; - changed = true; - } - } - if !changed { - break; - } - } - let mut reached = vec![vec![false; query_output_candidates.len()]; count]; - for (output, root) in query_output_candidates.iter().enumerate() { - let mut pending = root.clone(); - while let Some(id) = pending.pop() { - if realizable[id.0] && !reached[id.0][output] { - reached[id.0][output] = true; - pending.extend( - physical.candidates[id.0] - .producer_candidates_per_input - .iter() - .flat_map(|input| input.iter().copied()), - ); - } - } - } - // Intersect all allowed-reader restrictions before counting potential - // distinct queries. Potential reach is an upper bound, never a demand. - for constraint in &physical.reader_constraints { - if let ReaderConstraint::AllowedOutputs { candidate, outputs } = constraint { - for (output, value) in reached[candidate.0].iter_mut().enumerate() { - *value &= outputs.contains(&output); - } - } - } - let mut round: Vec> = realizable - .iter() - .zip(&reached) - .zip(&enabled) - .map(|((realizable, outputs), enabled)| { - if !enabled || *realizable && outputs.iter().any(|value| *value) { - None - } else if !realizable { - Some(PruneReason::InputHasNoBuildableProducer) - } else { - Some(PruneReason::ReachesNoQueryOutput) - } - }) - .collect(); - for constraint in &physical.reader_constraints { - if let ReaderConstraint::MinimumQueryReaders { - candidate, - query_outputs, - minimum, - } = constraint - { - let queries = query_outputs - .iter() - .filter(|outputs| outputs.iter().any(|output| reached[candidate.0][*output])) - .count(); - if queries < *minimum { - round[candidate.0].get_or_insert(PruneReason::TooFewQueryReaders); - } - } - } - let mut changed = false; - for (reason, round) in reasons.iter_mut().zip(round) { - if reason.is_none() && round.is_some() { - *reason = round; - changed = true; - } - } - if !changed { - return reasons; - } - } -} diff --git a/crates/physical-asap-optimization/src/catalog/realize.rs b/crates/physical-asap-optimization/src/catalog/realize.rs index f473709c1..9abbc6b8b 100644 --- a/crates/physical-asap-optimization/src/catalog/realize.rs +++ b/crates/physical-asap-optimization/src/catalog/realize.rs @@ -1,140 +1,20 @@ -//! Apply an explicit selected materialization assignment through real Stage 2. -//! -//! No fallback modes are chosen. Physical rewrites or timing splits that change -//! candidate identity are rejected until represented by physical enumeration. -//! This validates execution legality, not cost estimates or accuracy targets. -use std::collections::HashMap; -use std::rc::Rc; - -use crate::optimize_physical_asap_chosen; -use asap_types::ir::{OperatorNode, QueryRoot}; -use asap_types::physical::Materialization; -use asap_types::workload::{DataWorkload, QueryWorkloadEntry}; +//! Realize a selected DAG through Stage 2: each node's physical properties +//! come from its `PhysicalCandidate` (concepts doc, section 3, +//! "Realizer/export"). This validates execution legality, not cost estimates +//! or accuracy targets. +use asap_types::ir::QueryRoot; use asap_types::stages::{ AssignmentRealizer, DagOfSelectedCandidates, PhysicalASAPMemoCandidates, PhysicalCandidateError, RealizedPlan, }; - -use super::reconstruction::{ - output_contract_satisfied, paired_node_identities, reconstruct_selected_candidates, -}; - -/// Realize `selected` with each selected node at its candidate's -/// materialization. Output -/// references must follow `map_operator_refs` traversal of the original -/// roots in workload order (one reference for an operator root). Repeated -/// scalar references remain separate positions; literal-only scalar roots -/// use none. -pub fn materialize_selected_candidates( - physical_candidates: &PhysicalASAPMemoCandidates, - selected: &DagOfSelectedCandidates, - original_roots: &[QueryRoot], - entries: &[QueryWorkloadEntry], - data: &DataWorkload, - query_time_results_retained: bool, -) -> Result { - if entries.len() != original_roots.len() { - return Err("workload entries and query roots differ in length".into()); - } - let rebuilt = reconstruct_selected_candidates(physical_candidates, selected)?; - // `reconstruct_selected_candidates` checked every selected candidate exists. - let materializations: Vec<_> = selected - .candidate_of_node - .iter() - .map(|id| physical_candidates.candidates[id.0].materialization) - .collect(); - let mut node_by_pointer = HashMap::new(); - for (node, operator_node) in rebuilt.operator_node_of_selected_node.iter().enumerate() { - node_by_pointer.insert(Rc::as_ptr(operator_node), node); - } - let roots = restore_query_roots(original_roots, &rebuilt.outputs)?; - let mut assignment_error = None; - let mut seen = vec![false; materializations.len()]; - let physical = optimize_physical_asap_chosen( - "selected-candidate-connections", - &roots, - entries, - data, - query_time_results_retained, - &mut |logical, _, options| { - let node = node_by_pointer.get(&logical).copied(); - let requested = if let Some(node) = node { - seen[node] = true; - materializations[node] - } else { - Materialization::NotMaterialized - }; - if !options.contains(&requested) { - let message = format!( - "selected materialization {requested:?} is not offered for selected node {node:?}" - ); - assignment_error = Some(if node.is_some() { - PhysicalCandidateError::RejectedAssignment(message) - } else { - PhysicalCandidateError::InvalidOrUnsupported(message) - }); - return None; - } - Some(requested) - }, - ) - .map_err(|error| error.to_string())?; - if let Some(error) = assignment_error { - return Err(error); - } - let physical = physical.ok_or_else(|| { - PhysicalCandidateError::RejectedAssignment( - "selected materialization assignment violates Stage 2 timing or dependency rules" - .into(), - ) - })?; - physical.dag.validate().map_err(|error| error.to_string())?; - let physical_outputs = ordered_output_references(&physical.roots); - let identities = paired_node_identities(&rebuilt.outputs, &physical_outputs)?; - for (node, mode) in materializations.iter().enumerate() { - if *mode != Materialization::NotMaterialized && !seen[node] { - // The decoder already established node identity. Stage 2 did - // not offer this selected node (e.g. a lower shared non-summary), - // so the assignment is infeasible rather than an adapter failure. - return Err(PhysicalCandidateError::RejectedAssignment(format!( - "selected node {node} is not an eligible materialization node in the selected graph" - ))); - } - } - let operator_node_of_selected_node = rebuilt - .operator_node_of_selected_node - .into_iter() - .enumerate() - .map(|(node, operator_node)| { - let physical = identities - .get(&Rc::as_ptr(&operator_node)) - .cloned() - .ok_or_else(|| format!("selected node {node} has no physical identity"))?; - if physical.materialization != materializations[node] { - return Err(format!( - "selected node {node} materialization changed during physical reconstruction" - )); - } - Ok(physical) - }) - .collect::, String>>()?; - let range_evaluations: Vec<_> = entries.iter().map(|entry| entry.range_evaluation).collect(); - let tumbling_window_retention_ms = - asap_cost::tumbling_window_retention_ms_of_plan(&physical.dag, &range_evaluations); - Ok(RealizedPlan { - plan: physical, - tumbling_window_retention_ms, - operator_node_of_selected_node, - }) -} +use asap_types::workload::{DataWorkload, QueryWorkloadEntry}; /// Stage 2's [`AssignmentRealizer`], over the planning context: the /// workload's original query roots (one per entry of `entries`), its data, /// and whether the deployment keeps query-time results across /// evaluations. The facade builds it, passes it to Stage 3, and builds the /// selected plan with it. -#[derive(Debug, Clone, Copy)] pub struct Stage2AssignmentRealizer<'a> { pub original_roots: &'a [QueryRoot], pub entries: &'a [QueryWorkloadEntry], @@ -143,183 +23,12 @@ pub struct Stage2AssignmentRealizer<'a> { } impl AssignmentRealizer for Stage2AssignmentRealizer<'_> { - /// `materialize_selected_candidates`; each node also keeps its - /// candidate's timing. fn realize( &self, physical: &PhysicalASAPMemoCandidates, selected: &DagOfSelectedCandidates, ) -> Result { - let realized = materialize_selected_candidates( - physical, - selected, - self.original_roots, - self.entries, - self.data, - self.query_time_results_retained, - )?; - for (id, node) in selected - .candidate_of_node - .iter() - .zip(&realized.operator_node_of_selected_node) - { - let timing = physical.candidates[id.0].timing; - if node.timing != Some(timing) { - return Err(PhysicalCandidateError::InvalidOrUnsupported(format!( - "candidate {} requested timing {timing:?}, Stage 2 produced {:?}", - id.0, node.timing - ))); - } - } - Ok(realized) - } -} - -fn ordered_output_references(roots: &[QueryRoot]) -> Vec> { - asap_types::stages::output_references(roots) - .into_iter() - .map(|(_, node)| node) - .collect() -} - -pub fn restore_query_roots( - original: &[QueryRoot], - outputs: &[Rc], -) -> Result, String> { - let references = ordered_output_references(original); - if references.len() != outputs.len() { - return Err("output references do not match original query-root mapping".into()); - } - for (reference, output) in references.iter().zip(outputs) { - if !output_contract_satisfied(reference, output)? { - return Err(format!( - "selected output reference has incompatible result or schema: expected {:?} {:?}, received {:?} {:?}", - reference.result_kind, reference.schema, output.result_kind, output.schema - )); - } - } - let mut next = outputs.iter(); - Ok(original - .iter() - .map(|root| match root { - QueryRoot::Operator(_) => QueryRoot::Operator(Rc::clone(next.next().unwrap())), - QueryRoot::Scalar(expression) => QueryRoot::Scalar( - expression.map_operator_refs(&mut |_| Rc::clone(next.next().unwrap())), - ), - }) - .collect()) -} - -#[cfg(test)] -mod tests { - use super::*; - use asap_logical_asap_optimization::generate_dag_of_logical_memo_groups; - use asap_logical_asap_optimization::test_support::generate_per_computation_candidates; - use asap_logical_asap_optimization::test_support::try_populate_pass1; - use asap_logical_asap_optimization::test_support::{ - entry, lower_promql, sharing_rules_and_shared_state_groups, - }; - use asap_types::ir::properties::AccuracyTarget; - use asap_types::stages::{ - LogicalASAPMemoCandidates, PhysicalCandidateId, SelectedEdge, SelectedNodeId, - }; - use asap_types::workload::QueryRecurrence; - - /// Add a node running `candidate` and, for each input, a node of its - /// first unmaterialized producer (else its first producer), - /// recursively; returns the new node. - fn select_from( - physical: &PhysicalASAPMemoCandidates, - candidate: PhysicalCandidateId, - selected: &mut DagOfSelectedCandidates, - ) -> SelectedNodeId { - let node = SelectedNodeId(selected.candidate_of_node.len()); - selected.candidate_of_node.push(candidate); - for (input, producer_candidates) in physical.candidates[candidate.0] - .producer_candidates_per_input - .iter() - .enumerate() - { - let producer = producer_candidates - .iter() - .copied() - .find(|s| { - physical.candidates[s.0].materialization == Materialization::NotMaterialized - }) - .unwrap_or(producer_candidates[0]); - let producer = select_from(physical, producer, selected); - selected.edges.push(SelectedEdge { - producer, - consumer: node, - input, - }); - } - node - } - - // The trait object returns what the catalog function returns with each - // candidate's own materialization. - #[test] - fn stage2_realizer_matches_direct_catalog_calls() { - let root = QueryRoot::Operator(lower_promql( - "sum by (job) (sum_over_time(m[1m]))", - AccuracyTarget::exact(), - )); - let roots = vec![root.clone()]; - let initial = generate_dag_of_logical_memo_groups(&[root]); - let per_computation = - generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&[ - roots[0].clone() - ])) - .unwrap(); - let memo = try_populate_pass1(initial, &per_computation).unwrap(); - let logical = LogicalASAPMemoCandidates { - memo: sharing_rules_and_shared_state_groups(memo).unwrap(), - }; - let entries = vec![entry(QueryRecurrence::Unknown)]; - let data = DataWorkload::default(); - let physical = crate::catalog::generate_physical_asap_memo_candidates( - &logical, &entries, &data, false, true, - ) - .unwrap(); - let selected = physical - .query_output_candidates(0) - .map(|candidate| { - let mut selected = DagOfSelectedCandidates { - candidate_of_node: Vec::new(), - edges: Vec::new(), - node_of_output_reference: Vec::new(), - }; - let root = select_from(&physical, candidate, &mut selected); - selected.node_of_output_reference.push(root); - selected - }) - .find(|selected| reconstruct_selected_candidates(&physical, selected).is_ok()) - .expect("some selection reconstructs"); - assert!(selected.candidate_of_node.len() > 1); - let realizer = Stage2AssignmentRealizer { - original_roots: &roots, - entries: &entries, - data: &data, - query_time_results_retained: false, - }; - let realizer: &dyn AssignmentRealizer = &realizer; - - let direct = - materialize_selected_candidates(&physical, &selected, &roots, &entries, &data, false) - .unwrap(); - let through_trait = realizer.realize(&physical, &selected).unwrap(); - assert_eq!( - format!("{:?}", direct.plan), - format!("{:?}", through_trait.plan) - ); - assert_eq!( - direct.operator_node_of_selected_node, - through_trait.operator_node_of_selected_node - ); - assert_eq!( - direct.tumbling_window_retention_ms, - through_trait.tumbling_window_retention_ms - ); + let _ = (physical, selected); + todo!("#1207 PR 6: build the selected DAG, each node at its PhysicalCandidate's properties") } } diff --git a/crates/physical-asap-optimization/src/catalog/reconstruction.rs b/crates/physical-asap-optimization/src/catalog/reconstruction.rs deleted file mode 100644 index fd7f0068b..000000000 --- a/crates/physical-asap-optimization/src/catalog/reconstruction.rs +++ /dev/null @@ -1,231 +0,0 @@ -//! Reconstruct selected immediate operators, preserving candidate identity. -//! -//! This decoder checks typed connections, not Stage 2 execution legality or -//! query accuracy targets. The caller must run Stage 2 and the plan validator. -use std::collections::HashMap; -use std::rc::Rc; - -use asap_accuracy::propagation::recompose_guarantees; -use asap_types::ir::{OperatorNode, QueryRoot}; - -use asap_types::stages::{ - DagOfSelectedCandidates, PhysicalASAPMemoCandidates, PhysicalCandidateId, -}; - -/// Enumerate in exactly the traversal order used when rebinding. `children()` -/// puts direct children before scalar references; `map_children` follows fields. -pub fn ordered_inputs(node: &OperatorNode) -> Vec> { - let mut inputs = Vec::new(); - node.operator.map_children(|child| { - inputs.push(Rc::clone(child)); - Rc::clone(child) - }); - inputs -} - -/// Open schemas do not promise completeness; an equivalent derived output may -/// strengthen that promise to closed. Supplied uniqueness must imply every -/// required key; additional proven keys are allowed. Every field, configuration -/// parameter and time axis remains exact. Serialization includes sketch error/failure -/// metadata deliberately omitted by some parameter PartialEq implementations. -pub fn output_contract_satisfied( - expected: &OperatorNode, - output: &OperatorNode, -) -> Result { - let mut supplied = output.schema.clone(); - if !expected.schema.closed { - supplied.closed = false; - } - if !expected.schema.unique_keys.iter().all(|required| { - supplied - .unique_keys - .iter() - .any(|provided| provided.iter().all(|column| required.contains(column))) - }) { - return Ok(false); - } - supplied.unique_keys = expected.schema.unique_keys.clone(); - Ok(expected.result_kind == output.result_kind - && expected.schema == supplied - && serde_json::to_string(&expected.schema).map_err(|error| error.to_string())? - == serde_json::to_string(&supplied).map_err(|error| error.to_string())?) -} - -pub struct ReconstructedCandidates { - pub outputs: Vec>, - /// The operator node each selected node became, indexed by - /// [`SelectedNodeId`](asap_types::stages::SelectedNodeId), with - /// guarantees recomputed; each a distinct operator node. - pub(crate) operator_node_of_selected_node: Vec>, -} - -/// The nodes of `selected` rebuilt over their selected producers, one -/// operator node each, with guarantees recomposed. Checks the selection's -/// structure, not its cost. -pub fn reconstruct_selected_candidates( - physical: &PhysicalASAPMemoCandidates, - selected: &DagOfSelectedCandidates, -) -> Result { - super::validate_candidates(physical)?; - selected.validate(physical)?; - let count = selected.candidate_of_node.len(); - let mut inputs = selected - .candidate_of_node - .iter() - .map(|candidate| { - vec![ - None; - physical.candidates[candidate.0] - .producer_candidates_per_input - .len() - ] - }) - .collect::>(); - let mut consumers = vec![Vec::new(); count]; - for edge in &selected.edges { - inputs[edge.consumer.0][edge.input] = Some(edge.producer.0); - consumers[edge.producer.0].push(edge.consumer.0); - } - let mut remaining = inputs.iter().map(Vec::len).collect::>(); - let mut ready = Vec::new(); - for (node, candidate) in selected.candidate_of_node.iter().enumerate() { - let template = physical.physical_node(*candidate); - if template.children().len() != inputs[node].len() { - return Err(format!( - "candidate {} template inputs do not match producer requirements", - candidate.0 - )); - } - if remaining[node] == 0 { - ready.push(node); - } - } - let mut rebuilt: Vec>> = vec![None; count]; - while let Some(node) = ready.pop() { - let id = selected.candidate_of_node[node].0; - let template = physical.physical_node(PhysicalCandidateId(id)); - let expected = ordered_inputs(template); - let mut producers = Vec::with_capacity(inputs[node].len()); - for (input, producer) in inputs[node].iter().enumerate() { - let producer = rebuilt[producer.unwrap()].as_ref().unwrap(); - if !output_contract_satisfied(&expected[input], producer)? { - return Err(format!( - "candidate {id} input {input} has incompatible schema or state configuration" - )); - } - producers.push(Rc::clone(producer)); - } - let mut next = 0; - // Re-derive structural metadata over the selected inputs, preserving - // only explicit naming overrides. Logical rewrite templates can retain - // pre-rewrite keys/nullability; copying their entire schema is invalid. - let rebuilt_node = Rc::new( - template - .with_new_children(|_| { - let producer = Rc::clone(&producers[next]); - next += 1; - producer - }) - .map_err(|e| format!("candidate {id}: {e}"))? - .with_guarantee(template.guarantee.clone()), - ); - rebuilt_node - .validate_structure() - .map_err(|e| format!("candidate {id}: {e}"))?; - rebuilt[node] = Some(rebuilt_node); - for &consumer in &consumers[node] { - remaining[consumer] -= 1; - if remaining[consumer] == 0 { - ready.push(consumer); - } - } - } - let rebuilt = rebuilt - .into_iter() - .map(|node| node.ok_or_else(|| "selected nodes form a cycle".to_string())) - .collect::, _>>()?; - let roots = selected - .node_of_output_reference - .iter() - .map(|node| QueryRoot::Operator(Rc::clone(&rebuilt[node.0]))) - .collect::>(); - let outputs = recompose_guarantees(&roots) - .map_err(|e| e.to_string())? - .into_iter() - .map(|root| match root { - QueryRoot::Operator(node) => Ok(node), - QueryRoot::Scalar(_) => Err("unexpected scalar during operator reconstruction".into()), - }) - .collect::, String>>()?; - let old_outputs = roots - .iter() - .map(|root| match root { - QueryRoot::Operator(node) => Rc::clone(node), - QueryRoot::Scalar(_) => unreachable!("operator references were constructed above"), - }) - .collect::>(); - let identities = paired_node_identities(&old_outputs, &outputs)?; - let operator_node_of_selected_node = rebuilt - .into_iter() - .map(|node| { - identities - .get(&Rc::as_ptr(&node)) - .cloned() - .ok_or_else(|| "selected node lost during guarantee recomposition".to_string()) - }) - .collect::, _>>()?; - Ok(ReconstructedCandidates { - outputs, - operator_node_of_selected_node, - }) -} - -/// Match corresponding graph positions without structural interning. Reject -/// splitting or merging identities, and rewrites that change immediate operators. -pub fn paired_node_identities( - before: &[Rc], - after: &[Rc], -) -> Result>, String> { - if before.len() != after.len() { - return Err("output reference count changed".into()); - } - let mut identities = HashMap::<*const OperatorNode, Rc>::new(); - let mut reverse = HashMap::new(); - let mut pending = before - .iter() - .cloned() - .zip(after.iter().cloned()) - .collect::>(); - while let Some((old, new)) = pending.pop() { - if let Some(existing) = identities.get(&Rc::as_ptr(&old)) { - if !Rc::ptr_eq(existing, &new) { - return Err("selected node was split into multiple physical nodes".into()); - } - continue; - } - if reverse - .insert(Rc::as_ptr(&new), Rc::as_ptr(&old)) - .is_some_and(|previous| previous != Rc::as_ptr(&old)) - { - return Err("distinct selected nodes were merged".into()); - } - let old_operator = old.operator.map_children(|_| ()); - let new_operator = new.operator.map_children(|_| ()); - // Compare all serialized parameters, including sketch probability and - // non-finite scalar values. PartialEq rejects an unchanged NaN, while - // JSON erases the distinction between NaN and either infinity. - if rmp_serde::to_vec(&old_operator).map_err(|e| e.to_string())? - != rmp_serde::to_vec(&new_operator).map_err(|e| e.to_string())? - { - return Err("operator implementation changed; explicit physical candidate enumeration is required".into()); - } - let old_children = ordered_inputs(&old); - let new_children = ordered_inputs(&new); - if old_children.len() != new_children.len() { - return Err("candidate input count changed".into()); - } - pending.extend(old_children.into_iter().zip(new_children)); - identities.insert(Rc::as_ptr(&old), new); - } - Ok(identities) -} diff --git a/crates/physical-asap-optimization/src/lib.rs b/crates/physical-asap-optimization/src/lib.rs index aed2e101c..252aecf16 100644 --- a/crates/physical-asap-optimization/src/lib.rs +++ b/crates/physical-asap-optimization/src/lib.rs @@ -1,2437 +1,18 @@ -//! #509 Stage 2, physical ASAP-aware optimization: materialization and -//! physical operator implementation. +//! #509 Stage 2, physical ASAP-aware optimization (concepts doc, +//! section 3, "Stage 2: physical alternatives"). //! //! The planner facade calls [`generate_physical_asap_memo_candidates`] on -//! Pass 2's memo (`asap_types::stages::LogicalASAPMemoCandidates`): every -//! operator node of every memo candidate with each allowed timing and -//! materialization, the producers of each input and the pruned candidates +//! Pass 2's output (`asap_types::stages::LogicalASAPMemoCandidates`): each +//! logical candidate expanded into one physical candidate per valid +//! $(\tau, \ell, \sigma)$, each logical edge into its physical edges with +//! $\alpha^{\log}$ and $\alpha^{\mathrm{phys}}$, and the pruned alternatives //! (`asap_types::stages::PhysicalASAPMemoCandidates`). Stage 3 selects from -//! that output, and [`Stage2AssignmentRealizer`] builds the selected -//! assignment as a physical plan. The rules below apply to both, and to the -//! whole-workload reference pricer `optimize_physical_asap` (feature -//! `reference-pricer`, for the developer tools), which turns one -//! CandidateLogicalASAPDAG into its CandidatePhysicalASAPDAGs. The crate -//! depends on no other stage crate. -//! -//! ## Materialization -//! -//! Each summary build (a `SummaryAgg` or a `SummaryTumblingWindowChain`) is one -//! materialization candidate (`MaterializationCandidate`) whose -//! output materialization may store (#509 §Materialization, Algorithm 3). -//! So is the highest node of each other sub-DAG that two or more queries -//! read (e.g. a `Scan` that a 1 m and a 5 m window read): one a root or a -//! node only one query reads consumes. Such a node has only -//! `NotMaterialized` and `MatAtQuery(Memory)`, so a shared sub-DAG adds one -//! option, not one per node. A summary build has up to four options: -//! -//! * `NotMaterialized`: rebuilt at query time by every query that reads it; -//! * `MatAtIngestion(Memory)`: maintained as data arrives and kept for the -//! queries, when the build is eligible (`IngestionPruneReason`); -//! * `MatAtQuery(Memory)`: built once at query time for every query of one -//! evaluation that reads it and reused only within that evaluation, -//! offered for a build two or more queries read; -//! * `MatAtQueryKeptAcrossEvaluations(Memory)`: built at query time and -//! kept across evaluations (#509 Example 4's B3), offered when the -//! deployment keeps query-time results (the caller's -//! `query_time_results_retained`) for a -//! tumbling-window chain that builds one new tumbling window per evaluation -//! (`builds_one_tumbling_window_per_evaluation`): each evaluation builds the -//! newest tumbling window and reuses the others. Both query-time options are offered -//! when both apply, and Stage 3 prices each. -//! -//! A build at ingestion time needs every build below it at ingestion time too -//! (#509 Algorithm 3, line 13), and every shared non-summary node below it -//! `NotMaterialized`; other assignments are not generated. Every -//! other combination of options is one physical candidate, the -//! all-`NotMaterialized` one first: nothing is capped. A node that one -//! query reads at ingestion time and another at query time is copied, one -//! copy per timing (`split_shared_by_timing`). -//! -//! A build may run at ingestion time (S4) only when: -//! -//! * the data workload is continuously ingesting or mixed (unknown arrival -//! counts as not ingesting); -//! * every query reading it repeats and is predictable; -//! * it has no filter: an ingestion-time build reads raw data samples, which -//! carry none of the columns a filter reads; -//! * if it is a counter state (`Rate`, `Increase`, `IRate`), whose update -//! needs each series' samples in time order, the data is continuously -//! ingesting and every source below it is a time series. Assumption: -//! continuously ingesting PromQL data arrives in time order per series, -//! as Prometheus assumes (it rejects out-of-order samples). Table (SQL) -//! sources and mixed data, whose at-rest part may load in any order, are -//! not taken to be ordered; -//! * it is a tumbling-window chain whose readers all evaluate on its tumbling window -//! boundaries, anchored at the epoch (each fixed interval a multiple of the -//! tumbling-window width, and each known phase on that grid), or a whole-window build whose readers -//! all run at one fixed interval with agreeing declared phases, over a -//! window that is not shifted and whose evaluations all end on the grid -//! of its width anchored at the epoch (the interval and a declared phase -//! are multiples of the window; a reader with no declared phase is taken -//! to evaluate at multiples of its interval, as a chain's is), and none -//! is a range evaluation of more than one step per request (one window -//! per request cannot serve every step). -//! So a window equal to its interval (`m[5m]` every 5 minutes) is one -//! tumbling window maintained at ingestion time; -//! * everything below it can run at ingestion time. -//! -//! ## Range evaluations -//! -//! A query whose workload entry declares a -//! [`RangeEvaluation`](asap_types::workload::RangeEvaluation) evaluates at -//! every `step_ms` from `end − span_ms` forward to each request's end. -//! Every step reads the -//! tumbling windows of a tumbling-window chain, so each tumbling-window width must divide the step -//! of every range-evaluated reader. Stage 1 offers only such widths -//! (`QueryWorkloadEntry::evaluation_grid_ms`); a candidate with a chain off -//! a reader's step grid fails with -//! [`PhysicalASAPOptimizationError::TumblingWindowWidthOffRangeStep`] and the caller -//! prunes it with that reason. -//! -//! ## Physical operator implementation -//! -//! The runtime has one implementation per logical operator except exact -//! top-k, which it cannot run as `Aggregate{[TopK]}`. Stage 2 records that -//! choice in the DAG by rewriting it to a per-group sort followed by a -//! per-group limit, the shape the PromQL frontend uses for generic `topk`. -//! A `Project` over the limited rows restores the logical `[keys.., topk_k]` -//! columns, so an exact root has the same column names as a summary -//! candidate's and as the frontend declares. Over PromQL rows it also keeps -//! the series identity, after them: `topk` answers whole input series, so -//! each answer carries the chosen series' full label set. PromQL rows without -//! one (e.g. of `sum by (job)`) keep their other string label columns instead, -//! and SQL rows their other group keys (the ranked item, e.g. `host` of -//! `GROUP BY host ORDER BY COUNT(*) DESC LIMIT k`). -//! `topk_k` carries the ranked value's type and nullability (the frontend -//! declares a non-null `Utf8` placeholder); casting the value to `Utf8` would -//! not make it the same answer. -//! An `Aggregate{[TopK]}` over a per-entity reduction, or with other -//! measures, a filter or `HAVING`, has no such rewrite, and its candidate -//! fails with a [`PhysicalASAPOptimizationError`]; the caller prunes that -//! candidate alone, with the error as its reason. -//! A summary needs no rewrite: `SummaryAgg` → `SummaryEstimate` already is -//! build → estimate. Every node keeps the guarantee Stage 1 composed for it. +//! that output, and [`Stage2AssignmentRealizer`] builds the selected DAG as a +//! physical plan. The crate depends on no other stage crate. + mod catalog; -mod split_by_timing; pub use catalog::{ - generate_physical_asap_memo_candidates, CandidateInventoryError, ConfigurationError, - ErrorGeneratingPhysicalASAPMemoCandidates, Stage2AssignmentRealizer, -}; - -use std::collections::{HashMap, HashSet}; -use std::rc::Rc; - -use asap_types::ir::operator::operator_properties::Source; -use asap_types::ir::operator::{AggIntent, Reduction}; -use asap_types::ir::physical_export::compile_physical_asap_workload_with_node_ids; -use asap_types::ir::properties::{ExecutionDataStateError, ExecutionTiming}; -use asap_types::ir::schema::{ExactKind, FieldDataType}; -use asap_types::ir::{ - apply_materialization_timings, is_filtered_build, ASAPOp, MaterializationAssignment, - NodeToTimedNodeCache, NonASAPOp, Operator, OperatorNode, QueryRoot, ScalarExpr, - SchemaDerivationError, SortKey, -}; -use asap_types::physical::Materialization; -use asap_types::physical::Medium; -use asap_types::physical::PhysicalWorkloadPlan; -use asap_types::workload::{ - DataArrival, DataWorkload, Predictability, QueryRecurrence, QueryWorkloadEntry, RepeatedDemand, + generate_physical_asap_memo_candidates, ErrorGeneratingPhysicalASAPMemoCandidates, + Stage2AssignmentRealizer, }; -use split_by_timing::split_shared_by_timing; -use thiserror::Error; - -#[derive(Debug, Error)] -pub enum PhysicalASAPOptimizationError { - #[error(transparent)] - Structure(#[from] SchemaDerivationError), - #[error(transparent)] - Timing(#[from] ExecutionDataStateError), - #[error("exact top-k has no Sort + Limit implementation: {0}")] - ExactTopK(String), - /// Bad candidate for the workload: a tumbling-window chain whose tumbling window - /// width does not divide the step of a query that reads it as a range - /// request, so some step would end inside a tumbling window and could not read - /// the tumbling windows. Stage 1 offers only tumbling windows on every reader's step grid; - /// the candidate is pruned with this reason. - #[error( - "tumbling-window width {tumbling_window_width_ms} ms does not divide the {step_ms} ms step of \ - a range evaluation that reads it" - )] - TumblingWindowWidthOffRangeStep { - /// The chain's tumbling-window width. - tumbling_window_width_ms: u64, - /// The reader's range-evaluation step. - step_ms: u64, - }, -} - -/// Why a summary build cannot be materialized at ingestion time (S4). -/// Shared by complete-plan Stage 2 and explicit selected-reader eligibility. -#[derive(Debug, Clone, PartialEq, Eq)] -pub(crate) enum IngestionPruneReason { - /// The data workload is at rest, or its arrival is unknown. - NotIngesting, - /// A query reading it is not repeated. - NotRepeated, - /// A query reading it is not predictable. - NotPredictable, - /// It has a filter, which only a query-time build can apply. - Filtered, - /// It is a counter state, whose update needs each series' samples in - /// time order, over data not taken to arrive in that order: only - /// continuously ingesting time-series (PromQL) sources are. - CounterNeedsTimeOrder, - /// A whole-window build whose readers do not all run at one fixed - /// interval and one phase, or whose window is shifted or not a range. - WindowNotFixed, - /// A whole-window build whose evaluations do not all end on the grid of - /// its window's width anchored at the epoch, where the backend builds - /// it (e.g. `m[20s]` every 20 s at phase 5 s, every 30 s, or `m[15m]` - /// every 5 minutes, which tumbling windows serve): the interval or the declared - /// phase is not a multiple of the window. - WindowOffEpochGrid, - /// A whole-window build read by a range evaluation of more than one - /// step per request: it keeps one window per request, not one per step. - RangeEvaluatedWholeWindow, - /// A tumbling-window chain read at a known phase off its tumbling-window boundaries, - /// which are anchored at the epoch (e.g. 20 s tumbling windows read at phase 5 s): - /// built once each, its tumbling windows cannot end at every evaluation. - TumblingWindowsOffPhase, - /// A tumbling-window chain read at a fixed interval that its width does - /// not divide (e.g. 15 s tumbling windows read every 10 s): built once - /// each on the grid anchored at the epoch, its tumbling windows cannot - /// end at every evaluation. - TumblingWindowsOffInterval, - /// Something below it can only run at query time. - UpstreamAtQueryTime(String), - /// It is not a summary build (e.g. a shared `Scan`): only a summary's - /// state is maintained as data arrives. - NotASummaryBuild, -} - -/// Workload eligibility for one summary and only its selected query readers. -/// These are local options, not a complete physical assignment. Ingestion still -/// requires selected upstream implementations to pass timing/down-closure checks. -#[derive(Debug)] -pub(crate) struct SummaryMaterializationEligibility { - pub(crate) at_ingestion: Result<(), IngestionPruneReason>, - pub(crate) may_keep_across_evaluations: bool, - /// A tumbling window's selected readers must also meet their range-step grids. - pub(crate) range_compatibility: Result<(), PhysicalASAPOptimizationError>, -} - -#[derive(Debug, thiserror::Error)] -pub(crate) enum SummaryEligibilityError { - #[error("selected reader {query} has no workload entry")] - UnknownReader { query: usize }, - #[error("materialization eligibility requires a summary build or tumbling-window chain")] - NotSummary, - #[error("materialization eligibility requires at least one selected reader")] - NoSelectedReaders, - #[error("tumbling-window width must be positive")] - ZeroPaneWidth, -} - -/// Evaluate an implemented summary against explicit selected query indices. -/// Never supply the union of potential readers. Duplicate indices count once; -/// missing workload entries are rejected as malformed reader references. The node must carry the actual bound source/filter/window subtree, -/// and `data` describes that source's arrival. This function does not implement -/// operators, assign upstream timing, or prove complete execution legality. -pub(crate) fn summary_materialization_eligibility( - node: &OperatorNode, - selected_readers: &[usize], - entries: &[QueryWorkloadEntry], - data: &DataWorkload, - query_time_results_retained: bool, -) -> Result { - if !is_summary_build(node) { - return Err(SummaryEligibilityError::NotSummary); - } - if selected_readers.is_empty() { - return Err(SummaryEligibilityError::NoSelectedReaders); - } - if matches!( - node.asap(), - Some(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms: 0, - .. - }) - ) { - return Err(SummaryEligibilityError::ZeroPaneWidth); - } - if let Some(&query) = selected_readers - .iter() - .find(|&&query| query >= entries.len()) - { - return Err(SummaryEligibilityError::UnknownReader { query }); - } - let mut readers = selected_readers.to_vec(); - readers.sort_unstable(); - readers.dedup(); - Ok(summary_reader_eligibility( - node, - &readers, - entries, - data, - query_time_results_retained, - )) -} - -fn summary_reader_eligibility( - node: &OperatorNode, - readers: &[usize], - entries: &[QueryWorkloadEntry], - data: &DataWorkload, - query_time_results_retained: bool, -) -> SummaryMaterializationEligibility { - let recurrences: Option> = readers - .iter() - .map(|&query| entries.get(query).map(|entry| &entry.recurrence)) - .collect(); - let may_keep_across_evaluations = query_time_results_retained - && match (node.asap(), recurrences) { - ( - Some(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms, - .. - }), - Some(recurrences), - ) => builds_one_tumbling_window_per_evaluation(*tumbling_window_width_ms, recurrences), - _ => false, - }; - SummaryMaterializationEligibility { - at_ingestion: ingestion_eligibility(node, readers, entries, data), - may_keep_across_evaluations, - range_compatibility: check_summary_range_steps(node, readers, entries), - } -} - -/// One summary build, or one highest shared non-summary node, Stage 2 may -/// store, with its materialization options. -/// It is not a whole-workload plan like [`PhysicalWorkloadPlan`]: it is Stage -/// 2's enumeration bookkeeping, and the physical candidates are the -/// combinations of the materialization candidates' options. For example, -/// over one metric: -/// -/// ```text -/// q1: quantile_over_time(0.99, latency[5m]) every 1m -/// q2: max_over_time(latency[5m]) every 1m -/// ``` -/// -/// If Stage 1 chose KLL tumbling windows for q1 and exact max tumbling windows for q2, Stage 2 -/// finds two materialization candidates: -/// -/// | materialization candidate | readers | options | -/// |----------------------------------|---------|---------------------------------| -/// | Kll ×5 tumbling windows (q1) | q1 | NotMaterialized, MatAtIngestion | -/// | Max acc ×5 tumbling windows (q2) | q2 | NotMaterialized, MatAtIngestion | -/// -/// One read by two or more queries also gets `MatAtQuery` (built once per -/// evaluation and reused), and so does the highest node of a shared -/// non-summary sub-DAG, e.g. a `Scan` both queries read when Stage 1 -/// shares it. The physical candidates are the product of the -/// options, here 2 × 2 = 4, less the assignments that put a build at -/// ingestion time over one that is not; Stage 3 prices each. -/// `at_ingestion` records why ingestion is ruled out when it is (e.g. -/// `Filtered`, `NotRepeated`). Each physical candidate carries the -/// resulting assignment ([`PhysicalWorkloadPlan::materialization`]). -#[derive(Debug, Clone)] -pub(crate) struct MaterializationCandidate { - /// The build in the implemented DAG. - pub(crate) node: Rc, - /// The queries (indices into the roots) that read it. - pub(crate) readers: Vec, - /// `Ok` when `MatAtIngestion` is an option. - pub(crate) at_ingestion: Result<(), IngestionPruneReason>, - /// Whether `MatAtQueryKeptAcrossEvaluations` is an option: the - /// deployment keeps query-time results and it is a tumbling-window chain - /// that builds one new tumbling window per evaluation. - pub(crate) may_keep_across_evaluations: bool, -} - -impl MaterializationCandidate { - /// Its options, `NotMaterialized` first: `MatAtIngestion` when eligible, - /// `MatAtQuery` when two or more queries read it, and - /// `MatAtQueryKeptAcrossEvaluations` when it may be kept across - /// evaluations. - pub(crate) fn options(&self) -> Vec { - let mut options = vec![Materialization::NotMaterialized]; - if self.at_ingestion.is_ok() { - options.push(Materialization::MatAtIngestion(Medium::Memory)); - } - if self.readers.len() > 1 { - options.push(Materialization::MatAtQuery(Medium::Memory)); - } - if self.may_keep_across_evaluations { - options.push(Materialization::MatAtQueryKeptAcrossEvaluations( - Medium::Memory, - )); - } - options - } -} - -/// Whether a tumbling-window chain of `tumbling_window_width_ms`-wide tumbling windows read by -/// queries repeating as `readers` builds exactly one new tumbling window per -/// evaluation: there is a reader, every reader repeats at a fixed interval -/// equal to the tumbling-window width, and the readers whose evaluation phase is known -/// share one phase (modulo the interval). Its other tumbling windows are the previous -/// evaluations' newest, so a deployment that keeps query-time results -/// reuses them (#509 Example 4's B3). -/// -/// Co-scheduling assumption: readers with the same interval and no known -/// phase run at the same instants as each other and as the readers with a -/// known phase, as plan selection (`asap-plan-selection`) assumes when it counts -/// one evaluation per interval. Readers at different known phases would -/// each need their own newest tumbling window, so they do not share one kept chain. -pub(crate) fn builds_one_tumbling_window_per_evaluation<'a>( - tumbling_window_width_ms: u64, - readers: impl IntoIterator, -) -> bool { - if tumbling_window_width_ms == 0 { - return false; - } - let mut phase_ms = None; - let mut any_reader = false; - for recurrence in readers { - any_reader = true; - if recurrence.fixed_interval_ms() != Some(tumbling_window_width_ms) { - return false; - } - if let QueryRecurrence::Repeated(RepeatedDemand::FixedIntervalAt { - evaluation_phase, .. - }) = recurrence - { - let phase = evaluation_phase.0 % tumbling_window_width_ms; - if *phase_ms.get_or_insert(phase) != phase { - return false; - } - } - } - any_reader -} - -/// Stage 2 for one CandidateLogicalASAPDAG: implement every operator -/// (physical operator implementation), then one physical candidate per -/// materialization assignment of its mat_candidates, the all-`NotMaterialized` one -/// first. `entries[i]` is the workload entry of query `i` (`roots[i]`): -/// its recurrence and predictability decide whether state may be kept from -/// ingestion time; a missing entry is unknown. `id` is left empty for the -/// caller to name. Sharing between `roots` is preserved: one cache serves the -/// whole workload. `query_time_results_retained` says whether a query-time -/// result may be kept across evaluations. -/// -/// The reference pricer: only the developer tools (and tests) enumerate a -/// whole workload this way; planning goes through -/// [`generate_physical_asap_memo_candidates`]. -#[cfg(any(test, feature = "reference-pricer"))] -pub fn optimize_physical_asap( - from_logical: &str, - roots: &[QueryRoot], - entries: &[QueryWorkloadEntry], - data: &DataWorkload, - query_time_results_retained: bool, -) -> Result, PhysicalASAPOptimizationError> { - let (implemented, _, mat_candidates) = - implement_and_find_candidates(roots, entries, data, query_time_results_retained)?; - materialization_assignments(&mat_candidates) - .map(|assignment| { - assigned_candidate(from_logical, &implemented, &mat_candidates, assignment) - }) - .collect() -} - -/// Stage 2 for one CandidateLogicalASAPDAG and one materialization -/// assignment: the physical candidate [`optimize_physical_asap`] lists for -/// it. `choose(readers)` gives the option for each materialization -/// candidate (each summary build in discovery order, query by query, each -/// query's builds as a walk from its root meets them, then each highest -/// shared non-summary node in the same order), `readers` being the queries -/// that read it. `None` when -/// a chosen option is not one of the build's, or the assignment -/// materializes a build at ingestion time above one at query time. -#[cfg(feature = "test-support")] -pub fn optimize_physical_asap_assigned( - from_logical: &str, - roots: &[QueryRoot], - entries: &[QueryWorkloadEntry], - data: &DataWorkload, - query_time_results_retained: bool, - choose: &mut dyn FnMut(&[usize]) -> Option, -) -> Result, PhysicalASAPOptimizationError> { - optimize_physical_asap_chosen( - from_logical, - roots, - entries, - data, - query_time_results_retained, - &mut |_, readers, _| choose(readers), - ) -} - -/// [`optimize_physical_asap_chosen`]'s choice per materialization -/// candidate: from the logical node it implements, its readers and its -/// options. `None` rejects the assignment. -pub(crate) type ChooseMaterialization<'a> = - dyn FnMut(*const OperatorNode, &[usize], &[Materialization]) -> Option + 'a; - -/// Stage 2 for one CandidateLogicalASAPDAG and one materialization -/// assignment chosen node by node: `choose(logical, readers, options)` -/// gives the option for each materialization candidate, `logical` being -/// the address of the logical node of `roots` it implements (null when it -/// implements none, e.g. a node the implementation adds) and `readers` the -/// queries that read it. `None` when `choose` rejects, a chosen option is -/// not one of the candidate's, or the assignment materializes a build at ingestion time -/// above one at query time. The search that decodes one plan from a model -/// solution uses it to build that plan alone, not every assignment. -pub(crate) fn optimize_physical_asap_chosen( - from_logical: &str, - roots: &[QueryRoot], - entries: &[QueryWorkloadEntry], - data: &DataWorkload, - query_time_results_retained: bool, - choose: &mut ChooseMaterialization<'_>, -) -> Result, PhysicalASAPOptimizationError> { - let (implemented, cache, mat_candidates) = - implement_and_find_candidates(roots, entries, data, query_time_results_retained)?; - let logical_of: HashMap<*const OperatorNode, *const OperatorNode> = cache - .iter() - .map(|(logical, implemented)| (Rc::as_ptr(implemented), *logical)) - .collect(); - let mut assignment = Vec::with_capacity(mat_candidates.len()); - for mat_candidate in &mat_candidates { - let logical = logical_of - .get(&Rc::as_ptr(&mat_candidate.node)) - .copied() - .unwrap_or(std::ptr::null()); - let options = mat_candidate.options(); - let Some(m) = - choose(logical, &mat_candidate.readers, &options).filter(|m| options.contains(m)) - else { - return Ok(None); - }; - assignment.push(m); - } - let nodes: Vec<_> = mat_candidates.iter().map(|c| Rc::clone(&c.node)).collect(); - if !down_closed(&mat_candidates, &candidates_below(&nodes), &assignment) { - return Ok(None); - } - assigned_candidate(from_logical, &implemented, &mat_candidates, assignment).map(Some) -} - -/// The physical candidate of the implemented roots `implemented` with -/// `assignment[c]` chosen for `mat_candidates[c]`. -fn assigned_candidate( - from_logical: &str, - implemented: &[QueryRoot], - mat_candidates: &[MaterializationCandidate], - assignment: Vec, -) -> Result { - let materialized: Vec<_> = mat_candidates - .iter() - .zip(&assignment) - .filter(|(_, m)| **m != Materialization::NotMaterialized) - .map(|(mat_candidate, m)| (mat_candidate, *m)) - .collect(); - let mut options = MaterializationAssignment::with_default(Materialization::NotMaterialized); - for (mat_candidate, materialization) in &materialized { - options.set(&mat_candidate.node, *materialization); - } - let operators: Vec> = implemented - .iter() - .flat_map(|root| root.operator_refs().into_iter().cloned()) - .collect(); - let (split, options) = split_shared_by_timing(&operators, &options); - let replaced: HashMap<_, _> = operators - .iter() - .map(Rc::as_ptr) - .zip(split.iter().cloned()) - .collect(); - let split_roots = map_roots(implemented, &mut |node| { - Ok(replaced[&Rc::as_ptr(node)].clone()) - })?; - let mut timing = NodeToTimedNodeCache::new(); - let timed = map_roots(&split_roots, &mut |node| { - Ok(apply_materialization_timings(node, &options, &mut timing)?) - })?; - let dag = compile_physical_asap_workload_with_node_ids(&timed)?.dag; - let materialization = dag - .nodes - .iter() - .filter(|n| { - matches!( - n.payload, - Operator::ASAP( - ASAPOp::SummaryAgg { .. } | ASAPOp::SummaryTumblingWindowChain { .. } - ) - ) || n.materialization != Materialization::NotMaterialized - }) - .map(|n| (n.id, n.materialization)) - .collect(); - Ok(PhysicalWorkloadPlan { - id: String::new(), - from_logical: from_logical.to_string(), - roots: timed, - dag, - materialization, - }) -} - -/// `roots` implemented, the implementation of each logical node, and the -/// materialization candidates of the implemented roots. -fn implement_and_find_candidates( - roots: &[QueryRoot], - entries: &[QueryWorkloadEntry], - data: &DataWorkload, - query_time_results_retained: bool, -) -> Result< - ( - Vec, - NodeToImplementedNodeCache, - Vec, - ), - PhysicalASAPOptimizationError, -> { - let mut cache = HashMap::new(); - let implemented = map_roots(roots, &mut |node| implement(node, &mut cache))?; - let mat_candidates = - materialization_candidates(&implemented, entries, data, query_time_results_retained); - check_tumbling_windows_on_range_steps(&mat_candidates, entries)?; - Ok((implemented, cache, mat_candidates)) -} - -/// `Err` for the first tumbling-window chain whose tumbling-window width does not -/// divide the step of a reader's range evaluation -/// ([`PhysicalASAPOptimizationError::TumblingWindowWidthOffRangeStep`]): that step -/// would end inside a tumbling window. -fn check_tumbling_windows_on_range_steps( - mat_candidates: &[MaterializationCandidate], - entries: &[QueryWorkloadEntry], -) -> Result<(), PhysicalASAPOptimizationError> { - for candidate in mat_candidates { - check_summary_range_steps(&candidate.node, &candidate.readers, entries)?; - } - Ok(()) -} - -fn check_summary_range_steps( - node: &OperatorNode, - readers: &[usize], - entries: &[QueryWorkloadEntry], -) -> Result<(), PhysicalASAPOptimizationError> { - let Some(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms, - .. - }) = node.asap() - else { - return Ok(()); - }; - for &query in readers { - let Some(range) = entries - .get(query) - .and_then(|entry| entry.range_evaluation) - .filter(|range| range.evaluations_per_request() > 1) - else { - continue; - }; - if range.step_ms % tumbling_window_width_ms.max(&1) != 0 { - return Err( - PhysicalASAPOptimizationError::TumblingWindowWidthOffRangeStep { - tumbling_window_width_ms: *tumbling_window_width_ms, - step_ms: range.step_ms, - }, - ); - } - } - Ok(()) -} - -/// Whether `node` is a summary build (`SummaryAgg` or `SummaryTumblingWindowChain`). -fn is_summary_build(node: &OperatorNode) -> bool { - matches!( - node.asap(), - Some(ASAPOp::SummaryAgg { .. } | ASAPOp::SummaryTumblingWindowChain { .. }) - ) -} - -/// The materialization candidates of `roots` (implemented), with their -/// readers, whether each may run at ingestion time and whether it may be -/// kept from query time across evaluations: every summary build, in -/// discovery order, then every highest shared non-summary node, in -/// discovery order. A shared node is one two or more queries read; the -/// highest are those a root or an unshared node reads (e.g. a `Scan` that -/// two windows read). Only a highest one is offered `MatAtQuery`, so a -/// shared sub-DAG adds one candidate, not one per node. -pub(crate) fn materialization_candidates( - roots: &[QueryRoot], - entries: &[QueryWorkloadEntry], - data: &DataWorkload, - query_time_results_retained: bool, -) -> Vec { - let mut reached: Vec> = Vec::new(); - let mut readers: HashMap<*const OperatorNode, Vec> = HashMap::new(); - let mut consumers: HashMap<*const OperatorNode, Vec<*const OperatorNode>> = HashMap::new(); - let mut root_nodes: HashSet<*const OperatorNode> = HashSet::new(); - for (query, root) in roots.iter().enumerate() { - for operator in root.operator_refs() { - root_nodes.insert(Rc::as_ptr(operator)); - for node in OperatorNode::reachable(operator) { - let entry = readers.entry(Rc::as_ptr(&node)).or_default(); - if entry.is_empty() { - reached.push(Rc::clone(&node)); - for child in node.children() { - consumers - .entry(Rc::as_ptr(child)) - .or_default() - .push(Rc::as_ptr(&node)); - } - } - if !entry.contains(&query) { - entry.push(query); - } - } - } - } - let shared = |node: *const OperatorNode| readers[&node].len() > 1; - let highest_shared = |node: &Rc| { - let ptr = Rc::as_ptr(node); - shared(ptr) - && (root_nodes.contains(&ptr) - || consumers - .get(&ptr) - .is_some_and(|c| c.iter().any(|&consumer| !shared(consumer)))) - }; - let (nodes, shared_results): (Vec<_>, Vec<_>) = - reached.into_iter().partition(|n| is_summary_build(n)); - let below = candidates_below(&nodes); - let builds = nodes.iter().enumerate().map(|(c, node)| { - let readers = readers[&Rc::as_ptr(node)].clone(); - let eligibility = - summary_reader_eligibility(node, &readers, entries, data, query_time_results_retained); - let at_ingestion = eligibility.at_ingestion.and_then(|()| { - // Only this build and the builds below it at ingestion time. - let mut trial = - MaterializationAssignment::with_default(Materialization::NotMaterialized); - for &v in below[c].iter().chain([&c]) { - trial.set(&nodes[v], Materialization::MatAtIngestion(Medium::Memory)); - } - apply_materialization_timings(node, &trial, &mut NodeToTimedNodeCache::new()) - .map(|_| ()) - .map_err(|e| IngestionPruneReason::UpstreamAtQueryTime(e.to_string())) - }); - MaterializationCandidate { - node: Rc::clone(node), - readers, - at_ingestion, - may_keep_across_evaluations: eligibility.may_keep_across_evaluations, - } - }); - // A node fixed at ingestion time cannot be built at query time. - let shared_results = shared_results - .into_iter() - .filter(|n| highest_shared(n) && n.timing != Some(ExecutionTiming::IngestionTime)) - .map(|node| MaterializationCandidate { - readers: readers[&Rc::as_ptr(&node)].clone(), - node, - at_ingestion: Err(IngestionPruneReason::NotASummaryBuild), - may_keep_across_evaluations: false, - }); - builds.chain(shared_results).collect() -} - -/// Whether every scan below `node` reads a time series (a PromQL source). -fn reads_only_time_series(node: &OperatorNode) -> bool { - match node.non_asap() { - Some(NonASAPOp::Scan { source, .. }) => matches!(source, Source::TimeSeries { .. }), - _ => node.children().iter().all(|c| reads_only_time_series(c)), - } -} - -/// The phase, modulo `tumbling_window_width_ms` (which must be nonzero), of -/// every evaluation of `demand`, if known. A plain fixed interval has cadence but no phase, so -/// it is unknown. An explicit schedule has one only when every timestamp in -/// it has the same phase. -fn plan_tumbling_window_phase( - demand: &RepeatedDemand, - tumbling_window_width_ms: u64, -) -> Option { - match demand { - RepeatedDemand::FixedIntervalAt { - evaluation_phase, .. - } => Some(evaluation_phase.0 % tumbling_window_width_ms), - RepeatedDemand::Scheduled(times) => { - let first = times.first().map(|t| t.0 % tumbling_window_width_ms); - first.filter(|phase| { - times - .iter() - .all(|t| t.0 % tumbling_window_width_ms == *phase) - }) - } - RepeatedDemand::FixedInterval(_) | RepeatedDemand::EstimatedRate(_) => None, - } -} - -/// The workload rules of S4 for `node` (module doc). -fn ingestion_eligibility( - node: &OperatorNode, - readers: &[usize], - entries: &[QueryWorkloadEntry], - data: &DataWorkload, -) -> Result<(), IngestionPruneReason> { - use IngestionPruneReason as Why; - if !matches!( - data.arrival, - DataArrival::Ingesting { .. } | DataArrival::Mixed { .. } - ) { - return Err(Why::NotIngesting); - } - let demands: Vec> = - readers.iter().map(|&q| entries.get(q)).collect(); - if demands - .iter() - .any(|d| !d.is_some_and(|d| matches!(d.recurrence, QueryRecurrence::Repeated(_)))) - { - return Err(Why::NotRepeated); - } - if demands - .iter() - .any(|d| !d.is_some_and(|d| matches!(d.predictability, Predictability::Predictable { .. }))) - { - return Err(Why::NotPredictable); - } - if is_filtered_build(node) { - return Err(Why::Filtered); - } - let (family, chain_tumbling_window_width_ms) = match node.asap() { - Some(ASAPOp::SummaryAgg { family, .. }) => (family, None), - Some(ASAPOp::SummaryTumblingWindowChain { - family, - tumbling_window_width_ms, - .. - }) => (family, Some(*tumbling_window_width_ms)), - _ => unreachable!("a materialization candidate is a summary build"), - }; - if matches!( - family, - FieldDataType::ExactAggregate(ExactKind::Rate | ExactKind::Increase | ExactKind::IRate, _) - ) { - // Assumption (module doc): continuously ingesting PromQL data - // arrives in time order per series, as Prometheus assumes. - let ordered = - matches!(data.arrival, DataArrival::Ingesting { .. }) && reads_only_time_series(node); - if !ordered { - return Err(Why::CounterNeedsTimeOrder); - } - } - if let Some(tumbling_window_width_ms) = chain_tumbling_window_width_ms { - // A chain's tumbling windows are built once each, on the grid anchored at the - // epoch (the plan carries no tumbling-window origin): every reader must - // evaluate on its boundaries. A reader at a fixed interval needs an - // interval the width divides, whatever its phase; a reader of known - // phase also needs that phase on the grid. A reader of unknown phase - // is taken to run at multiples of its interval (phase 0), as for a - // whole-window build below. - for d in demands.iter().flatten() { - let QueryRecurrence::Repeated(demand) = &d.recurrence else { - continue; - }; - if d.recurrence - .fixed_interval_ms() - .is_some_and(|interval_ms| !interval_ms.is_multiple_of(tumbling_window_width_ms)) - { - return Err(Why::TumblingWindowsOffInterval); - } - if tumbling_window_width_ms == 0 { - return Err(Why::TumblingWindowsOffPhase); - } - let Some(phase_ms) = plan_tumbling_window_phase(demand, tumbling_window_width_ms) - else { - if matches!(demand, RepeatedDemand::Scheduled(_)) { - // A schedule whose times are not all on one tumbling-window phase. - return Err(Why::TumblingWindowsOffPhase); - } - continue; - }; - if phase_ms != 0 { - return Err(Why::TumblingWindowsOffPhase); - } - } - return Ok(()); - } - if demands.iter().flatten().any(|d| { - d.range_evaluation - .is_some_and(|range| range.evaluations_per_request() > 1) - }) { - return Err(Why::RangeEvaluatedWholeWindow); - } - // A whole-window build: one tumbling window per evaluation, built (like - // a chain's tumbling windows) on the grid of its width anchored at the epoch, so - // every evaluation must end on that grid. The readers share one fixed - // interval and their declared phases agree; a reader with no declared - // phase is taken to evaluate at multiples of its interval (phase 0), as - // a chain's is, which lies on the grid once the interval does. - let mut interval_ms = None; - let mut phase_ms = None; - for d in demands.into_iter().flatten() { - let (interval, phase) = match &d.recurrence { - QueryRecurrence::Repeated(RepeatedDemand::FixedInterval(interval)) => { - (u64::from(interval.0), None) - } - QueryRecurrence::Repeated(RepeatedDemand::FixedIntervalAt { - interval, - evaluation_phase, - }) => ( - u64::from(interval.0), - evaluation_phase.0.checked_rem(u64::from(interval.0)), - ), - _ => return Err(Why::WindowNotFixed), - }; - if interval == 0 { - return Err(Why::WindowNotFixed); - } - if *interval_ms.get_or_insert(interval) != interval { - return Err(Why::WindowNotFixed); - } - if let Some(phase) = phase { - if *phase_ms.get_or_insert(phase) != phase { - return Err(Why::WindowNotFixed); - } - } - } - let Some(interval_ms) = interval_ms else { - return Err(Why::WindowNotFixed); - }; - let Some(NonASAPOp::TimeRange { - range, - kind: asap_types::ir::TimeRangeKind::Range, - child, - }) = node.children()[0].non_asap() - else { - return Err(Why::WindowNotFixed); - }; - if matches!(child.non_asap(), Some(NonASAPOp::TimeShift { .. })) { - return Err(Why::WindowNotFixed); - } - let window_ms = u64::try_from(range.as_millis()) - .ok() - .filter(|&ms| ms > 0) - .ok_or(Why::WindowNotFixed)?; - let on_grid = |ms: u64| ms.is_multiple_of(window_ms); - if on_grid(interval_ms) && phase_ms.is_none_or(on_grid) { - Ok(()) - } else { - Err(Why::WindowOffEpochGrid) - } -} - -/// For each of `nodes`, the indices of the others below it. -fn candidates_below(nodes: &[Rc]) -> Vec> { - nodes - .iter() - .map(|node| { - let reachable = OperatorNode::reachable(node); - (0..nodes.len()) - .filter(|&v| { - !Rc::ptr_eq(&nodes[v], node) - && reachable.iter().any(|n| Rc::ptr_eq(n, &nodes[v])) - }) - .collect() - }) - .collect() -} - -/// Whether, below every ingestion-time build of `assignment`, every build -/// is at ingestion time too and every shared non-summary node is -/// `NotMaterialized`: ingestion work reads it as it arrives, and its -/// query-time readers read a copy ([`split_shared_by_timing`]). -/// Simplification: that copy is not offered `MatAtQuery` even when two -/// query-time readers remain. `below` is [`candidates_below`] of -/// `mat_candidates`. -fn down_closed( - mat_candidates: &[MaterializationCandidate], - below: &[Vec], - assignment: &[Materialization], -) -> bool { - let at_ingestion = |m: &Materialization| m.timing() == ExecutionTiming::IngestionTime; - assignment.iter().enumerate().all(|(c, m)| { - !at_ingestion(m) - || below[c].iter().all(|&v| { - if is_summary_build(&mat_candidates[v].node) { - at_ingestion(&assignment[v]) - } else { - assignment[v] == Materialization::NotMaterialized - } - }) - }) -} - -/// Every assignment of an option to each materialization candidate (mixed -/// radix, the last varying fastest, all `NotMaterialized` first) that is -/// [`down_closed`]. -#[cfg(any(test, feature = "reference-pricer"))] -fn materialization_assignments( - mat_candidates: &[MaterializationCandidate], -) -> impl Iterator> + '_ { - let options: Vec> = mat_candidates.iter().map(|c| c.options()).collect(); - let nodes: Vec<_> = mat_candidates.iter().map(|c| Rc::clone(&c.node)).collect(); - let below = candidates_below(&nodes); - let mut digits = Some(vec![0usize; mat_candidates.len()]); - std::iter::from_fn(move || loop { - let current = digits.as_mut()?; - let assignment: Vec = current - .iter() - .zip(&options) - .map(|(&d, options)| options[d]) - .collect(); - // Advance (mixed radix); `None` after the last assignment. - let mut carried = true; - for (digit, options) in current.iter_mut().zip(&options).rev() { - *digit += 1; - if *digit < options.len() { - carried = false; - break; - } - *digit = 0; - } - if carried { - digits = None; - } - if down_closed(mat_candidates, &below, &assignment) { - return Some(assignment); - } - digits.as_ref()?; - }) -} - -/// `roots` with every operator node they read replaced by `f`'s result. -fn map_roots( - roots: &[QueryRoot], - f: &mut impl FnMut(&Rc) -> Result, PhysicalASAPOptimizationError>, -) -> Result, PhysicalASAPOptimizationError> { - roots - .iter() - .map(|root| match root { - QueryRoot::Operator(node) => Ok(QueryRoot::Operator(f(node)?)), - QueryRoot::Scalar(expr) => { - let mut mapped = HashMap::new(); - for node in expr.operator_refs() { - mapped.insert(Rc::as_ptr(node), f(node)?); - } - Ok(QueryRoot::Scalar(expr.map_operator_refs(&mut |node| { - mapped[&Rc::as_ptr(node)].clone() - }))) - } - }) - .collect() -} - -/// Lower the private operators of one candidate while preserving its already -/// canonical input-group nodes. This reuses the complete-plan implementation -/// rules (currently exact top-k -> Project/Limit/Sort), without assigning timing -/// or materialization and without recursively implementing another group's input. -/// Returned new operators must be enumerated and priced before plan selection. -pub(crate) fn implement_candidate_operators( - root: &Rc, - inputs: &[Rc], -) -> Result, PhysicalASAPOptimizationError> { - // The inventory rejects a root aliasing an input; an input outside the - // subtree is unused here and rejected by the inventory as `InvalidMemo`. - debug_assert!(!inputs.iter().any(|input| Rc::ptr_eq(input, root))); - let mut implementations = inputs - .iter() - .map(|input| (Rc::as_ptr(input), Rc::clone(input))) - .collect(); - implement(root, &mut implementations) -} - -/// Each node's implementation. -type NodeToImplementedNodeCache = HashMap<*const OperatorNode, Rc>; - -fn implement( - node: &Rc, - cache: &mut NodeToImplementedNodeCache, -) -> Result, PhysicalASAPOptimizationError> { - if let Some(done) = cache.get(&Rc::as_ptr(node)) { - return Ok(done.clone()); - } - for child in node.children() { - implement(child, cache)?; - } - let implemented = match node.non_asap() { - Some(NonASAPOp::Aggregate { measures, .. }) - if measures.iter().any(|m| matches!(m, AggIntent::TopK { .. })) => - { - implement_topk(node, cache)? - } - _ => rebuilt(node, cache)?, - }; - cache.insert(Rc::as_ptr(node), implemented.clone()); - Ok(implemented) -} - -/// Exact top-k as `Limit(Sort)` per group, which outputs the ranked rows of -/// the child, under a `Project` restoring the logical `[keys.., topk_k]` -/// output (the ranked value under `topk_k`): a root returns those columns, -/// and a node reading the top-k resolved its columns against them. -fn implement_topk( - node: &Rc, - cache: &NodeToImplementedNodeCache, -) -> Result, PhysicalASAPOptimizationError> { - let unsupported = |why: &str| PhysicalASAPOptimizationError::ExactTopK(why.to_string()); - let Some(NonASAPOp::Aggregate { - reduction, - measures, - filters, - having, - child, - .. - }) = node.non_asap() - else { - unreachable!("implement_topk is called on an Aggregate") - }; - let Reduction::Reduce(keys) = reduction else { - return Err(unsupported("per-entity reduction")); - }; - let [AggIntent::TopK { k, .. }] = measures.as_slice() else { - return Err(unsupported("measures besides the top-k")); - }; - if filters.iter().any(Option::is_some) { - return Err(unsupported("filtered measure")); - } - if having.is_some() { - return Err(unsupported("HAVING")); - } - let child = cache[&Rc::as_ptr(child)].clone(); - let value = asap_types::ir::schema::topk::ranked_column(&child) - .map_err(|e| unsupported(&format!("no ranked column: {e}")))?; - let sorted = Rc::new( - OperatorNode::new(Operator::NonASAP(NonASAPOp::Sort { - keys: vec![SortKey { - expr: ScalarExpr::Column(value), - ascending: false, - nulls_first: false, - }], - partition_by: keys.clone(), - child: child.clone(), - }))? - .with_guarantee(child.guarantee.clone()), - ); - let limited = Rc::new( - OperatorNode::new(Operator::NonASAP(NonASAPOp::Limit { - n: Some(*k), - offset: 0, - partition_by: keys.clone(), - child: sorted, - }))? - .with_guarantee(node.guarantee.clone()), - ); - let output_name = node.schema.fields[keys.len()].name.clone(); - let cols = asap_types::ir::schema::topk::ranked_row_projection( - &child, - &Reduction::Reduce(keys.clone()), - &output_name, - ) - .map_err(|error| unsupported(&format!("ranked rows: {error}")))?; - Ok(Rc::new( - OperatorNode::new(Operator::NonASAP(NonASAPOp::Project { - cols, - qualifier: None, - child: limited, - }))? - .with_guarantee(node.guarantee.clone()), - )) -} - -/// `node` over its implemented children; the same `Rc` when none changed. -fn rebuilt( - node: &Rc, - cache: &NodeToImplementedNodeCache, -) -> Result, PhysicalASAPOptimizationError> { - let input = |child: &Rc| cache[&Rc::as_ptr(child)].clone(); - let changed = node - .children() - .iter() - .any(|child| !Rc::ptr_eq(child, &input(child))); - Ok(if changed { - Rc::new( - node.with_new_children(|child| input(child))? - .with_guarantee(node.guarantee.clone()), - ) - } else { - node.clone() - }) -} - -/// Stage 2 internals that other crates' tests call: building one physical -/// candidate for a chosen assignment, the catalog steps, and the timing split. -#[cfg(feature = "test-support")] -pub mod test_support { - use asap_types::ir::QueryRoot; - use asap_types::workload::{DataWorkload, QueryWorkloadEntry}; - - use crate::{PhysicalASAPOptimizationError, PhysicalWorkloadPlan}; - - pub use crate::catalog::test_support as catalog; - pub use crate::split_by_timing::split_shared_by_timing; - - /// See `optimize_physical_asap_chosen` in this crate. - pub fn optimize_physical_asap_chosen( - from_logical: &str, - roots: &[QueryRoot], - entries: &[QueryWorkloadEntry], - data: &DataWorkload, - query_time_results_retained: bool, - choose: &mut crate::ChooseMaterialization<'_>, - ) -> Result, PhysicalASAPOptimizationError> { - crate::optimize_physical_asap_chosen( - from_logical, - roots, - entries, - data, - query_time_results_retained, - choose, - ) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use asap_logical_asap_optimization::test_support::lower_promql; - use asap_types::ir::physical_export::PhysicalASAPNodeId; - use asap_types::ir::physical_export::PhysicalASAPOperatorPayload as Payload; - use asap_types::ir::properties::AccuracyTarget; - use asap_types::ir::properties::ExecutionTiming; - use asap_types::ir::ASAPOp; - - // Candidate-local implementation must leave another group's canonical root untouched. - #[test] - fn candidate_operator_lowering_preserves_input_boundaries() { - let root = lower_promql( - "topk by (job) (3, sum_over_time(m[1m]))", - AccuracyTarget::exact(), - ); - let boundary = Rc::clone(root.children()[0]); - let lowered = - implement_candidate_operators(&root, std::slice::from_ref(&boundary)).unwrap(); - assert_eq!(lowered.schema, root.schema); - let nodes = OperatorNode::reachable(&lowered); - assert!(nodes.iter().any(|node| node.operator.kind_name() == "Sort")); - assert!(nodes - .iter() - .any(|node| node.operator.kind_name() == "Limit")); - assert!(nodes.iter().any(|node| Rc::ptr_eq(node, &boundary))); - } - - // Logical and exact ranked rows retain the full series identity contract. - #[test] - fn topk_logical_contract_preserves_series_identity() { - let template = lower_promql("topk(3, sum_over_time(m[1m]))", AccuracyTarget::exact()); - let source = asap_logical_asap_optimization::test_support::metric_scan(&[ - "job", - "instance", - asap_types::ir::schema::PROMQL_SERIES_IDENTITY, - ]); - let root = Rc::new(template.with_new_children(|_| Rc::clone(&source)).unwrap()); - let boundary = Rc::clone(root.children()[0]); - let lowered = implement_candidate_operators(&root, &[boundary]).unwrap(); - assert_eq!(lowered.schema, root.schema); - assert_eq!( - names(&root), - [ - "topk_3", - "job", - "instance", - asap_types::ir::schema::PROMQL_SERIES_IDENTITY - ] - ); - assert!(root - .schema - .fields - .iter() - .any(|field| field.name == asap_types::ir::schema::PROMQL_SERIES_IDENTITY)); - assert_eq!( - root.schema.fields[0].dtype, - asap_types::ir::schema::FieldDataType::Plain(asap_types::ir::schema::DataType::Float64) - ); - } - - /// The all-`NotMaterialized` candidate, with no workload declared. - fn first_candidate(roots: &[QueryRoot]) -> PhysicalWorkloadPlan { - optimize_physical_asap("L1", roots, &[], &DataWorkload::default(), false) - .unwrap() - .remove(0) - } - - fn operator_root(candidate: &PhysicalWorkloadPlan, i: usize) -> Rc { - match &candidate.roots[i] { - QueryRoot::Operator(node) => node.clone(), - QueryRoot::Scalar(_) => panic!("operator root"), - } - } - - fn names(node: &OperatorNode) -> Vec { - node.schema.fields.iter().map(|f| f.name.clone()).collect() - } - - /// The `Limit` under an exact top-k's restoring `Project`. - fn topk_limit(project: &OperatorNode) -> Rc { - let NonASAPOp::Project { child, .. } = project.expect_non_asap() else { - panic!("project, found {}", project.operator.kind_name()) - }; - child.clone() - } - - /// The single sort key of the `Sort` under the `Limit` at `limit`. - fn topk_sort_key(limit: &OperatorNode) -> (ScalarExpr, bool) { - let NonASAPOp::Limit { child: sort, .. } = limit.expect_non_asap() else { - panic!("limit, found {}", limit.operator.kind_name()) - }; - let NonASAPOp::Sort { keys, .. } = sort.expect_non_asap() else { - panic!("sort below limit") - }; - let [key] = keys.as_slice() else { - panic!("one sort key") - }; - (key.expr.clone(), key.ascending) - } - - /// Example 1's shape, `topk by (job) (k, per-series rows)`, sorts each - /// group by the sample value, descending. - #[test] - fn promql_topk_over_series_sorts_by_value_descending() { - let root = lower_promql( - "topk by (job) (10, sum_over_time(m[1m]))", - AccuracyTarget::exact(), - ); - let candidate = first_candidate(&[root.clone().into()]); - let physical = operator_root(&candidate, 0); - assert_eq!(names(&physical), names(&root)); - let limit = topk_limit(&physical); - assert_eq!(limit.schema.fields[1].name, "value"); - assert_eq!(topk_sort_key(&limit), (ScalarExpr::Column(1), false)); - } - - /// `topk(k, sum by (job) (..))` sorts by the summed measure. - #[test] - fn promql_topk_over_grouped_aggregate_sorts_by_its_measure() { - let root = lower_promql("topk(5, sum by (job) (m))", AccuracyTarget::exact()); - let candidate = first_candidate(&[root.into()]); - let physical = operator_root(&candidate, 0); - // The answers keep their `job` label. - assert_eq!(names(&physical), ["topk_5", "job"]); - let limit = topk_limit(&physical); - assert_eq!(names(&limit), ["job", "sum"]); - assert_eq!(topk_sort_key(&limit), (ScalarExpr::Column(1), false)); - } - - /// A node reading an exact top-k keeps resolving the columns it bound - /// against the logical `[keys.., topk_k]` output: `sum by (job)` still - /// groups by `job`, not by whatever column now sits at its index. - #[test] - fn consumer_of_exact_topk_reads_its_logical_columns() { - let root = lower_promql( - "sum by (job) (topk by (job) (10, sum_over_time(m[1m])))", - AccuracyTarget::exact(), - ); - let candidate = first_candidate(&[root.clone().into()]); - let physical = operator_root(&candidate, 0); - assert_eq!(names(&physical), names(&root)); - let NonASAPOp::Aggregate { child: input, .. } = physical.expect_non_asap() else { - panic!("sum root") - }; - let NonASAPOp::Project { - cols, child: limit, .. - } = input.expect_non_asap() - else { - panic!("project over the top-k rows") - }; - let read: Vec<_> = cols - .iter() - .map(|item| { - let ScalarExpr::Column(c) = item.expr else { - panic!("column") - }; - ( - item.alias.clone().unwrap(), - limit.schema.fields[c].name.clone(), - ) - }) - .collect(); - assert_eq!( - read, - [ - ("job".to_string(), "job".to_string()), - ("topk_10".to_string(), "value".to_string()) - ] - ); - assert_eq!(topk_sort_key(limit), (ScalarExpr::Column(1), false)); - } - - /// An `Aggregate{[TopK]}` that Sort + Limit cannot express (another - /// measure, `HAVING`) fails, so its candidate is pruned with the reason. - #[test] - fn topk_with_extra_measure_or_having_fails() { - use asap_logical_asap_optimization::test_support::{aggregate, metric_scan}; - use asap_types::ir::scalar::expr_ir::ScalarValue; - use asap_types::ir::Predicate; - let topk = AggIntent::TopK { - k: 3, - accuracy: AccuracyTarget::exact(), - }; - let count = AggIntent::Count { - accuracy: AccuracyTarget::exact(), - }; - let by_job = || Reduction::by(vec![2]); - let cases = [ - aggregate( - by_job(), - vec![topk.clone(), count], - vec![], - None, - metric_scan(&["job"]), - ), - aggregate( - by_job(), - vec![topk], - vec![], - Some(Predicate(ScalarExpr::Literal(ScalarValue::Boolean(true)))), - metric_scan(&["job"]), - ), - ]; - for root in cases { - let error = - optimize_physical_asap("L1", &[root.into()], &[], &DataWorkload::default(), false) - .unwrap_err(); - assert!( - matches!(error, PhysicalASAPOptimizationError::ExactTopK(_)), - "{error}" - ); - } - } - - /// Exact `topk by (job)` becomes Limit(Sort) partitioned by `job`, and a - /// child shared by two roots stays one node (one `Rc`, one DAG node). - #[test] - fn exact_topk_becomes_sort_then_limit_and_keeps_shared_child() { - let exact = AccuracyTarget::exact(); - let topk = lower_promql("topk by (job) (10, sum_over_time(m[1m]))", exact.clone()); - let sum = lower_promql("sum by (job) (sum_over_time(m[1m]))", exact); - // Share Q2's per-series child with Q1, as Stage 1 sharing would. - let NonASAPOp::Aggregate { child: shared, .. } = topk.expect_non_asap() else { - panic!("topk aggregate") - }; - let sum = Rc::new(sum.with_new_children(|_| shared.clone()).unwrap()); - let candidate = first_candidate(&[topk.into(), sum.into()]); - let root = |i: usize| match &candidate.roots[i] { - QueryRoot::Operator(node) => node.clone(), - QueryRoot::Scalar(_) => panic!("operator root"), - }; - - let NonASAPOp::Limit { - n: Some(10), - partition_by, - child: sort, - .. - } = topk_limit(&root(0)).expect_non_asap().clone() - else { - panic!("limit under the root") - }; - assert!(!partition_by.keys().is_empty()); - let NonASAPOp::Sort { - child: below_sort, - partition_by: sort_partition, - .. - } = sort.expect_non_asap() - else { - panic!("sort below limit") - }; - assert_eq!(*sort_partition, partition_by); - let NonASAPOp::Aggregate { - child: below_sum, .. - } = root(1).expect_non_asap().clone() - else { - panic!("sum root") - }; - assert!(Rc::ptr_eq(below_sort, &below_sum)); - - let dag = &candidate.dag; - let kind = |id: PhysicalASAPNodeId| match &dag - .nodes - .iter() - .find(|n| n.id == id) - .unwrap() - .payload - { - Payload::NonASAP(NonASAPOp::Sort { .. }) => "sort", - Payload::NonASAP(NonASAPOp::Limit { .. }) => "limit", - _ => "other", - }; - let producer_of = |consumer| { - dag.edges - .iter() - .find(|e| e.consumer == consumer) - .unwrap() - .producer - }; - let limit_id = producer_of(dag.roots[0]); - assert_eq!(kind(limit_id), "limit"); - let sort_id = producer_of(limit_id); - assert_eq!(kind(sort_id), "sort"); - assert_eq!(producer_of(sort_id), producer_of(dag.roots[1])); - } - - /// With summaries chosen, every node, the summary build included, runs at - /// query time. - #[test] - fn everything_runs_at_query_time() { - let root = lower_promql( - "topk by (job) (10, sum_over_time(m[1m]))", - AccuracyTarget::max_error(0.01).failure_probability(0.001), - ); - let inventory = - asap_logical_asap_optimization::test_support::generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(root), - ]), - ) - .unwrap(); - // A summary (the last candidate) for every target. - let choice: Vec<_> = inventory - .targets - .iter() - .map(|t| t.choice_count() - 1) - .collect(); - let roots: Vec<_> = - asap_logical_asap_optimization::test_support::compose_logical_candidate( - &inventory, &choice, - ) - .unwrap() - .into_iter() - .collect(); - let candidate = first_candidate(&roots); - assert!(candidate - .dag - .nodes - .iter() - .any(|n| matches!(n.payload, Payload::ASAP(ASAPOp::SummaryAgg { .. })))); - assert!(candidate - .dag - .nodes - .iter() - .all(|n| n.output_state.timing == ExecutionTiming::QueryTime)); - } - - // ── Materialization ────────────────────────────────────────────────── - - use asap_types::workload::{DataArrival, RepetitionInterval, TimestampMs}; - - const MINUTE: u64 = 60_000; - - /// The composed logical candidate of `query` that builds its last - /// summary option (DDSketch for a quantile) for every target. - fn summary_root(query: &str) -> QueryRoot { - let root = lower_promql( - query, - AccuracyTarget::max_error(0.01) - .failure_probability(0.001) - .accuracy_metric(asap_types::ir::properties::AccuracyMetric::Rank), - ); - let inventory = - asap_logical_asap_optimization::test_support::generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(root), - ]), - ) - .unwrap(); - let choice: Vec<_> = inventory - .targets - .iter() - .map(|t| t.choice_count() - 1) - .collect(); - asap_logical_asap_optimization::test_support::compose_logical_candidate(&inventory, &choice) - .unwrap() - .remove(0) - } - - /// `root` with its summary build in 1-min tumbling windows. - fn in_tumbling_windows(root: QueryRoot) -> QueryRoot { - let roots = [root]; - let rule_candidates = - asap_logical_asap_optimization::test_support::tumbling_window_candidates( - &roots, - &[Some(MINUTE)], - ); - assert_eq!(rule_candidates.len(), 1, "{rule_candidates:?}"); - let mut pass2_candidates = - asap_logical_asap_optimization::test_support::window_composition_pass2_candidates( - &roots, - &rule_candidates, - ) - .unwrap(); - pass2_candidates.pop().unwrap().roots.unwrap().remove(0) - } - - const AT_INGESTION: Materialization = Materialization::MatAtIngestion(Medium::Memory); - - /// The options `candidate` chose other than `NotMaterialized`, by node. - fn materialized(candidate: &PhysicalWorkloadPlan) -> Vec { - candidate - .materialization - .values() - .copied() - .filter(|m| *m != Materialization::NotMaterialized) - .collect() - } - - // An unselected unpredictable query must not restrict an ingestion candidate. - #[test] - fn selected_reader_eligibility_ignores_optional_unpredictable_consumers() { - let root = in_tumbling_windows(summary_root("quantile_over_time(0.99, m[5m])")); - let node = root - .operator_refs() - .into_iter() - .flat_map(OperatorNode::reachable) - .find(|node| matches!(node.asap(), Some(ASAPOp::SummaryTumblingWindowChain { .. }))) - .unwrap(); - let entries = vec![ - every_minute(), - QueryWorkloadEntry { - predictability: Predictability::AdHoc, - ..every_minute() - }, - ]; - let data = arriving(); - let query_time_results_retained = false; - let selected = summary_materialization_eligibility( - &node, - &[0, 0], - &entries, - &data, - query_time_results_retained, - ) - .unwrap(); - assert_eq!(selected.at_ingestion, Ok(())); - assert!(matches!( - summary_materialization_eligibility(&node, &[0, entries.len()], &entries, &data, query_time_results_retained), - Err(SummaryEligibilityError::UnknownReader { query }) if query == entries.len() - )); - let combined = summary_materialization_eligibility( - &node, - &[0, 1], - &entries, - &data, - query_time_results_retained, - ) - .unwrap(); - assert_eq!( - combined.at_ingestion, - Err(IngestionPruneReason::NotPredictable) - ); - // The complete-DAG path uses the same selected-reader workload rules. - let candidates = - materialization_candidates(&[root], &entries, &data, query_time_results_retained); - let original = candidates - .iter() - .find(|candidate| { - matches!( - candidate.node.asap(), - Some(ASAPOp::SummaryTumblingWindowChain { .. }) - ) - }) - .unwrap(); - assert_eq!(original.at_ingestion, selected.at_ingestion); - assert_eq!( - original.may_keep_across_evaluations, - selected.may_keep_across_evaluations - ); - } - - // Phase and range-step restrictions apply only to the selected reader set. - #[test] - fn selected_reader_eligibility_checks_phase_and_range_compatibility() { - use asap_types::workload::RangeEvaluation; - let root = in_tumbling_windows(summary_root("quantile_over_time(0.99, m[5m])")); - let node = root - .operator_refs() - .into_iter() - .flat_map(OperatorNode::reachable) - .find(|node| matches!(node.asap(), Some(ASAPOp::SummaryTumblingWindowChain { .. }))) - .unwrap(); - let at = |phase| QueryWorkloadEntry { - recurrence: QueryRecurrence::Repeated(RepeatedDemand::FixedIntervalAt { - interval: RepetitionInterval(MINUTE as u32), - evaluation_phase: TimestampMs(phase), - }), - ..every_minute() - }; - let entries = vec![ - at(0), - at(MINUTE / 2), - QueryWorkloadEntry { - range_evaluation: Some(RangeEvaluation { - step_ms: MINUTE / 2, - span_ms: 10 * MINUTE, - }), - ..at(0) - }, - ]; - let query_time_results_retained = true; - let data = arriving(); - let one = summary_materialization_eligibility( - &node, - &[0], - &entries, - &data, - query_time_results_retained, - ) - .unwrap(); - assert!(one.may_keep_across_evaluations); - assert!(one.range_compatibility.is_ok()); - let different_phase = summary_materialization_eligibility( - &node, - &[0, 1], - &entries, - &data, - query_time_results_retained, - ) - .unwrap(); - assert!(!different_phase.may_keep_across_evaluations); - assert_eq!( - different_phase.at_ingestion, - Err(IngestionPruneReason::TumblingWindowsOffPhase) - ); - let off_grid = summary_materialization_eligibility( - &node, - &[0, 2], - &entries, - &data, - query_time_results_retained, - ) - .unwrap(); - assert!(matches!( - off_grid.range_compatibility, - Err(PhysicalASAPOptimizationError::TumblingWindowWidthOffRangeStep { .. }) - )); - assert!(matches!( - summary_materialization_eligibility( - &node, - &[], - &entries, - &data, - query_time_results_retained - ), - Err(SummaryEligibilityError::NoSelectedReaders) - )); - } - - /// A predictable query evaluated every minute. - fn every_minute() -> QueryWorkloadEntry { - QueryWorkloadEntry { - predictability: Predictability::Predictable { known_at: None }, - ..asap_logical_asap_optimization::test_support::entry(QueryRecurrence::Repeated( - RepeatedDemand::FixedInterval(RepetitionInterval(MINUTE as u32)), - )) - } - } - - fn arriving() -> DataWorkload { - DataWorkload { - arrival: DataArrival::Ingesting { - rows_per_second: 1_000.0, - }, - ..Default::default() - } - } - - /// The distinct nodes of `kind` the candidate's roots read. - fn node_of(candidate: &PhysicalWorkloadPlan, kind: &str) -> Vec> { - let mut nodes: Vec> = Vec::new(); - for root in candidate.roots.iter().flat_map(QueryRoot::operator_refs) { - for node in OperatorNode::reachable(root) { - if node.operator.kind_name() == kind && !nodes.iter().any(|n| Rc::ptr_eq(n, &node)) - { - nodes.push(node); - } - } - } - nodes - } - - /// #509 Example 4, B1 and B2: a tumbling-window chain read by a repeating, - /// predictable query over arriving data is either rebuilt at query time - /// or maintained at ingestion time. Maintained, the chain and the scan - /// below it run at ingestion time, and the merge and estimate at query - /// time. - #[test] - fn a_tumbling_window_chain_over_arriving_data_may_be_maintained_at_ingestion_time() { - let root = in_tumbling_windows(summary_root("quantile_over_time(0.99, m[5m])")); - let candidates = - optimize_physical_asap("L1", &[root], &[every_minute()], &arriving(), false).unwrap(); - assert_eq!(candidates.len(), 2); - let [b2, b1] = &candidates[..] else { - unreachable!() - }; - assert!(!b2.materializes_any()); - assert!(b2 - .dag - .nodes - .iter() - .all(|n| n.output_state.timing == ExecutionTiming::QueryTime)); - assert_eq!(materialized(b1), [AT_INGESTION]); - let [chain] = &node_of(b1, "SummaryTumblingWindowChain")[..] else { - panic!("one chain") - }; - assert_eq!( - chain.materialization, - Materialization::MatAtIngestion(Medium::Memory) - ); - for below in OperatorNode::reachable(chain) { - assert_eq!(below.timing, Some(ExecutionTiming::IngestionTime)); - } - for kind in ["SummaryMerge", "SummaryEstimate"] { - for node in node_of(b1, kind) { - assert_eq!(node.timing, Some(ExecutionTiming::QueryTime), "{kind}"); - } - } - } - - /// A chain of 1-min tumbling windows, anchored at the epoch, read by two queries - /// every minute is maintained at ingestion time when they evaluate on - /// its tumbling-window boundaries (phases 0 and 2 min, or one phase unknown), not - /// when one is at phase 30 s: each tumbling window is built once, so it would read - /// tumbling windows 30 s off. - #[test] - fn a_tumbling_window_chain_is_maintained_only_on_its_readers_phases() { - use IngestionPruneReason as Why; - let root = in_tumbling_windows(summary_root("quantile_over_time(0.99, m[5m])")); - let at = |phase_ms: u64| QueryWorkloadEntry { - recurrence: QueryRecurrence::Repeated(RepeatedDemand::FixedIntervalAt { - interval: RepetitionInterval(MINUTE as u32), - evaluation_phase: TimestampMs(phase_ms), - }), - ..every_minute() - }; - let at_ingestion = |demands: [QueryWorkloadEntry; 2]| { - let mut cache = HashMap::new(); - // One chain read by both queries. - let implemented = map_roots(&[root.clone(), root.clone()], &mut |n| { - implement(n, &mut cache) - }) - .unwrap(); - materialization_candidates(&implemented, &demands, &arriving(), false) - .into_iter() - .find(|c| { - matches!( - c.node.asap(), - Some(ASAPOp::SummaryTumblingWindowChain { .. }) - ) - }) - .expect("the chain is a materialization candidate") - .at_ingestion - }; - assert_eq!(at_ingestion([at(0), at(2 * MINUTE)]), Ok(())); - assert_eq!(at_ingestion([at(0), every_minute()]), Ok(())); - assert_eq!( - at_ingestion([at(0), at(30_000)]), - Err(Why::TumblingWindowsOffPhase) - ); - assert_eq!( - at_ingestion([at(30_000), every_minute()]), - Err(Why::TumblingWindowsOffPhase) - ); - } - - /// A chain of 1-min tumbling windows, anchored at the epoch, read every - /// 90 s ends a window at only every other evaluation (#1173), with or - /// without a declared phase on the grid: it is maintained at ingestion - /// time only when every reader's interval is a multiple of its width. - #[test] - fn a_tumbling_window_chain_is_maintained_only_on_its_readers_intervals() { - use IngestionPruneReason as Why; - let root = in_tumbling_windows(summary_root("quantile_over_time(0.99, m[5m])")); - let every = |interval_ms: u64| QueryWorkloadEntry { - recurrence: QueryRecurrence::Repeated(RepeatedDemand::FixedInterval( - RepetitionInterval(interval_ms as u32), - )), - ..every_minute() - }; - let every_at_phase_0 = |interval_ms: u64| QueryWorkloadEntry { - recurrence: QueryRecurrence::Repeated(RepeatedDemand::FixedIntervalAt { - interval: RepetitionInterval(interval_ms as u32), - evaluation_phase: TimestampMs(0), - }), - ..every_minute() - }; - let at_ingestion = |demands: [QueryWorkloadEntry; 2]| { - let mut cache = HashMap::new(); - // One chain read by both queries. - let implemented = map_roots(&[root.clone(), root.clone()], &mut |n| { - implement(n, &mut cache) - }) - .unwrap(); - materialization_candidates(&implemented, &demands, &arriving(), false) - .into_iter() - .find(|c| { - matches!( - c.node.asap(), - Some(ASAPOp::SummaryTumblingWindowChain { .. }) - ) - }) - .expect("the chain is a materialization candidate") - .at_ingestion - }; - assert_eq!(at_ingestion([every_minute(), every(2 * MINUTE)]), Ok(())); - assert_eq!( - at_ingestion([every_minute(), every(90_000)]), - Err(Why::TumblingWindowsOffInterval) - ); - assert_eq!( - at_ingestion([every_minute(), every_at_phase_0(90_000)]), - Err(Why::TumblingWindowsOffInterval) - ); - } - - /// S4: maintaining state at ingestion time needs arriving data and - /// queries that repeat and are predictable; otherwise the chain only - /// runs at query time, and the materialization candidate says why. - #[test] - fn ingestion_time_needs_arriving_data_and_repeated_predictable_queries() { - use IngestionPruneReason as Why; - let root = in_tumbling_windows(summary_root("quantile_over_time(0.99, m[5m])")); - let at_rest = DataWorkload { - arrival: DataArrival::AtRest, - ..Default::default() - }; - let unknown = QueryWorkloadEntry { - predictability: Predictability::Unknown, - ..every_minute() - }; - let once = QueryWorkloadEntry { - recurrence: QueryRecurrence::OneTime { - invocations: 1, - execute_at: None, - }, - ..every_minute() - }; - for (demand, data, why) in [ - (every_minute(), at_rest, Why::NotIngesting), - (every_minute(), DataWorkload::default(), Why::NotIngesting), - (unknown, arriving(), Why::NotPredictable), - (once, arriving(), Why::NotRepeated), - ] { - let roots = [root.clone()]; - let candidates = - optimize_physical_asap("L1", &roots, std::slice::from_ref(&demand), &data, false) - .unwrap(); - assert_eq!(candidates.len(), 1, "{why:?}"); - let mut cache = HashMap::new(); - let implemented = map_roots(&roots, &mut |n| implement(n, &mut cache)).unwrap(); - let mat_candidates = materialization_candidates(&implemented, &[demand], &data, false); - assert_eq!(mat_candidates.len(), 1); - assert_eq!(mat_candidates[0].at_ingestion, Err(why)); - } - } - - /// A counter state (`rate`, `increase`) needs each series' samples in - /// time order. Continuously ingesting time-series (PromQL) data is taken - /// to arrive in that order, as Prometheus assumes, so its counter tumbling windows - /// may be maintained at ingestion time. - #[test] - fn counter_tumbling_windows_over_ingesting_promql_data_run_at_ingestion_time() { - for query in ["rate(m[5m])", "increase(m[5m])"] { - let roots = [in_tumbling_windows(summary_root(query))]; - let mut cache = HashMap::new(); - let implemented = map_roots(&roots, &mut |n| implement(n, &mut cache)).unwrap(); - let mat_candidates = - materialization_candidates(&implemented, &[every_minute()], &arriving(), false); - assert_eq!(mat_candidates[0].at_ingestion, Ok(()), "{query}"); - let candidates = - optimize_physical_asap("L1", &roots, &[every_minute()], &arriving(), false) - .unwrap(); - assert_eq!(candidates.len(), 2, "{query}"); - } - } - - /// Only continuously ingesting time-series data is taken to arrive in - /// time order: a counter over a table source, or over mixed data (whose - /// at-rest part may be loaded in any order), stays at query time. - #[test] - fn counter_tumbling_windows_over_other_sources_stay_at_query_time() { - use IngestionPruneReason as Why; - let root = in_tumbling_windows(summary_root("rate(m[5m])")); - let QueryRoot::Operator(node) = &root else { - panic!("{root:?}") - }; - let table = QueryRoot::Operator(to_table(node)); - let mixed = DataWorkload { - arrival: DataArrival::Mixed { - rows_per_second: 1_000.0, - }, - ..Default::default() - }; - for (root, data) in [(table, arriving()), (root.clone(), mixed)] { - let roots = [root]; - let mut cache = HashMap::new(); - let implemented = map_roots(&roots, &mut |n| implement(n, &mut cache)).unwrap(); - let mat_candidates = - materialization_candidates(&implemented, &[every_minute()], &data, false); - assert_eq!( - mat_candidates[0].at_ingestion, - Err(Why::CounterNeedsTimeOrder) - ); - } - } - - /// `node` with every time-series scan turned into a table scan of the - /// same name and schema. - fn to_table(node: &Rc) -> Rc { - let mut out = (**node).clone(); - out.operator = node.operator.map_children(to_table); - if let Operator::NonASAP(NonASAPOp::Scan { source, .. }) = &mut out.operator { - if let Source::TimeSeries { metric } = source { - *source = Source::Table { - table_ref: metric.clone(), - }; - } - } - Rc::new(out) - } - - /// A whole-window build is maintained only for one window per - /// evaluation on the epoch grid of its width, where the backend builds - /// it: a fixed interval that is a multiple of the window, a phase (if - /// known) on that grid, and no filter. `m[1m]` every minute with no - /// declared phase is one tumbling 1-minute window per evaluation, taken - /// to run on the grid like a chain's reader of unknown phase; at phase - /// 30 s its windows would end off the grid, and every 30 s or every - /// 90 s some evaluations would. - #[test] - fn a_whole_window_build_needs_evaluations_on_the_epoch_grid_of_its_window() { - use IngestionPruneReason as Why; - let root = summary_root("quantile_over_time(0.99, m[1m])"); - let at_phase = |interval_ms: u32, phase_ms: u64| QueryWorkloadEntry { - recurrence: QueryRecurrence::Repeated(RepeatedDemand::FixedIntervalAt { - interval: RepetitionInterval(interval_ms), - evaluation_phase: TimestampMs(phase_ms), - }), - ..every_minute() - }; - let at = |interval_ms: u32| at_phase(interval_ms, 0); - let mat_candidates = |root: &QueryRoot, demand: QueryWorkloadEntry| { - let mut cache = HashMap::new(); - let implemented = map_roots(std::slice::from_ref(root), &mut |n| { - implement(n, &mut cache) - }) - .unwrap(); - materialization_candidates(&implemented, &[demand], &arriving(), false) - .remove(0) - .at_ingestion - }; - assert_eq!(mat_candidates(&root, at(MINUTE as u32)), Ok(())); - assert_eq!(mat_candidates(&root, every_minute()), Ok(())); - assert_eq!(mat_candidates(&root, at(2 * MINUTE as u32)), Ok(())); - assert_eq!( - mat_candidates(&root, at_phase(MINUTE as u32, 3 * MINUTE)), - Ok(()) - ); - assert_eq!( - mat_candidates(&root, at_phase(MINUTE as u32, 30_000)), - Err(Why::WindowOffEpochGrid) - ); - assert_eq!( - mat_candidates(&root, at(30_000)), - Err(Why::WindowOffEpochGrid) - ); - assert_eq!( - mat_candidates(&root, at(90_000)), - Err(Why::WindowOffEpochGrid) - ); - // Several readers of one build: a declared phase on the grid and an - // undeclared one agree; two declared phases, or two intervals, do not. - let mat_candidates_of = |demands: &[QueryWorkloadEntry]| { - let roots = vec![root.clone(); demands.len()]; - let mut cache = HashMap::new(); - let implemented = map_roots(&roots, &mut |n| implement(n, &mut cache)).unwrap(); - materialization_candidates(&implemented, demands, &arriving(), false) - .into_iter() - .find(|c| matches!(c.node.asap(), Some(ASAPOp::SummaryAgg { .. }))) - .expect("the whole-window build is a materialization candidate") - .at_ingestion - }; - assert_eq!( - mat_candidates_of(&[at_phase(MINUTE as u32, MINUTE), every_minute()]), - Ok(()) - ); - assert_eq!( - mat_candidates_of(&[ - at(MINUTE as u32), - at_phase(MINUTE as u32, 2 * MINUTE + MINUTE / 2) - ]), - Err(Why::WindowNotFixed) - ); - assert_eq!( - mat_candidates_of(&[at(MINUTE as u32), at(2 * MINUTE as u32)]), - Err(Why::WindowNotFixed) - ); - let candidates = optimize_physical_asap( - "L1", - std::slice::from_ref(&root), - &[at(MINUTE as u32)], - &arriving(), - false, - ) - .unwrap(); - assert_eq!(materialized(&candidates[1]), [AT_INGESTION]); - - // The same build with a filter. - let QueryRoot::Operator(node) = &root else { - unreachable!() - }; - let build = OperatorNode::reachable(node) - .into_iter() - .find(|n| matches!(n.asap(), Some(ASAPOp::SummaryAgg { .. }))) - .unwrap(); - let Operator::ASAP(ASAPOp::SummaryAgg { - child, - family, - input, - reduction, - grouping, - .. - }) = build.operator.clone() - else { - unreachable!() - }; - let filtered = Rc::new( - OperatorNode::with_schema( - Operator::ASAP(ASAPOp::SummaryAgg { - child, - family, - input, - reduction, - grouping, - filter: Some(asap_types::ir::Predicate(ScalarExpr::Literal( - asap_types::ir::scalar::expr_ir::ScalarValue::Boolean(true), - ))), - }), - build.schema.clone(), - ) - .with_guarantee(build.guarantee.clone()), - ); - assert_eq!( - mat_candidates(&QueryRoot::Operator(filtered), at(MINUTE as u32)), - Err(Why::Filtered) - ); - } - - /// A whole-window build read by a range evaluation of several steps per - /// request is not maintained at ingestion time: it would hold one - /// window per request, not one per step. A one-step range (`span_ms` - /// 0) is one evaluation per request and stays eligible. - #[test] - fn a_range_evaluated_whole_window_build_stays_at_query_time() { - use asap_types::workload::RangeEvaluation; - let root = summary_root("quantile_over_time(0.99, m[1m])"); - let at_minute = |span_ms: u64| QueryWorkloadEntry { - recurrence: QueryRecurrence::Repeated(RepeatedDemand::FixedIntervalAt { - interval: RepetitionInterval(MINUTE as u32), - evaluation_phase: TimestampMs(0), - }), - range_evaluation: Some(RangeEvaluation { - step_ms: MINUTE, - span_ms, - }), - ..every_minute() - }; - let at_ingestion = |entry: QueryWorkloadEntry| { - let mut cache = HashMap::new(); - let implemented = map_roots(std::slice::from_ref(&root), &mut |n| { - implement(n, &mut cache) - }) - .unwrap(); - materialization_candidates(&implemented, &[entry], &arriving(), false) - .remove(0) - .at_ingestion - }; - assert_eq!( - at_ingestion(at_minute(60 * MINUTE)), - Err(IngestionPruneReason::RangeEvaluatedWholeWindow) - ); - assert_eq!(at_ingestion(at_minute(0)), Ok(())); - } - - /// A tumbling-window chain whose width does not divide a range-evaluated reader's - /// step is not offered: the candidate fails with the widths, and the - /// caller prunes it. 1-min tumbling windows serve 2-min steps. - #[test] - fn a_tumbling_window_chain_off_the_range_step_grid_is_rejected() { - use asap_types::workload::RangeEvaluation; - let root = in_tumbling_windows(summary_root("quantile_over_time(0.99, m[5m])")); - let stepping = |step_ms: u64| QueryWorkloadEntry { - range_evaluation: Some(RangeEvaluation { - step_ms, - span_ms: 60 * MINUTE, - }), - ..every_minute() - }; - let implement_for = |entry: QueryWorkloadEntry| { - optimize_physical_asap( - "L1", - std::slice::from_ref(&root), - &[entry], - &arriving(), - false, - ) - }; - let error = implement_for(stepping(30_000)).unwrap_err(); - assert!( - matches!( - error, - PhysicalASAPOptimizationError::TumblingWindowWidthOffRangeStep { - tumbling_window_width_ms: MINUTE, - step_ms: 30_000, - } - ), - "{error}" - ); - assert_eq!( - error.to_string(), - "tumbling-window width 60000 ms does not divide the 30000 ms step of a range \ - evaluation that reads it" - ); - assert_eq!(implement_for(stepping(2 * MINUTE)).unwrap().len(), 2); - // A one-step range (span 0) is one evaluation per request, at the - // refresh: its step is not read. - let one_step = QueryWorkloadEntry { - range_evaluation: Some(RangeEvaluation { - step_ms: 30_000, - span_ms: 0, - }), - ..every_minute() - }; - assert_eq!(implement_for(one_step).unwrap().len(), 2); - } - - /// A build two queries read is also offered built once per evaluation - /// for both (`MatAtQuery`); a build one query reads is not. - #[test] - fn a_build_two_queries_read_may_be_built_once_per_evaluation() { - let QueryRoot::Operator(p99) = summary_root("quantile_over_time(0.99, m[1m])") else { - unreachable!() - }; - let build = OperatorNode::reachable(&p99) - .into_iter() - .find(|n| matches!(n.asap(), Some(ASAPOp::SummaryAgg { .. }))) - .unwrap(); - let p50 = Rc::new( - OperatorNode::new(Operator::ASAP(ASAPOp::SummaryEstimate { - summary_input: Rc::clone(&build), - query: asap_types::ir::schema::SketchStatistic::Quantile { q: 0.5 }, - })) - .unwrap(), - ); - let roots = [QueryRoot::Operator(p99), QueryRoot::Operator(p50)]; - let candidates = - optimize_physical_asap("L1", &roots, &[], &DataWorkload::default(), false).unwrap(); - assert_eq!(candidates.len(), 2); - assert_eq!( - materialized(&candidates[1]), - [Materialization::MatAtQuery(Medium::Memory)] - ); - let [build] = &node_of(&candidates[1], "SummaryAgg")[..] else { - panic!("one shared build") - }; - assert_eq!( - build.materialization, - Materialization::MatAtQuery(Medium::Memory) - ); - } - - /// Two queries share one scan: one reads it through tumbling windows maintained at - /// ingestion time, the other at query time. The scan is split, one copy - /// per timing; with nothing materialized it stays one node. The shared - /// scan may also be built once per evaluation (`MatAtQuery`), but not - /// under ingestion-time tumbling windows, where only one query reads it at query - /// time. - #[test] - fn a_scan_two_queries_read_at_two_timings_is_split() { - let QueryRoot::Operator(tumbling_windows) = - in_tumbling_windows(summary_root("quantile_over_time(0.99, m[5m])")) - else { - unreachable!() - }; - let scan = OperatorNode::reachable(&tumbling_windows) - .into_iter() - .find(|n| matches!(n.non_asap(), Some(NonASAPOp::Scan { .. }))) - .unwrap(); - let ranged = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeRange { - range: std::time::Duration::from_millis(5 * MINUTE), - kind: asap_types::ir::TimeRangeKind::Range, - child: Rc::clone(&scan), - })) - .unwrap(); - let exact = asap_logical_asap_optimization::test_support::aggregate( - Reduction::by(vec![]), - vec![AggIntent::Max { col: None }], - vec![], - None, - ranged, - ); - let roots = [ - QueryRoot::Operator(tumbling_windows), - QueryRoot::Operator(exact), - ]; - let candidates = optimize_physical_asap( - "L1", - &roots, - &[every_minute(), every_minute()], - &arriving(), - false, - ) - .unwrap(); - assert_eq!(candidates.len(), 3); - let scans = |c: &PhysicalWorkloadPlan| { - let mut scans: Vec<_> = c - .dag - .nodes - .iter() - .filter(|n| matches!(n.payload, Payload::NonASAP(NonASAPOp::Scan { .. }))) - .map(|n| (n.output_state.timing, n.materialization)) - .collect(); - scans.sort_by_key(|(t, _)| *t == ExecutionTiming::QueryTime); - scans - }; - let not_materialized = Materialization::NotMaterialized; - assert_eq!( - scans(&candidates[0]), - [(ExecutionTiming::QueryTime, not_materialized)] - ); - assert_eq!( - scans(&candidates[1]), - [( - ExecutionTiming::QueryTime, - Materialization::MatAtQuery(Medium::Memory) - )] - ); - assert_eq!( - scans(&candidates[2]), - [ - (ExecutionTiming::IngestionTime, not_materialized), - (ExecutionTiming::QueryTime, not_materialized) - ] - ); - } - - /// Only the highest shared non-summary nodes are materialization - /// candidates, each with `NotMaterialized` and `MatAtQuery` only: a - /// 5 m range two queries read (not the scan below it, which only that - /// shared range reads for them), and the scan, which a third query - /// also reads through its own 1 m range. - #[test] - fn only_the_highest_shared_non_summary_nodes_are_candidates() { - let QueryRoot::Operator(p99) = summary_root("quantile_over_time(0.99, m[5m])") else { - unreachable!() - }; - let scan = OperatorNode::reachable(&p99) - .into_iter() - .find(|n| matches!(n.non_asap(), Some(NonASAPOp::Scan { .. }))) - .unwrap(); - let ranged = |ms: u64| { - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeRange { - range: std::time::Duration::from_millis(ms), - kind: asap_types::ir::TimeRangeKind::Range, - child: Rc::clone(&scan), - })) - .unwrap() - }; - let aggregate = |intent: AggIntent, input: Rc| { - QueryRoot::Operator(asap_logical_asap_optimization::test_support::aggregate( - Reduction::by(vec![]), - vec![intent], - vec![], - None, - input, - )) - }; - let five_minutes = ranged(5 * MINUTE); - let roots = [ - aggregate(AggIntent::Max { col: None }, Rc::clone(&five_minutes)), - aggregate(AggIntent::Min { col: None }, five_minutes), - aggregate(AggIntent::Max { col: None }, ranged(MINUTE)), - ]; - let mat_candidates = - materialization_candidates(&roots, &[], &DataWorkload::default(), false); - let found: Vec<_> = mat_candidates - .iter() - .map(|c| (c.node.operator.kind_name(), c.readers.clone(), c.options())) - .collect(); - let options = vec![ - Materialization::NotMaterialized, - Materialization::MatAtQuery(Medium::Memory), - ]; - assert_eq!( - found, - [ - ("TimeRange", vec![0, 1], options.clone()), - ("Scan", vec![0, 1, 2], options) - ] - ); - } - - /// Readers share one kept tumbling-window chain only when they evaluate at the - /// same instants: two readers every minute at phases 0 and 2 min do, - /// at phases 0 and 30 s they do not (each would need its own newest - /// tumbling window). A reader with no known phase is assumed to run with the - /// others. - #[test] - fn readers_at_different_phases_do_not_build_one_tumbling_window_per_evaluation() { - let at = |phase_ms: u64| { - QueryRecurrence::Repeated(RepeatedDemand::FixedIntervalAt { - interval: RepetitionInterval(MINUTE as u32), - evaluation_phase: TimestampMs(phase_ms), - }) - }; - let unknown_phase = every_minute().recurrence; - assert!(builds_one_tumbling_window_per_evaluation( - MINUTE, - [&at(0), &at(2 * MINUTE)] - )); - assert!(!builds_one_tumbling_window_per_evaluation( - MINUTE, - [&at(0), &at(30_000)] - )); - assert!(builds_one_tumbling_window_per_evaluation( - MINUTE, - [&at(30_000), &unknown_phase] - )); - assert!(!builds_one_tumbling_window_per_evaluation( - MINUTE, - [&at(0), &unknown_phase, &at(30_000)] - )); - } - - // A known evaluation phase is reduced modulo the width; a bare interval has - // none; a schedule has one only when all its times share it. - #[test] - fn tumbling_window_phase_of_a_repeated_demand() { - use asap_types::workload::{RepetitionInterval, TimestampMs}; - let at = |phase| RepeatedDemand::FixedIntervalAt { - interval: RepetitionInterval(60_000), - evaluation_phase: TimestampMs(phase), - }; - let scheduled = |times: &[u64]| { - RepeatedDemand::Scheduled(times.iter().copied().map(TimestampMs).collect()) - }; - let phase = |demand| plan_tumbling_window_phase(&demand, 60_000); - assert_eq!(phase(at(56_000)), Some(56_000)); - assert_eq!(phase(at(176_000)), Some(56_000)); - assert_eq!( - phase(RepeatedDemand::FixedInterval(RepetitionInterval(60_000))), - None - ); - assert_eq!(phase(scheduled(&[56_000, 176_000])), Some(56_000)); - assert_eq!(phase(scheduled(&[56_000, 150_000])), None); - } -} diff --git a/crates/physical-asap-optimization/src/split_by_timing.rs b/crates/physical-asap-optimization/src/split_by_timing.rs deleted file mode 100644 index 72dad6d63..000000000 --- a/crates/physical-asap-optimization/src/split_by_timing.rs +++ /dev/null @@ -1,375 +0,0 @@ -//! Splitting a shared sub-DAG by execution timing before Stage 2 applies a -//! materialization assignment ([`split_shared_by_timing`]). - -use std::collections::HashMap; -use std::rc::Rc; - -use asap_types::ir::properties::ExecutionTiming; -use asap_types::ir::{timing_under_consumer, MaterializationAssignment, OperatorNode}; - -/// Each (node, timing) pair's rebuilt node, with the original it copies. -type Copies = HashMap<(*const OperatorNode, ExecutionTiming), (Rc, Rc)>; - -/// Copy, for one consumer, every sub-DAG that `assignment` would reach with -/// two different timings from `roots` (one query each, or the roots of one -/// query), so that a workload whose CSE shared a `Scan` between an -/// ingestion-time summary and a query-time computation, in one query or -/// across queries, can still be timed. Only the conflicting sub-DAGs are -/// copied, one copy per timing; a sub-DAG reached with one timing stays one -/// `Rc`, shared by the roots that read it. Returns the (possibly rewritten) -/// roots, in order, and `assignment` remapped onto them: a rebuilt node -/// keeps its original's explicit materialization. -pub fn split_shared_by_timing( - roots: &[Rc], - assignment: &MaterializationAssignment, -) -> (Vec>, MaterializationAssignment) { - // First pass: the set of timings each node is reached with. - let mut reached: HashMap<*const OperatorNode, Vec> = HashMap::new(); - let mut forced = HashMap::new(); - fn collect( - node: &Rc, - consumer: ExecutionTiming, - assignment: &MaterializationAssignment, - reached: &mut HashMap<*const OperatorNode, Vec>, - forced: &mut HashMap<*const OperatorNode, bool>, - ) { - let timing = timing_under_consumer(node, consumer, assignment, forced); - let entry = reached.entry(Rc::as_ptr(node)).or_default(); - if entry.contains(&timing) { - return; - } - entry.push(timing); - for child in node.children() { - collect(child, timing, assignment, reached, forced); - } - } - for root in roots { - collect( - root, - ExecutionTiming::QueryTime, - assignment, - &mut reached, - &mut forced, - ); - } - if reached.values().all(|timings| timings.len() <= 1) { - return (roots.to_vec(), assignment.clone()); - } - // Second pass: rebuild, giving each (node, timing) pair its own copy, - // kept with its original. - let mut copies: Copies = HashMap::new(); - fn rebuild( - node: &Rc, - consumer: ExecutionTiming, - assignment: &MaterializationAssignment, - reached: &HashMap<*const OperatorNode, Vec>, - copies: &mut Copies, - forced: &mut HashMap<*const OperatorNode, bool>, - ) -> Rc { - let timing = timing_under_consumer(node, consumer, assignment, forced); - let key = (Rc::as_ptr(node), timing); - if let Some((_, done)) = copies.get(&key) { - return Rc::clone(done); - } - let conflicted = reached - .get(&Rc::as_ptr(node)) - .is_some_and(|timings| timings.len() > 1); - let mut changed = conflicted; - let operator = node.operator.map_children(|child| { - let rebuilt = rebuild(child, timing, assignment, reached, copies, forced); - changed |= !Rc::ptr_eq(&rebuilt, child); - rebuilt - }); - let out = if changed { - Rc::new( - OperatorNode::with_schema(operator, node.schema.clone()) - .with_guarantee(node.guarantee.clone()) - .with_timing(node.timing) - .with_materialization(node.materialization), - ) - } else { - Rc::clone(node) - }; - copies.insert(key, (Rc::clone(node), Rc::clone(&out))); - out - } - let split = roots - .iter() - .map(|root| { - rebuild( - root, - ExecutionTiming::QueryTime, - assignment, - &reached, - &mut copies, - &mut forced, - ) - }) - .collect(); - let mut remapped = assignment.clone(); - for (original, copy) in copies.values() { - if let Some(materialization) = assignment.get(original) { - remapped.set(copy, materialization); - } - } - (split, remapped) -} - -#[cfg(test)] -mod tests { - use super::*; - use asap_types::ir::operator::agg_intent::AggIntent; - use asap_types::ir::operator::operator_properties::{Reduction, Source}; - use asap_types::ir::properties::{ExecutionDataState, ExecutionDataStateError}; - use asap_types::ir::scalar::expr_ir::ColumnRef; - use asap_types::ir::schema::sketch::{ - ExactKind, ExactParams, GroupingStrategy, SketchAlgorithm, SketchKind, SketchParams, - SketchStatistic, SummaryUpdate, - }; - use asap_types::ir::schema::{DataType, Field, FieldDataType, Schema}; - use asap_types::ir::{ - apply_materialization_timings, ASAPOp, NodeToTimedNodeCache, NonASAPOp, Operator, - }; - use asap_types::physical::{Materialization, Medium}; - - const AT_INGESTION: Materialization = Materialization::MatAtIngestion(Medium::Memory); - - fn scan() -> Rc { - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Scan { - source: Source::TimeSeries { metric: "m".into() }, - predicates: vec![], - schema: Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("value", DataType::Float64, false), - Field::plain("zone", DataType::Utf8, true), - ], - 0, - vec![], - ), - })) - .unwrap() - } - - fn kll() -> FieldDataType { - FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: 200, - failure_probability: None, - }, - ), - GroupingStrategy::default(), - ) - } - - fn exact_sum() -> FieldDataType { - FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum) - } - - fn agg(child: Rc, family: FieldDataType) -> Rc { - Rc::new( - OperatorNode::with_schema( - Operator::ASAP(ASAPOp::SummaryAgg { - child, - family: family.clone(), - input: SummaryUpdate::column(ColumnRef::SampleValue), - reduction: Reduction::by(vec![]), - grouping: GroupingStrategy::default(), - filter: None, - }), - Schema::lifted(vec![Field::new("state", family, false)], None), - ) - .with_guarantee(None), - ) - } - - fn estimate(child: Rc) -> Rc { - Rc::new( - OperatorNode::with_schema( - Operator::ASAP(ASAPOp::SummaryEstimate { - summary_input: child, - query: SketchStatistic::Quantile { q: 0.99 }, - }), - Schema::lifted( - vec![Field::plain("quantile_0_99", DataType::Float64, false)], - None, - ), - ) - .with_guarantee(None), - ) - } - - fn max(child: Rc) -> Rc { - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Aggregate { - reduction: Reduction::by(vec![]), - measures: vec![AggIntent::Max { col: None }], - output_names: vec![], - filters: vec![], - having: None, - child, - })) - .unwrap() - } - - fn concat(children: Vec>) -> Rc { - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Concat { children })).unwrap() - } - - fn finalize(child: Rc) -> Rc { - OperatorNode::new_shared(Operator::ASAP(ASAPOp::FinalizeExactAccumulator { child })) - .unwrap() - } - - fn child(node: &Rc) -> Rc { - Rc::clone(node.children()[0]) - } - - /// One shared sub-DAG reached as ingestion-time input and as query-time - /// input cannot be executed once for both; splitting it by timing first - /// makes the plan timeable. - #[test] - fn a_shared_subtree_reached_at_two_timings_conflicts() { - let shared = scan(); - let root = OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Concat { - children: vec![ - max(estimate(agg(Rc::clone(&shared), kll()))), - max(Rc::clone(&shared)), - ], - })) - .unwrap(); - assert_eq!( - apply_materialization_timings( - &root, - &MaterializationAssignment::with_default(AT_INGESTION), - &mut NodeToTimedNodeCache::new() - ) - .err(), - Some(ExecutionDataStateError::ConflictingTiming { - first: ExecutionDataState::INGESTION_ROWS, - second: ExecutionDataState::QUERY_ROWS, - }) - ); - let (split, assignment) = split_shared_by_timing( - std::slice::from_ref(&root), - &MaterializationAssignment::with_default(AT_INGESTION), - ); - assert!(apply_materialization_timings( - &split[0], - &assignment, - &mut NodeToTimedNodeCache::new() - ) - .is_ok()); - } - - /// Splitting rebuilds the ancestors of a copied sub-DAG; an explicit - /// assignment on one of them follows it onto the rebuilt node. - #[test] - fn split_keeps_explicit_assignments() { - let shared = scan(); - let summary = agg(Rc::clone(&shared), kll()); - let root = OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Concat { - children: vec![max(estimate(Rc::clone(&summary))), max(Rc::clone(&shared))], - })) - .unwrap(); - let mut assignment = MaterializationAssignment::default(); - assignment.set(&summary, AT_INGESTION); - let (split, assignment) = split_shared_by_timing(std::slice::from_ref(&root), &assignment); - let timed = - apply_materialization_timings(&split[0], &assignment, &mut NodeToTimedNodeCache::new()) - .unwrap(); - let summary = child(&child(&child(&timed))); - assert!(matches!( - summary.operator, - Operator::ASAP(ASAPOp::SummaryAgg { .. }) - )); - assert_eq!(summary.materialization, AT_INGESTION); - assert_eq!(summary.timing, Some(ExecutionTiming::IngestionTime)); - } - - /// Two queries share a scan: one reads it through a summary built at - /// ingestion time, the other at query time. Splitting the workload's - /// roots together copies the scan for one phase, so one timing pass - /// over both roots succeeds; a node both queries reach at one timing - /// stays one node. - #[test] - fn split_across_roots_copies_only_what_two_queries_read_at_two_timings() { - let shared = scan(); - let summary = agg(Rc::clone(&shared), kll()); - let by_summary = max(estimate(Rc::clone(&summary))); - let raw = max(Rc::clone(&shared)); - let common = max(estimate(agg(scan(), kll()))); - let roots = [ - concat(vec![Rc::clone(&by_summary), Rc::clone(&common)]), - concat(vec![raw, Rc::clone(&common)]), - ]; - let mut assignment = MaterializationAssignment::default(); - assignment.set(&summary, AT_INGESTION); - let timed_unsplit = { - let mut cache = NodeToTimedNodeCache::new(); - roots - .iter() - .map(|root| apply_materialization_timings(root, &assignment, &mut cache)) - .collect::, _>>() - }; - assert!(matches!( - timed_unsplit, - Err(ExecutionDataStateError::ConflictingTiming { .. }) - )); - let (split, assignment) = split_shared_by_timing(&roots, &assignment); - assert_eq!(split.len(), 2); - let mut cache = NodeToTimedNodeCache::new(); - let timed: Vec<_> = split - .iter() - .map(|root| apply_materialization_timings(root, &assignment, &mut cache).unwrap()) - .collect(); - let summary = child(&child(&child(&timed[0]))); - assert_eq!(summary.materialization, AT_INGESTION); - assert_eq!(child(&summary).timing, Some(ExecutionTiming::IngestionTime)); - assert_eq!( - child(&child(&timed[1])).timing, - Some(ExecutionTiming::QueryTime) - ); - assert!(Rc::ptr_eq(timed[0].children()[1], timed[1].children()[1])); - } - - /// Splitting a workload that also shares a scan at two timings keeps a - /// conflicting assignment on a fixed-timing node: the split roots still - /// fail with the conflict instead of dropping the assignment. - #[test] - fn a_fixed_timing_conflict_survives_the_split() { - let shared = scan(); - let summary = agg(Rc::clone(&shared), kll()); - let pair = concat(vec![ - max(estimate(Rc::clone(&summary))), - max(Rc::clone(&shared)), - ]); - let state = agg(Rc::clone(&shared), exact_sum()); - let read = Rc::new( - (*finalize(Rc::clone(&state))) - .clone() - .with_timing(Some(ExecutionTiming::QueryTime)), - ); - let mut assignment = MaterializationAssignment::default(); - assignment.set(&summary, AT_INGESTION); - assignment.set(&state, AT_INGESTION); - assignment.set(&read, AT_INGESTION); - let (split, assignment) = split_shared_by_timing(&[pair, read], &assignment); - let mut cache = NodeToTimedNodeCache::new(); - let timed: Result, _> = split - .iter() - .map(|root| apply_materialization_timings(root, &assignment, &mut cache)) - .collect(); - assert_eq!( - timed.err(), - Some( - ExecutionDataStateError::MaterializationConflictsWithFixedTiming { - operator: "FinalizeExactAccumulator", - fixed: ExecutionTiming::QueryTime, - materialization: AT_INGESTION, - } - ) - ); - } -} diff --git a/crates/plan-selection/Cargo.toml b/crates/plan-selection/Cargo.toml index ac64258a5..ab85b889e 100644 --- a/crates/plan-selection/Cargo.toml +++ b/crates/plan-selection/Cargo.toml @@ -9,33 +9,12 @@ edition = "2021" # the facade's `AssignmentRealizer`: no dependency on Stages 1 or 2, nor on # the executor (#1054 X1, #1047 P7). Tests build their inputs with them. [dependencies] -asap_sketchlib = { workspace = true } asap-types = { path = "../types" } asap-accuracy = { path = "../accuracy" } asap-cost = { path = "../cost" } thiserror = "2" serde = { version = "1", features = ["derive"] } serde_json = "1" -rmp-serde = "1" good_lp = { version = "=1.15.3", default-features = false, features = ["highs"] } highs = "=2.4.0" - -[features] -# `reference_pricer`: the whole-workload selector over enumerated plans, for -# the developer tools' reference pricing (`asap-devtools`). -reference-pricer = [] -# `lp-diagnosis`: with `ASAP_STAGE3_LP_DIAGNOSIS=`, each selection MILP -# solve also reports where its LP relaxation loses its bound (measurement -# only, for the developer tools; a diagnosed run's timings are not -# comparable with an undiagnosed one's). -lp-diagnosis = [] - -[dev-dependencies] -asap-types = { path = "../types", features = ["test-support"] } -asap-accuracy = { path = "../accuracy", features = ["test-support"] } -asap-logical-asap-optimization = { path = "../logical-asap-optimization", features = ["test-support"] } -asap-physical-asap-optimization = { path = "../physical-asap-optimization", features = ["test-support"] } -asap-frontend-promql = { path = "../frontend-promql" } -# The executor's charge for each summary state, checked against the bytes -# plan selection prices for it plus the accumulator's own fields (#947). diff --git a/crates/plan-selection/src/accuracy_binding.rs b/crates/plan-selection/src/accuracy_binding.rs deleted file mode 100644 index 853440ad0..000000000 --- a/crates/plan-selection/src/accuracy_binding.rs +++ /dev/null @@ -1,306 +0,0 @@ -//! Binding the statistics a root guarantee leaves symbolic: from the data -//! workload and the operand-domain contracts. - -use std::collections::HashMap; -use std::rc::Rc; - -use asap_accuracy::composition::{ - parse_raw_data_samples_sign_statistic, parse_source_norm_statistic, INPUT_ROW_COUNT, - PRODUCT_OPERAND_DOMAINS, STREAM_L1_NORM, -}; -use asap_accuracy::estimators::{hll, univmon}; -use asap_accuracy::propagation::{ - operand_domain_statistic, operand_guarantee, operand_of, project_column_guarantee, - root_guarantee, -}; -use asap_accuracy::OperandDomain; -use asap_types::ir::operator::operator_properties::BinaryOpKind; -use asap_types::ir::properties::{AccuracyGuarantee, AccuracyMetric}; -use asap_types::ir::scalar::ScalarValue; -use asap_types::ir::{BinaryOperator, NonASAPOp, OperatorNode, QueryRoot, ScalarExpr}; -use asap_types::workload::DataWorkload; - -use crate::SelectionModels; -use asap_types::workload::bounds::declared_non_negative; - -/// [`root_guarantee`] with the statistics Stage 1 left symbolic bound: -/// - the input row count, `series` ([`series_read`](asap_types::workload::bounds::series_read) of `root`); -/// - each source's stream norm ([`source_norm_statistic`]), from the data -/// workload's per-source `stream_norms`; -/// - classic HLL's failure probability and the chance a UnivMon's top layer -/// is incomplete, from the data workload's per-source -/// `max_distinct_per_evaluation`; -/// - a product's or quotient's bound, from the deployment's enforced -/// operand domains ([`operand_domain_bounds`]). -/// -/// Every data-workload fact is declared: the caller drops facts that no -/// longer hold before planning. Any other statistic stays unknown, and an -/// unknown statistic never meets a target. -pub(crate) fn bound_root_guarantee( - root: &QueryRoot, - series: Option, - data: &DataWorkload, - models: &SelectionModels<'_>, -) -> Option { - let domains = operand_domain_bounds(root, series, data, models); - let bind = |guarantee: &AccuracyGuarantee| { - bind_statistics(guarantee, &|statistic| { - workload_statistic(statistic, series, data) - .or_else(|| domains.get(statistic).copied().flatten()) - }) - }; - match root_guarantee(root) { - Some(guarantee) => Some(bind(&guarantee)), - None => asap_accuracy::propagation::bind_guarantees(root, &bind), - } -} - -/// `guarantee` with the statistics `value` supplies bound. A sign gate -/// ([`raw_data_samples_sign_statistic`](asap_accuracy::composition::raw_data_samples_sign_statistic)) -/// is an indicator that only 1 (proven) opens: any other value leaves it -/// unbound, so it can never read as a perfect bound. -pub(crate) fn bind_statistics( - guarantee: &AccuracyGuarantee, - value: &dyn Fn(&str) -> Option, -) -> AccuracyGuarantee { - guarantee.bind(&|statistic| { - let supplied = value(statistic); - match parse_raw_data_samples_sign_statistic(statistic) { - Some(_) => supplied.filter(|v| *v == 1.0), - None => supplied, - } - }) -} - -/// A data-workload statistic Stage 1 left symbolic: the input row count -/// (`series`, the series of the sources the query reads), one source's -/// stream norm, or classic HLL's failure probability or a UnivMon's -/// incomplete-top-layer probability given one source's distinct bound, -/// from source facts that hold at any planning time, or the sign of one -/// metric's samples, from its declared type. -pub(crate) fn workload_statistic( - statistic: &str, - series: Option, - data: &DataWorkload, -) -> Option { - // The sign of one metric's samples: proven (1) when the metric is - // declared a counter or declared non-negative; nothing is inferred from - // its name. - if let Some(metric) = parse_raw_data_samples_sign_statistic(statistic) { - return declared_non_negative(data, metric).then_some(1.0); - } - if statistic == INPUT_ROW_COUNT { - return series.map(|rows| rows as f64); - } - if let Some((precision, relative_error, source)) = - hll::parse_classic_failure_statistic(statistic) - { - let bound = data.sources.get(source)?.max_distinct_per_evaluation?; - return Some(hll::classic_failure_probability( - precision, - relative_error, - bound, - )); - } - if let Some((heap_size, layers, source)) = - univmon::parse_incomplete_top_layer_statistic(statistic) - { - let bound = data.sources.get(source)?.max_distinct_per_evaluation?; - return Some(univmon::incomplete_top_layer_probability( - heap_size, layers, bound, - )); - } - let (norm, source) = parse_source_norm_statistic(statistic)?; - let norms = data.sources.get(source)?.stream_norms?; - match norm { - STREAM_L1_NORM => norms.l1, - _ => norms.l2, - } -} - -/// One operand of a product or quotient, as Stage 3 finds its domain. -enum DomainOperand<'a> { - /// A planned operator: the deployment's `OperandDomain` for it. - Node(&'a Rc), - /// A literal: itself. - Literal(f64), - /// Anything else: no domain. - Other, -} - -impl<'a> DomainOperand<'a> { - fn of(expr: &'a ScalarExpr) -> Self { - match expr { - ScalarExpr::PromqlScalarFromVector(node) | ScalarExpr::ScalarSubquery(node) => { - Self::Node(node) - } - ScalarExpr::Literal(ScalarValue::Float64(value)) => Self::Literal(*value), - ScalarExpr::Literal(ScalarValue::Int64(value)) => Self::Literal(*value as f64), - _ => Self::Other, - } - } - - fn domain(&self, models: &SelectionModels<'_>) -> Option { - match self { - Self::Node(node) => models.evidence.operand_domain(node), - Self::Literal(value) => Some(OperandDomain { - lower: *value, - upper: *value, - }), - Self::Other => None, - } - } -} - -/// A product or quotient whose bound Stage 1 left symbolic on its operands' -/// domains ([`operand_domain_statistic`]). -struct DomainOperation<'a> { - statistic: &'static str, - operands: [(DomainOperand<'a>, AccuracyGuarantee); 2], -} - -/// Each operand-domain statistic in `root`, bound to the largest bound -/// among the operations that leave it symbolic, or `None` (unbound) when -/// any of them has no bound. Operations are found where the bound -/// originates (the same rule Stage 1 uses), in vector arithmetic, a scalar -/// root, and projection expressions. `series` is the root's input row -/// count ([`series_read`](asap_types::workload::bounds::series_read)). -fn operand_domain_bounds( - root: &QueryRoot, - series: Option, - data: &DataWorkload, - models: &SelectionModels<'_>, -) -> HashMap<&'static str, Option> { - let mut operations = Vec::new(); - if let QueryRoot::Scalar(expr) = root { - scalar_domain_operations(expr, &|_| None, &mut operations); - } - let nodes: Vec> = root - .operator_refs() - .iter() - .flat_map(|node| OperatorNode::reachable(node)) - .collect(); - for node in &nodes { - match node.non_asap() { - Some(NonASAPOp::BinaryOp { - operator: - BinaryOperator { - kind: BinaryOpKind::Arithmetic(op), - .. - }, - lhs, - rhs, - .. - }) => { - let (Some(l), Some(r)) = (operand_guarantee(lhs), operand_guarantee(rhs)) else { - continue; - }; - if let Some(statistic) = operand_domain_statistic(op, &l, &r) { - operations.push(DomainOperation { - statistic, - operands: [(DomainOperand::Node(lhs), l), (DomainOperand::Node(rhs), r)], - }); - } - } - Some(NonASAPOp::Project { cols, child, .. }) => { - for item in cols { - scalar_domain_operations( - &item.expr, - &|column| project_column_guarantee(child, column), - &mut operations, - ); - } - } - _ => {} - } - } - let mut bounds: HashMap<&'static str, Option> = HashMap::new(); - for operation in &operations { - let bound = domain_bound(operation, series, data, models); - let entry = bounds.entry(operation.statistic).or_insert(Some(0.0)); - *entry = entry.zip(bound).map(|(a, b)| a.max(b)); - } - bounds -} - -/// The operations in `expr` (and its sub-expressions) whose bound Stage 1 -/// left symbolic on operand domains. -fn scalar_domain_operations<'a>( - expr: &'a ScalarExpr, - column: &dyn Fn(usize) -> Option, - operations: &mut Vec>, -) { - if let ScalarExpr::Arithmetic { - op, left, right, .. - } = expr - { - if let (Some(l), Some(r)) = (operand_of(left, column), operand_of(right, column)) { - if let Some(statistic) = operand_domain_statistic(op, &l, &r) { - operations.push(DomainOperation { - statistic, - operands: [(DomainOperand::of(left), l), (DomainOperand::of(right), r)], - }); - } - } - } - for child in expr.children() { - scalar_domain_operations(child, column, operations); - } -} - -/// The bound of one product or quotient from its operands' domains: -/// - a product: `|x|·e_y + |y|·e_x + e_x·e_y`; -/// - a quotient: `(|x|·e_y + |y|·e_x) / (m·(m − e_y))` with `m` the least -/// `|y|`, which must exceed `e_y` (the denominator's domain is bounded -/// away from zero). -/// -/// `|x|`, `|y|` are the domains' largest magnitudes, and `e` each -/// operand's absolute error (a relative or cardinality bound times that -/// magnitude) after binding its data-workload statistics (`series`, the -/// root's input row count). `None` when an operand has no domain or an -/// unbound error. -fn domain_bound( - operation: &DomainOperation<'_>, - series: Option, - data: &DataWorkload, - models: &SelectionModels<'_>, -) -> Option { - let [(x, gx), (y, gy)] = &operation.operands; - let (dx, dy) = (x.domain(models)?, y.domain(models)?); - let bind = - |g: &AccuracyGuarantee| g.bind(&|statistic| workload_statistic(statistic, series, data)); - let (gx, gy) = (bind(gx), bind(gy)); - let valid = - |d: &OperandDomain| d.lower.is_finite() && d.upper.is_finite() && d.lower <= d.upper; - if !(valid(&dx) && valid(&dy)) { - return None; - } - let magnitude = |d: &OperandDomain| d.lower.abs().max(d.upper.abs()); - let (mx, my) = (magnitude(&dx), magnitude(&dy)); - let error = |g: &AccuracyGuarantee, m: f64| { - let bound = g.spec.error.evaluate()?; - match g.spec.accuracy_metric { - _ if g.is_exact() => Some(0.0), - AccuracyMetric::ABSOLUTE => Some(bound), - AccuracyMetric::Relative => Some(bound * m), - _ => None, - } - }; - let (ex, ey) = (error(&gx, mx)?, error(&gy, my)?); - let bound = if operation.statistic == PRODUCT_OPERAND_DOMAINS { - mx * ey + my * ex + ex * ey - } else { - let least = if dy.lower > 0.0 { - dy.lower - } else if dy.upper < 0.0 { - -dy.upper - } else { - return None; - }; - if least <= ey { - return None; - } - (mx * ey + my * ex) / (least * (least - ey)) - }; - bound.is_finite().then_some(bound) -} diff --git a/crates/plan-selection/src/checks.rs b/crates/plan-selection/src/checks.rs deleted file mode 100644 index 3ac348d12..000000000 --- a/crates/plan-selection/src/checks.rs +++ /dev/null @@ -1,305 +0,0 @@ -//! Validity checks on one candidate: accuracy targets and latency. What -//! the executor or deployment cannot run (per-entity aggregates without a -//! kernel, query-time scans of raw data not retained) Stage 2 prunes. - -use asap_cost::{family_name, PlanCost, QueryLatency}; -use asap_types::ir::properties::{ - AccuracyGuarantee, AccuracyMetric, AccuracyScope, AccuracyTarget, -}; -use asap_types::ir::schema::{FieldDataType, SketchAlgorithm, WeightDomain}; -use asap_types::ir::{ASAPOp, Operator, OperatorNode, QueryRoot}; -use asap_types::physical::PhysicalWorkloadPlan; -use asap_types::workload::{DataWorkload, QueryWorkloadEntry}; - -use crate::accuracy_binding::bound_root_guarantee; -use crate::report::{AcceptedAccuracy, RejectionCause}; -use crate::SelectionModels; -use asap_types::workload::bounds::{confirmed_weights, series_read, values_per_result}; - -/// The first query whose latency per evaluation is over its -/// `max_latency_ms` bound or, with no bound broken, over its repeat -/// interval (a query evaluated every `I` ms whose evaluation takes longer -/// than `I` falls further behind with every evaluation), as a rejection -/// cause and reason. -pub(crate) fn latency_violation( - cost: &PlanCost, - entries: &[QueryWorkloadEntry], -) -> Option<(RejectionCause, String)> { - let per_query = || cost.usage.latency_per_query.iter().zip(entries).enumerate(); - let latency_text = |query: usize, latency: &QueryLatency| { - format!( - "q{}: latency {:.1} ms per evaluation (compaction {:.1} ms + query {:.1} ms)", - query + 1, - latency.total_ms(), - latency.compaction_ms, - latency.query_ms - ) - }; - let over_bound = - per_query().find_map( - |(query, (latency, entry))| match entry.requirements.max_latency_ms { - Some(bound) if latency.total_ms() > bound => Some(( - RejectionCause::LatencyBoundExceeded, - format!( - "{} is over its {bound} ms bound", - latency_text(query, latency) - ), - )), - _ => None, - }, - ); - over_bound.or_else(|| { - per_query().find_map(|(query, (latency, entry))| { - match entry.recurrence.fixed_interval_ms() { - Some(interval_ms) if latency.total_ms() > interval_ms as f64 => Some(( - RejectionCause::LatencyOverRepeatInterval, - format!( - "{} is over its {interval_ms} ms repeat interval, so its evaluations \ - cannot keep up", - latency_text(query, latency) - ), - )), - _ => None, - } - }) - }) -} - -/// Each entry's accuracy target, as the per-query targets Stage 3 checks. -pub(crate) fn accuracy_targets(entries: &[QueryWorkloadEntry]) -> Vec { - entries - .iter() - .map(|entry| entry.requirements.accuracy.clone()) - .collect() -} - -/// What Stage 3 accepted for each query ([`AcceptedAccuracy`]), or the -/// first query whose output is not shown to meet its target, as a reason. An output meets its target when its theoretical root guarantee -/// (composed in Stage 1 with the built-in model) satisfies it, or when the -/// deployment's empirical evidence shows that this candidate's output does -/// ([`AccuracyEvidenceProvider::output_guarantee`]). A -/// certified estimate inside the plan is not enough on its own: a difference -/// or ratio of certified estimates can be unbounded. The root guarantee's -/// symbolic statistics are bound first ([`bound_root_guarantee`]); one still -/// unknown leaves the target unmet. Count-Min is admitted only over update -/// weights proven non-negative, which its bound and the executor assume: an open -/// [`WeightDomain::RawDataSamplesOfMetric`] requirement is met when `data` declares -/// the metric a counter or non-negative ([`confirmed_weights`]), and unmet -/// otherwise. -/// -/// A target that sets no metric accepts a guarantee in any metric that meets -/// its ε, δ and scope ([`AccuracyModel::satisfies`]); one that sets a metric -/// accepts only guarantees in it (or convertible to it). -/// -/// The Count-Min weight check runs first and rejects as -/// [`RejectionCause::CountMinWeightsNotProvenNonNegative`]: the executor -/// cannot run such a plan. Then the deployment's accuracy model is asked -/// ([`AccuracyModel::accepts_output_regardless_of_guarantee`], through -/// `SelectionModels::accuracy`): an output it accepts is accepted -/// whatever its guarantee says, and the guarantee check's rejection, if any, -/// is only recorded ([`AcceptedAccuracy::guarantee_check_rejection`]). -/// Otherwise, an output with no guarantee from either -/// source (Stage 1 had no rule, and the evidence has none) is rejected as -/// [`RejectionCause::NoAccuracyGuarantee`], and any other miss as -/// [`RejectionCause::AccuracyTargetMissed`]. -pub(crate) fn accuracy_check( - candidate: &PhysicalWorkloadPlan, - targets: &[AccuracyTarget], - data: &DataWorkload, - models: &SelectionModels<'_>, -) -> Result, CheckFailure> { - let mut accepted = Vec::with_capacity(targets.len()); - for (query, (root, target)) in candidate.roots.iter().zip(targets).enumerate() { - let query = query + 1; - let mut sketches: Vec = Vec::new(); - for node in root - .operator_refs() - .iter() - .flat_map(|node| OperatorNode::reachable(node)) - { - let Operator::ASAP( - ASAPOp::SummaryAgg { family, input, .. } - | ASAPOp::SummaryTumblingWindowChain { family, input, .. }, - ) = &node.operator - else { - continue; - }; - if let FieldDataType::Sketch(kind, _) = family { - let name = format!("{:?}", kind.algorithm()); - if !sketches.contains(&name) { - sketches.push(name); - } - } - if matches!(family, FieldDataType::Sketch(kind, _) - if matches!(kind.algorithm(), SketchAlgorithm::Cms | SketchAlgorithm::CmsWithHeap)) - && !matches!( - confirmed_weights(&input.weight_domain, data), - WeightDomain::NonNegative { .. } - ) - { - let unconfirmed = match &input.weight_domain { - WeightDomain::RawDataSamplesOfMetric { metric } => format!( - " (samples of `{metric}`, which is not declared a counter or \ - non-negative)" - ), - WeightDomain::CounterDerivativeOfMetric { metric } => format!( - " (rates or increases of `{metric}`, which is not declared a counter)" - ), - _ => String::new(), - }; - // Count-Min (plain, CmsWithHeap or Hydra Count-Min) over - // weights not proven non-negative does not run: the executor - // refuses CmsWithHeap and HydraCms when it builds the plan, - // and plain Count-Min adds a negative weight, which voids its - // never-underestimate bound. So - // this rejects the plan whatever the deployment's accuracy - // model accepts. - return Err(CheckFailure { - cause: RejectionCause::CountMinWeightsNotProvenNonNegative, - reason: format!( - "q{query}: {} needs non-negative update weights, and these are not \ - proven non-negative{unconfirmed}", - family_name(family) - ), - size_independent: true, - }); - } - } - // The guarantee check: the theoretical guarantee, then the - // deployment's evidence. - let checked = guarantee_check(query, root, target, data, models); - // The deployment's model may accept the output directly, whatever - // its guarantee says; the check's result is then only recorded. - if models - .accuracy - .accepts_output_regardless_of_guarantee(root, target) - { - accepted.push(AcceptedAccuracy { - target_metric: target.accuracy_metric, - accepted_metric: None, - accepted_regardless_of_guarantee: true, - guarantee_check_rejection: checked.err().map(|failure| failure.cause), - sketches, - }); - continue; - } - accepted.push(AcceptedAccuracy { - target_metric: target.accuracy_metric, - accepted_metric: checked?, - accepted_regardless_of_guarantee: false, - guarantee_check_rejection: None, - sketches, - }); - } - Ok(accepted) -} - -/// A check a candidate failed: why, with its numbers, and whether every -/// size of the same plan's summaries fails it too. -#[derive(Debug, Clone, PartialEq)] -pub(crate) struct CheckFailure { - pub(crate) cause: RejectionCause, - pub(crate) reason: String, - /// `true` when no other size of the plan's summaries can pass the check - /// (e.g. no rule composes a guarantee, or a bound stays unknown) and the - /// deployment's accuracy model and evidence cannot judge another size - /// otherwise; `false` when that is not proven. - pub(crate) size_independent: bool, -} - -/// Whether query `query`'s output `root` meets `target` by its theoretical -/// root guarantee or, failing that, the deployment's evidence -/// ([`accuracy_check`]): the metric of the guarantee that met it (`None` -/// for an exact output), or why not, as -/// [`RejectionCause::NoAccuracyGuarantee`] or -/// [`RejectionCause::AccuracyTargetMissed`]. -pub(crate) fn guarantee_check( - query: usize, - root: &QueryRoot, - target: &AccuracyTarget, - data: &DataWorkload, - models: &SelectionModels<'_>, -) -> Result, CheckFailure> { - // A guarantee per value is stated over the values the target's scope - // covers: the whole result's `G`, or at most `⌊1 / share⌋` groups. - let series = series_read(root, data); - let values = values_per_result(root, series, data); - let over_scope = |g: AccuracyGuarantee| g.over_scope(target.accuracy_scope, values); - let guarantee = bound_root_guarantee(root, series, data, models).map(over_scope); - let meets = |g: &AccuracyGuarantee| models.accuracy.satisfies(g, target); - // The deployment's evidence is read only when the theoretical - // guarantee does not meet the target. - let evidence = || models.evidence.output_guarantee(root).map(over_scope); - let (met, evidence) = match guarantee.clone().filter(meets) { - Some(met) => (Some(met), None), - None => { - let evidence = evidence(); - (evidence.clone().filter(meets), evidence) - } - }; - if let Some(met) = met { - return Ok((!met.is_exact()).then_some(met.spec.accuracy_metric)); - } - // Whether the deployment's model or evidence may judge another size of - // the plan's summaries otherwise. - let model_or_evidence_may_judge = models.accuracy.may_accept_regardless_of_guarantee() - || models.evidence.may_provide_evidence(); - if guarantee.is_none() && evidence.is_none() { - return Err(CheckFailure { - cause: RejectionCause::NoAccuracyGuarantee, - reason: format!( - "q{query}: the output has no guarantee, and the deployment has no evidence \ - for it, so it cannot meet target {target:?}" - ), - // No rule composes one, whatever the sizes. - size_independent: !model_or_evidence_may_judge, - }); - } - // A bound or failure probability that stays unknown once bound is - // unknown at every size; a known one may shrink with larger summaries. - let unknown = |g: &AccuracyGuarantee| { - g.spec.error.evaluate().is_none() - || (target.failure_probability.is_some() - && g.spec.failure_probability.evaluate().is_none()) - }; - let size_independent = evidence.is_none() - && guarantee.as_ref().is_some_and(unknown) - && !model_or_evidence_may_judge; - Err(CheckFailure { - cause: RejectionCause::AccuracyTargetMissed, - size_independent, - reason: match guarantee { - Some(g) - if values.is_none() - && target.accuracy_scope == AccuracyScope::WholeResult - && target.failure_probability.is_some() - && g.spec.failure_probability.evaluate().is_none() => - { - format!( - "q{query}: the output's guarantee holds for each value, and the number \ - of values one evaluation returns is not bound (no declared series \ - count for a source a PromQL vector reads, and no declared \ - `distinct_values_per_label` for every group key nor a \ - `distinct_values_per_label_set` covering them, or a key that is not \ - an unchanged source column), so it does not meet \ - whole-result target {target:?}, and \ - the deployment has no evidence that it meets it (a target on the groups \ - holding a share of the records, `groups_with_record_share_at_least`, \ - covers a bound number of values)" - ) - } - Some(g) => format!( - "q{query}: the output guarantees a {:?} bound {:?}, failure probability \ - {:?}, which misses target {target:?}, and the deployment has no evidence \ - that it meets it", - g.spec.accuracy_metric, - g.spec.error.evaluate(), - g.spec.failure_probability.evaluate(), - ), - None => format!( - "q{query}: the output has no guarantee, and the deployment has no \ - evidence that it meets target {target:?}" - ), - }, - }) -} diff --git a/crates/plan-selection/src/groups.rs b/crates/plan-selection/src/groups.rs deleted file mode 100644 index 1421f5d2a..000000000 --- a/crates/plan-selection/src/groups.rs +++ /dev/null @@ -1,2163 +0,0 @@ -//! #1095 C3: what the owner-scoped candidate MILP (§2.0) is built from. Given -//! Stage 2's physical candidates (one per operator node, timing and -//! materialization; Stage 2 makes no copies), it computes: -//! -//! 1. **Candidate dominance** ([`prune_dominated_candidates`]): a candidate no optimal -//! plan under cost model 1 uses is deleted, with its links. -//! 2. **Per-candidate data** ([`CandidateData`]): whether the candidate outputs -//! approximate values and the budget parts it is sized for, and its usage -//! per execution, standing cost, held bytes and own latency, every number -//! from [`asap_cost::node_usage`]. -//! 3. **Owners** ([`OwnerRegion`]): one per query and one per possible -//! instance of each shareable candidate, each with its region of stateless -//! candidates, the links inside it and the shareable candidates it may read. -//! 4. **Accuracy weights** ([`AccuracyWeight`]) per query and approximate -//! candidate, and pairs excluded together, from the composition of the -//! query's outputs. -//! -//! Nothing calls it yet; the MILP (C4) will. -//! -//! ## Usage per execution and its context -//! -//! A candidate's usage is [`asap_cost::node_usage`] of its operator node at one -//! execution per second (`rate = (1, 1.0)`); its standing cost is the same -//! call's cost at no execution (`rate = (1, 0.0)`), and its execution cost -//! the difference. `node_usage` also needs facts the candidate alone does not -//! fix, its *context*: -//! - the outputs of the producers it reads (one per input; the producer -//! candidates of an input may output different statistics); -//! - for a query-time scan, the window each reader reads, through time -//! shifts ([`window_read_through`], which follows -//! [`PhysicalASAPDAG::scan_window`] for a plan with that one reader); -//! - for a pane chain kept beyond one evaluation, the panes it keeps, which -//! grow with the range-evaluation span of the queries reaching it -//! ([`asap_cost::tumbling_window_retention_ms`], [`asap_cost::retained_tumbling_windows`]). -//! -//! Every combination of these options is costed; a combination -//! `asap-cost` cannot cost is skipped -//! (no plan can use it). When the numbers agree, the usage is -//! plan-independent; otherwise [`CandidateData::usage_depends_on`] says which -//! options changed it, and [`CandidateData::usage`] holds the first costed -//! combination's numbers. An ingestion-time node's maintained window is -//! read from the first producer candidate of each input -//! ([`asap_cost::maintained_window_ms`]). -//! -//! Candidate links can close a cycle (on #1036 workload-01, a widened scan's -//! `Project` that reads its own memo group). Candidates are costed producers first; on a -//! cycle, a candidate is costed from its producers already costed and marked -//! [`UsageContext::ProducerOnACycle`]. - -//! ## Dominance (exact under cost model 1) -//! -//! Candidate `k` is deleted when another candidate `k'` (the *dominating* candidate): -//! - is in the same memo group, at the same timing and materialization, -//! with the same held state; -//! - has the same producer candidates per input, the same readers (every -//! (reader candidate, input) that may read `k` may read `k'`, and nothing -//! else), the same output references it may produce, the same reader -//! constraints and keep flags; -//! - has the same `(approx, parts)`; -//! - has a guarantee at least as strong ([`guarantee_at_least_as_strong`]); -//! - outputs the same schema, sources, state family and unchanged columns -//! with an operator of the same kind (pane chains: the same panes), so its -//! readers are costed with the same formulas; -//! - and has every number at most `k`'s: CPU operations and scanned bytes -//! per execution, latency (work per execution), output rows and bytes, -//! kept memory (average and peak), disk bytes, held bytes and raw data -//! kept. -//! -//! Both candidates' usage must be plan-independent (otherwise the comparison -//! would hold for one context only). Readers' usage is assumed -//! non-decreasing in their inputs' rows and bytes. Among candidates that -//! dominate each other the one with the smallest id is kept. Every link to -//! a deleted candidate is deleted; its readers still read the dominating candidate. -//! -//! ## Accuracy weights -//! -//! `w(q, k)` says whether query `q` may read approximate candidate `k` (a -//! sketch [`ASAPOp::SummaryAgg`] or [`ASAPOp::SummaryTumblingWindowChain`], what -//! [`asap_accuracy::propagation::approximate_inputs`] counts), and pairs of -//! approximate candidates `q` may not read together. Both come from the -//! composition itself, never from an operator table -//! ([`accuracy_positions`]): `q`'s outputs are built over the candidates -//! with `k` approximate and every other candidate exact, their guarantees -//! composed by `asap-accuracy` ([`asap_accuracy::propagation::composed_guarantee`]) -//! and checked by the same check the selected plan passes -//! ([`crate::checks::guarantee_check`], with the deployment's model): -//! - every output exact: `k`'s values do not reach the output (only a row -//! count does), and `k` gets no weight; -//! - every output meets the target: `w(q, k) = 1`, one share of `q`'s -//! budget ([`asap_accuracy::budget`]); `k`'s sizes are those of its -//! `parts`, so a lone denominator sized for the whole target -//! (`ε/(1 − ε) > ε`) fails here; -//! - otherwise [`AccuracyWeight::Unusable`], with the check's cause -//! ([`UnusableReason`]): e.g. a rank error through any arithmetic, or an -//! HLL whose declared distinct bound is outside its contract. -//! -//! For each pair `k1, k2` of candidates usable alone, both sized for at -//! least two shares, whose values meet in one plan of `q` (under two inputs -//! of one candidate, under two references of a scalar root, or one below -//! the other), the outputs are built and checked with both approximate; a -//! pair that fails is excluded (`a[q, k1] + a[q, k2] ≤ 1`). A pair with a -//! member sized for one share is excluded by the shares row already. The -//! rows still bound the shares a plan reads by each candidate's `parts`, -//! which the equal split proves for one product or quotient of two inputs; -//! three or more approximate inputs are not built together. -use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; - -use asap_cost::{CostInputs, NodeUsage, ProducerOutput}; -use asap_types::ir::materialized_view::RefreshWindow; -use asap_types::ir::physical_export::{ - PhysicalASAPDAG, PhysicalASAPDAGNode, PhysicalASAPNodeId, ScanWindow, -}; -use asap_types::ir::properties::execution_data_state::{DataPrimitive, ExecutionDataState}; -use asap_types::ir::properties::{AccuracyGuarantee, ExecutionTiming}; -use asap_types::ir::schema::{FieldDataType, Schema}; -use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode}; -use asap_types::physical::Materialization; -use asap_types::stages::{ - output_references, MemoGroupId, OperatorNodeOfCandidateForMemoGroupId, - PhysicalASAPMemoCandidates, -}; -use asap_types::workload::{DataWorkload, QueryWorkloadEntry}; - -use crate::SelectionModels; - -use asap_types::stages::{PhysicalCandidateId, ReaderConstraint}; - -/// What the MILP is built from (module documentation). -#[derive(Debug, Clone)] -pub(crate) struct MilpInputs { - /// Indexed by [`PhysicalCandidateId`]: each candidate's data, pruned or not. - pub(crate) candidate_data: Vec, - /// The candidates dominance deleted, ascending. - pub(crate) pruned_candidates: Vec, - /// Indexed by [`PhysicalCandidateId`]: per input, the producer candidates left - /// after pruning (empty for a pruned candidate). - pub(crate) producer_candidates_per_input: Vec>>, - /// Per output reference (`output_references` of the query roots): the - /// candidates left that may produce it, `Out(r)`, ascending. - pub(crate) output_candidates: Vec>, - /// Indexed by [`PhysicalCandidateId`]: where it runs. - pub(crate) kind: Vec, - /// Indexed by [`PhysicalCandidateId`]: the result it gives. - pub(crate) slot: Vec, - /// The execution contexts: one per query (`0..Q`, in workload order), - /// then the ingestion pipelines `I_r` some context reads, ascending by - /// [`IngestionPipeline`], then one per shareable candidate some context - /// may read, ascending by candidate. - pub(crate) contexts: Vec, - /// Per shareable candidate with a context, its index in `contexts`. - pub(crate) context_of_shareable: BTreeMap, - /// Per ingestion-time shareable candidate whose context reads a - /// candidate of an `I_r`: the index of that `I_r` in `contexts`. - pub(crate) ingestion_context_of_shareable: BTreeMap, - /// Per query: `w(q, k)` for each approximate candidate `k` whose values - /// reach its output. - pub(crate) accuracy_weights: Vec>, - /// Per query: the pairs `(k1, k2)`, `k1 < k2`, of candidates usable alone - /// that it may not read together. - pub(crate) accuracy_excluded_pairs: Vec>, -} - -/// One candidate's data for the MILP. -#[derive(Debug, Clone)] -pub(crate) struct CandidateData { - /// Whether it outputs approximate values: a sketch `SummaryAgg` or - /// `SummaryTumblingWindowChain`, each an approximate input of the queries reading it. - pub(crate) approximate: bool, - /// `parts(k)`: its operator node's `accuracy_budget_parts` (Stage 2's - /// `OperatorNodeOfCandidateForMemoGroup`: for a factored node, the - /// largest among its candidates). - pub(crate) accuracy_budget_parts: usize, - /// Its usage under the first context, or why `asap-cost` could not cost - /// it. - pub(crate) usage: Result, - /// The context options that changed its usage, ascending; empty when it - /// is plan-independent. - pub(crate) usage_depends_on: Vec, - /// The inputs whose producer (or that producer's output) changes its - /// usage or its own output, ascending. - pub(crate) inputs_changing_usage: Vec, - /// Its usage and its own output (an index below `output_count`) per - /// choice of a producer and that producer's output for each of - /// `inputs_changing_usage` (in that order); `None` when `asap-cost` could - /// not cost that choice, so no plan can make it. Empty when - /// `inputs_changing_usage` is. - pub(crate) usage_per_producers: Vec<(Vec, Option)>, - /// For a query-time scan whose usage depends on the window its reader - /// reads ([`UsageContext::ScanWindowOfReader`]): its usage per window it - /// can be costed at, with its output there. Empty otherwise. - pub(crate) usage_per_window: Vec<(ScanWindow, CandidateUsage, usize)>, - /// How many distinct outputs (statistics its readers are costed from) - /// it can give, by context. - pub(crate) output_count: usize, - /// For a query-time scan: per reader, the windows it may read the scan - /// at (through time shifts, [`window_read_through`]). Empty for any - /// other candidate. - pub(crate) windows_per_reader: Vec<(PhysicalCandidateId, Vec)>, - /// For a kept pane chain whose usage depends on the panes it retains - /// ([`UsageContext::PaneRetentionOfReachingQueries`]): its usage per - /// retained pane count. Empty otherwise. - pub(crate) usage_per_retained_panes: Vec<(u64, CandidateUsage)>, - /// With `usage_per_retained_panes`: per query that can reach it, the - /// panes it needs retained. - pub(crate) retained_panes_per_query: Vec<(usize, u64)>, - /// Whether some entry of `usage_per_producers` stands for several usages - /// (a producer whose own output depends on its context), of which the - /// costliest is kept. - pub(crate) producer_usage_ambiguous: bool, -} - -impl Default for CandidateData { - fn default() -> Self { - Self { - approximate: false, - accuracy_budget_parts: 1, - usage: Err("not costed".to_string()), - usage_depends_on: Vec::new(), - inputs_changing_usage: Vec::new(), - usage_per_producers: Vec::new(), - usage_per_window: Vec::new(), - output_count: 1, - windows_per_reader: Vec::new(), - usage_per_retained_panes: Vec::new(), - retained_panes_per_query: Vec::new(), - producer_usage_ambiguous: false, - } - } -} - -/// One candidate's usage, from [`asap_cost::node_usage`] (module -/// documentation). -#[derive(Debug, Clone, Copy, PartialEq)] -pub(crate) struct CandidateUsage { - /// `u(k)`: CPU operations of one execution. - pub(crate) cpu_ops_per_execution: f64, - /// `u(k)`: bytes one execution scans. - pub(crate) scanned_bytes_per_execution: f64, - /// Milliseconds of one execution on one core. - pub(crate) work_ms_per_execution: f64, - /// `d_k`: what it adds to a reaching query's latency in - /// [`asap_cost::plan_cost`], its work per execution at query time; 0 - /// at ingestion time, whose latency is the plan-level compaction term. - pub(crate) own_latency_ms: f64, - /// Cost model 1: the cost of one execution; `None` under other models. - pub(crate) cost_per_execution: Option, - /// Cost model 1: the cost per second of what it keeps whatever runs it - /// (kept memory and disk); `None` under other models. - pub(crate) standing_cost_per_second: Option, - /// Memory kept between executions, averaged over time. - pub(crate) average_memory_bytes: f64, - pub(crate) peak_memory_bytes: u64, - pub(crate) disk_bytes: u64, - /// Query-time memory held while an evaluation reading it runs (a - /// `MatAtQuery` state, or a not-materialized node's working memory). - pub(crate) held_bytes: u64, - /// Raw data a query-time scan keeps on disk. - pub(crate) raw_data_kept_bytes: u64, - /// An in-memory ingestion-time state's bytes per window. - pub(crate) maintained_window_bytes: u64, - pub(crate) output_rows: u64, - pub(crate) output_bytes: u64, -} - -/// A candidate's usage and its output (an index below -/// [`CandidateData::output_count`]) in one context. -pub(crate) type UsageAndOutput = (CandidateUsage, usize); - -/// One producer choice for an input: a producer candidate, and for one -/// with several outputs ([`CandidateData::output_count`]), which output. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(crate) struct ProducerChoice { - pub(crate) class: PhysicalCandidateId, - pub(crate) output: Option, -} - -/// A context option that changed a candidate's usage. -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub(crate) enum UsageContext { - /// Which producer candidate (of different output) an input reads. - ProducerOutput, - /// Which reader reads a query-time scan, so which window it scans. - ScanWindowOfReader, - /// Which queries reach a kept pane chain, so how many panes it keeps. - PaneRetentionOfReachingQueries, - /// A producer candidate on a cycle of candidate links was not costed yet, so - /// its outputs were not among the options. - ProducerOnACycle, - /// `asap-cost` could not cost some contexts (e.g. a scan read with no - /// window from a source without `rows_held`); the usage is the first - /// costed one's. - UncostableContext, -} - -/// A candidate dominance deleted. -#[derive(Debug, Clone, PartialEq, Eq)] -pub(crate) struct DominatedCandidate { - pub(crate) class: PhysicalCandidateId, - pub(crate) dominated_by: PhysicalCandidateId, - pub(crate) reason: String, -} - -/// `w(q, k)`: one share of `q`'s budget, or unusable. -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub(crate) enum AccuracyWeight { - One, - /// `∞`: `k` is unusable for `q`. - Unusable(UnusableReason), -} - -/// Why a position's weight is `∞`: the check of `q`'s outputs with `k` -/// approximate failed. -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub(crate) enum UnusableReason { - /// Some output has no guarantee - /// ([`RejectionCause::NoAccuracyGuarantee`](crate::report::RejectionCause::NoAccuracyGuarantee)). - NoAccuracyGuarantee, - /// Some output's guarantee misses the target - /// ([`RejectionCause::AccuracyTargetMissed`](crate::report::RejectionCause::AccuracyTargetMissed)). - AccuracyTargetMissed, - /// Not checked, so not usable: a node that does not rebuild over its - /// producers. - NotChecked, -} - -/// Where a candidate runs (#1095 §2.0, "Memo groups and execution -/// contexts"), by its materialization alone, as `asap_cost::plan_cost` -/// counts its executions: -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub(crate) enum RunKind { - /// `NotMaterialized` at query time: computed once in each execution - /// context (a query evaluation, or a shared query-time candidate's - /// inputs) that selects it. - PerContext, - /// `NotMaterialized` at ingestion time: runs in the ingestion pipeline - /// `I_r` of its readers' refresh window `r`, once per second of - /// arriving data for all of them. - Ingestion, - /// Materialized (`MatAtIngestion`, `MatAtQuery`, - /// `MatAtQueryKeptAcrossEvaluations`): selected once, its own execution - /// context for its inputs, charged once. - Shareable, -} - -/// The [`RunKind`] of a candidate at `timing` and `materialization`. -pub(crate) fn run_kind(timing: ExecutionTiming, materialization: Materialization) -> RunKind { - match (materialization, timing) { - (Materialization::NotMaterialized, ExecutionTiming::QueryTime) => RunKind::PerContext, - (Materialization::NotMaterialized, ExecutionTiming::IngestionTime) => RunKind::Ingestion, - _ => RunKind::Shareable, - } -} - -/// What "one candidate per memo group per context" (row 2) ranges over: -/// the candidates that give one result. A memo group's implementation -/// roots give the group's result; an implementation's private nodes -/// (`is_implementation_root == false`) give an intermediate result of -/// their own, so each private operator node is its own slot (its -/// candidates differ only in timing and materialization). -/// -/// The memo group graph is not acyclic: a widened scan puts a `Project` in -/// the memo group of the scan it reads, so a group's candidate may read -/// its own group. Such a candidate and its producer cannot share one slot -/// (one context would need both), so a group's roots are split by -/// *self-read depth*: 0 for a root reading no root of its own group (the -/// scan), 1 for one that may (the `Project`). A depth-1 root reading -/// another depth-1 root of its group in one context (a `Project` over a -/// `Project` of the widened scan) is then a read of its own slot, which -/// [`CandidateLinks`] drops for candidates that run in a context. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub(crate) enum ResultSlot { - Group(MemoGroupId, u32), - PrivateNode(OperatorNodeOfCandidateForMemoGroupId), -} - -/// The [`ResultSlot`] of every candidate (see there). -pub(crate) fn result_slots(physical: &PhysicalASAPMemoCandidates) -> Vec { - let count = physical.candidates.len(); - let root = |candidate: usize| { - let operator_node = physical.candidates[candidate].operator_node; - let node = &physical.operator_nodes_of_candidates_for_memo_groups[operator_node.0]; - node.is_implementation_root - .then(|| node.group()) - .ok_or(operator_node) - }; - // Self-read depth: 1 for a root that may read a root of its own group. - let depth: Vec = (0..count) - .map(|candidate| { - let Ok(group) = root(candidate) else { - return 0; - }; - let reads_own_group = physical.candidates[candidate] - .producer_candidates_per_input - .iter() - .flat_map(|producers| producers.iter()) - .any(|producer| producer.0 != candidate && root(producer.0) == Ok(group)); - u32::from(reads_own_group) - }) - .collect(); - (0..count) - .map(|candidate| match root(candidate) { - Ok(group) => ResultSlot::Group(group, depth[candidate]), - Err(operator_node) => ResultSlot::PrivateNode(operator_node), - }) - .collect() -} - -/// Per candidate of a [`PhysicalASAPMemoCandidates`]: its producer -/// candidates per input, ascending and without repeats, and its -/// [`RunKind`]. -struct CandidateLinks { - producers_per_input: Vec>>, - kind: Vec, -} - -impl CandidateLinks { - fn new(physical: &PhysicalASAPMemoCandidates) -> Self { - let producers_per_input: Vec>> = physical - .candidates - .iter() - .map(|candidate| { - candidate - .producer_candidates_per_input - .iter() - .map(|producers| { - let set: BTreeSet<_> = producers.iter().copied().collect(); - set.into_iter().collect() - }) - .collect() - }) - .collect(); - let kind: Vec = physical - .candidates - .iter() - .map(|candidate| run_kind(candidate.timing, candidate.materialization)) - .collect(); - // With one candidate per result slot per context, a candidate that - // runs in a context cannot read its own slot there (only a self-loop - // could): such links are dropped (a widened scan's `Project` lists - // its own memo group). A shareable candidate reads in its own - // context, so it keeps them, but never reads itself. - let slot = result_slots(physical); - let producers_per_input = producers_per_input - .into_iter() - .enumerate() - .map(|(reader, inputs): (usize, Vec>)| { - inputs - .into_iter() - .map(|producers| { - producers - .into_iter() - .filter(|producer| { - producer.0 != reader - && (kind[reader] == RunKind::Shareable - || slot[producer.0] != slot[reader]) - }) - .collect() - }) - .collect() - }) - .collect(); - Self { - producers_per_input, - kind, - } - } - - /// Whether `candidate` is materialized: selected once, its own context. - fn is_shareable(&self, candidate: PhysicalCandidateId) -> bool { - self.kind[candidate.0] == RunKind::Shareable - } -} - -/// Per candidate, its reader constraints, each as an id equal for equal -/// constraints once the constrained candidate is left out, ascending and -/// without repeats. -fn reader_constraint_ids_per_candidate(physical: &PhysicalASAPMemoCandidates) -> Vec> { - let mut ids: HashMap = HashMap::new(); - let mut per_candidate = vec![Vec::new(); physical.candidates.len()]; - for constraint in &physical.reader_constraints { - let mut unowned = constraint.clone(); - *unowned.candidate_mut() = PhysicalCandidateId(usize::MAX); - let next = ids.len() as u32; - per_candidate[constraint.candidate().0].push(*ids.entry(unowned).or_insert(next)); - } - for constraint_ids in &mut per_candidate { - constraint_ids.sort_unstable(); - constraint_ids.dedup(); - } - per_candidate -} - -/// [`MilpInputs`] for `physical`'s candidates; `entries[q]` is query `q`'s -/// workload entry. Accuracy is checked with the built-in model and no -/// evidence ([`prepare_milp_inputs_with`]). -#[cfg(test)] -pub(crate) fn prepare_milp_inputs( - physical: &PhysicalASAPMemoCandidates, - entries: &[QueryWorkloadEntry], - data: &DataWorkload, - cost_inputs: &CostInputs, -) -> MilpInputs { - let models = SelectionModels { - accuracy: &asap_accuracy::BuiltinAccuracyModel, - evidence: &asap_accuracy::NoAccuracyEvidence, - cost_inputs, - }; - prepare_milp_inputs_with(physical, entries, data, &models) -} - -/// [`MilpInputs`] for `physical`'s candidates, its accuracy positions -/// checked with `models` (the deployment's accuracy model and evidence) and -/// its usage costed with `models.cost_inputs`. -pub(crate) fn prepare_milp_inputs_with( - physical: &PhysicalASAPMemoCandidates, - entries: &[QueryWorkloadEntry], - data: &DataWorkload, - models: &SelectionModels<'_>, -) -> MilpInputs { - let cost_inputs = models.cost_inputs; - let links = &CandidateLinks::new(physical); - let class_count = physical.candidates.len(); - let all_alive = vec![true; class_count]; - let unpruned_links = producer_candidates_left(links, &all_alive); - let unpruned_outputs = output_candidates(physical, &all_alive); - let unpruned_reach = queries_reaching(physical, &unpruned_links, &unpruned_outputs); - let refresh_windows = refresh_windows(physical, links); - // Every context before pruning, for the windows each scan may be read - // at within one context. - let all_contexts = execution_contexts( - physical, - &links.kind, - &unpruned_links, - &unpruned_outputs, - &refresh_windows, - |_| true, - ) - .contexts; - let candidate_data = candidate_data( - physical, - links, - entries, - &unpruned_reach, - &all_contexts, - &unpruned_outputs, - data, - cost_inputs, - ); - let pruned_candidates = prune_dominated_candidates( - &dominance_profiles(physical, links, &unpruned_outputs), - &candidate_data, - ); - let mut alive = all_alive; - for pruned in &pruned_candidates { - alive[pruned.class.0] = false; - } - let producer_candidates_per_input = producer_candidates_left(links, &alive); - let output_candidates = output_candidates(physical, &alive); - let kind = links.kind.clone(); - let slot = result_slots(physical); - let usable = |class: PhysicalCandidateId| candidate_data[class.0].usage.is_ok(); - let ExecutionContexts { - contexts, - context_of_shareable, - ingestion_context_of_shareable, - } = execution_contexts( - physical, - &kind, - &producer_candidates_per_input, - &output_candidates, - &refresh_windows, - usable, - ); - let AccuracyPositions { - weights: accuracy_weights, - excluded_pairs: accuracy_excluded_pairs, - } = accuracy_positions::accuracy_positions( - physical, - &candidate_data, - &producer_candidates_per_input, - &output_candidates, - entries, - data, - models, - ); - MilpInputs { - candidate_data, - pruned_candidates, - producer_candidates_per_input, - output_candidates, - kind, - slot, - contexts, - context_of_shareable, - ingestion_context_of_shareable, - accuracy_weights, - accuracy_excluded_pairs, - } -} - -/// Per candidate, per input, its producer candidates that are `alive` (none for a -/// candidate that is not). -fn producer_candidates_left( - links: &CandidateLinks, - alive: &[bool], -) -> Vec>> { - links - .producers_per_input - .iter() - .enumerate() - .map(|(class, producers_per_input)| match alive[class] { - false => Vec::new(), - true => producers_per_input - .iter() - .map(|producers| { - producers - .iter() - .copied() - .filter(|producer| alive[producer.0]) - .collect() - }) - .collect(), - }) - .collect() -} - -/// `Out(r)` per output reference: the `alive` candidates among its output -/// candidates, ascending. -fn output_candidates( - physical: &PhysicalASAPMemoCandidates, - alive: &[bool], -) -> Vec> { - (0..physical.output_reference_count()) - .map(|reference| { - let set: BTreeSet<_> = physical - .query_output_candidates(reference) - .filter(|candidate| alive[candidate.0]) - .collect(); - set.into_iter().collect() - }) - .collect() -} - -/// Per candidate, the queries whose output candidates reach it through the links -/// `producer_candidates_per_input`, ascending. -fn queries_reaching( - physical: &PhysicalASAPMemoCandidates, - producer_candidates_per_input: &[Vec>], - output_candidates: &[Vec], -) -> Vec> { - let mut reaching = vec![Vec::new(); producer_candidates_per_input.len()]; - let output_refs = output_references(&physical.query_roots); - for query in 0..physical.query_roots.len() { - let mut seen = vec![false; producer_candidates_per_input.len()]; - let mut stack: Vec = output_refs - .iter() - .enumerate() - .filter(|(_, (q, _))| *q == query) - .flat_map(|(reference, _)| output_candidates[reference].iter().copied()) - .collect(); - while let Some(class) = stack.pop() { - if std::mem::replace(&mut seen[class.0], true) { - continue; - } - reaching[class.0].push(query); - stack.extend( - producer_candidates_per_input[class.0] - .iter() - .flatten() - .copied(), - ); - } - } - reaching -} - -/// Which execution context (#1095 §2.0): a query evaluation, an ingestion -/// pipeline `I_r`, or a shareable candidate's own inputs. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub(crate) enum ContextKind { - Query(usize), - Ingestion(IngestionPipeline), - Shared(PhysicalCandidateId), -} - -/// Which ingestion pipeline `I_r` (#1095 §2.5 O3(b)): the deployment -/// refreshes the ingestion-time views of one refresh window in one run, so -/// `asap_cost::execution_demand::computations` charges a `NotMaterialized` -/// ingestion-time node once per refresh window of the views reading it. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub(crate) enum IngestionPipeline { - /// The views of this refresh window. - Window(RefreshWindow), - /// An ingestion-time shareable candidate whose window is unbounded or - /// empty: refreshed alone. - Alone(PhysicalCandidateId), -} - -/// The [`IngestionPipeline`] of ingestion-time shareable candidate -/// `shareable`, from its refresh window ([`refresh_windows`]). -pub(crate) fn ingestion_pipeline( - shareable: PhysicalCandidateId, - refresh_windows: &HashMap>, -) -> IngestionPipeline { - match refresh_windows.get(&shareable) { - Some(Some(window)) => IngestionPipeline::Window(*window), - _ => IngestionPipeline::Alone(shareable), - } -} - -/// Each ingestion-time shareable candidate's refresh window, as -/// `asap_cost::execution_demand::computations` computes it on the realized -/// plan ([`PhysicalASAPDAG::refresh_windows`]): on the DAG of the candidates -/// below it, each input read from its first producer candidate. That is -/// exact when the producer candidates of each input read one window, which -/// is debug-asserted ([`producer_window_disagreements`]). -fn refresh_windows( - physical: &PhysicalASAPMemoCandidates, - links: &CandidateLinks, -) -> HashMap> { - let views: Vec = (0..physical.candidates.len()) - .map(PhysicalCandidateId) - .filter(|&class| { - links.is_shareable(class) - && physical.candidates[class.0].timing == ExecutionTiming::IngestionTime - }) - .collect(); - let dag = dag_below(physical, links, &views); - debug_assert!( - producer_window_disagreements(links, &dag).is_empty(), - "producer candidates of one input read different windows: {:?}", - producer_window_disagreements(links, &dag) - ); - dag.refresh_windows(views.iter().map(|view| view.0)) - .into_iter() - .map(|(id, window)| (PhysicalCandidateId(id), window)) - .collect() -} - -/// The DAG of `tops` and every candidate below them, without edges: each -/// candidate a node ([`catalog_node`]) whose inputs are its first producer -/// candidates. -fn dag_below( - physical: &PhysicalASAPMemoCandidates, - links: &CandidateLinks, - tops: &[PhysicalCandidateId], -) -> PhysicalASAPDAG { - let mut nodes = Vec::new(); - let mut seen = HashSet::new(); - let mut stack = tops.to_vec(); - while let Some(class) = stack.pop() { - if seen.insert(class) { - nodes.push(catalog_node(physical, links, class)); - stack.extend(links.producers_per_input[class.0].iter().flatten()); - } - } - let nodes: Vec<&PhysicalASAPDAGNode> = nodes.iter().collect(); - nodes_dag(&nodes) -} - -/// The `(candidate, input)` pairs of `dag`'s candidates -/// ([`dag_below`]) whose producer candidates have different view windows -/// ([`PhysicalASAPDAG::view_windows`]). -fn producer_window_disagreements( - links: &CandidateLinks, - dag: &PhysicalASAPDAG, -) -> Vec<(PhysicalCandidateId, usize)> { - let windows = dag.view_windows(dag.nodes.iter().map(|node| node.id)); - let mut disagreements = Vec::new(); - for node in &dag.nodes { - for (input, producers) in links.producers_per_input[node.id].iter().enumerate() { - let mut read = producers.iter().map(|producer| windows[&producer.0]); - if let Some(first) = read.next() { - if read.any(|window| window != first) { - disagreements.push((PhysicalCandidateId(node.id), input)); - } - } - } - } - disagreements -} - -/// [`producer_window_disagreements`] over every candidate of `physical`. -#[cfg(test)] -pub(crate) fn producer_window_disagreements_of_all( - physical: &PhysicalASAPMemoCandidates, -) -> Vec<(PhysicalCandidateId, usize)> { - let links = CandidateLinks::new(physical); - let all: Vec = (0..physical.candidates.len()) - .map(PhysicalCandidateId) - .collect(); - producer_window_disagreements(&links, &dag_below(physical, &links, &all)) -} - -/// One execution context and the candidates that may give a result in it. -#[derive(Debug, Clone, PartialEq, Eq)] -pub(crate) struct ExecutionContext { - pub(crate) kind: ContextKind, - /// Whether it runs at query time (a query, or a query-time shareable - /// candidate's inputs): its `PerContext` candidates are computed in it. - pub(crate) at_query_time: bool, - /// The candidates that may give a result slot in it, ascending: each - /// computed in it ([`ExecutionContext::computes`]) or read across its - /// boundary (a shareable candidate, or a candidate of an `I_r`). - pub(crate) options: Vec, -} - -impl ExecutionContext { - /// Whether a candidate of `kind` is computed in this context (otherwise - /// it is read across the boundary). - pub(crate) fn computes(&self, kind: RunKind) -> bool { - match kind { - RunKind::PerContext => self.at_query_time, - RunKind::Ingestion => matches!(self.kind, ContextKind::Ingestion(_)), - RunKind::Shareable => false, - } - } -} - -impl MilpInputs { - /// The context where context `context` reads a candidate of an `I_r`: - /// itself for an `I_r`, `I_{r(m)}` for ingestion-time shareable `m`'s - /// context; `None` for any other context (it reads none). - pub(crate) fn ingestion_context_read_by(&self, context: usize) -> Option { - match self.contexts[context].kind { - ContextKind::Ingestion(_) => Some(context), - ContextKind::Shared(m) => self.ingestion_context_of_shareable.get(&m).copied(), - ContextKind::Query(_) => None, - } - } -} - -/// [`execution_contexts`]' result (see [`MilpInputs`] for each field). -pub(crate) struct ExecutionContexts { - pub(crate) contexts: Vec, - pub(crate) context_of_shareable: BTreeMap, - pub(crate) ingestion_context_of_shareable: BTreeMap, -} - -/// The execution contexts and their options (module documentation): from -/// each query's output candidates, each shareable candidate's producers -/// and, per ingestion pipeline `I_r`, every candidate of `I_r` an -/// ingestion-time shareable candidate of refresh window `r` -/// (`refresh_windows`) reads across its boundary, expanded through the -/// inputs of the candidates each context computes. Candidates `usable` -/// rejects (no usage) are left out. A query-time `PerContext` candidate -/// that only an ingestion-time context could read is left out too: it -/// cannot run there. -pub(crate) fn execution_contexts( - physical: &PhysicalASAPMemoCandidates, - kind: &[RunKind], - producer_candidates_per_input: &[Vec>], - output_candidates: &[Vec], - refresh_windows: &HashMap>, - usable: impl Fn(PhysicalCandidateId) -> bool, -) -> ExecutionContexts { - let query_count = physical.query_roots.len(); - let output_refs = output_references(&physical.query_roots); - // Expands `seeds` in a context: the options and, through the candidates - // it computes, their producers. - let expand = |context: &ExecutionContext, seeds: Vec| { - let mut seen: BTreeSet = BTreeSet::new(); - let mut stack = seeds; - while let Some(class) = stack.pop() { - if !usable(class) || seen.contains(&class) { - continue; - } - let class_kind = kind[class.0]; - if class_kind == RunKind::PerContext && !context.at_query_time { - continue; - } - seen.insert(class); - if context.computes(class_kind) { - stack.extend( - producer_candidates_per_input[class.0] - .iter() - .flatten() - .copied(), - ); - } - } - seen.into_iter().collect::>() - }; - let mut contexts: Vec = Vec::new(); - for query in 0..query_count { - let mut context = ExecutionContext { - kind: ContextKind::Query(query), - at_query_time: true, - options: Vec::new(), - }; - let seeds = output_refs - .iter() - .enumerate() - .filter(|(_, (q, _))| *q == query) - .flat_map(|(reference, _)| output_candidates[reference].iter().copied()) - .collect(); - context.options = expand(&context, seeds); - contexts.push(context); - } - // Shareable contexts, discovered from the options, then each `I_r` from - // every candidate of `I_r` read across a boundary; repeated until - // nothing new. - let mut context_of_shareable: BTreeMap = BTreeMap::new(); - let mut shared: Vec = Vec::new(); - // Per `I_r`: its context and its seeds. - let mut ingestion: BTreeMap< - IngestionPipeline, - (ExecutionContext, BTreeSet), - > = BTreeMap::new(); - let mut scanned = 0; - loop { - let mut pending: Vec = Vec::new(); - let mut seeds: Vec<(IngestionPipeline, PhysicalCandidateId)> = Vec::new(); - let unscanned = contexts.iter().chain(shared.iter()).skip(scanned); - for context in unscanned.chain(ingestion.values().map(|(context, _)| context)) { - // Only an ingestion-time shareable candidate reads a candidate - // of an `I_r` (Stage 2's timing rule). - let pipeline = match context.kind { - ContextKind::Shared(m) if !context.at_query_time => { - Some(ingestion_pipeline(m, refresh_windows)) - } - _ => None, - }; - for &class in &context.options { - match kind[class.0] { - RunKind::Shareable => { - if !context_of_shareable.contains_key(&class) && !pending.contains(&class) { - pending.push(class); - } - } - RunKind::Ingestion => { - if let Some(pipeline) = pipeline { - seeds.push((pipeline, class)); - } - } - RunKind::PerContext => {} - } - } - } - scanned = contexts.len() + shared.len(); - let mut grown: BTreeSet = BTreeSet::new(); - for (pipeline, class) in seeds { - let (_, known) = ingestion.entry(pipeline).or_insert_with(|| { - let context = ExecutionContext { - kind: ContextKind::Ingestion(pipeline), - at_query_time: false, - options: Vec::new(), - }; - (context, BTreeSet::new()) - }); - if known.insert(class) { - grown.insert(pipeline); - } - } - if pending.is_empty() && grown.is_empty() { - break; - } - pending.sort_unstable(); - for class in pending { - let at_query_time = physical.candidates[class.0].timing == ExecutionTiming::QueryTime; - let mut context = ExecutionContext { - kind: ContextKind::Shared(class), - at_query_time, - options: Vec::new(), - }; - context.options = expand( - &context, - producer_candidates_per_input[class.0] - .iter() - .flatten() - .copied() - .collect(), - ); - context_of_shareable.insert(class, usize::MAX); - shared.push(context); - } - for pipeline in grown { - let (context, known) = ingestion.get_mut(&pipeline).expect("seeded above"); - context.options = expand(context, known.iter().copied().collect()); - } - } - shared.sort_by_key(|context| context.kind); - let mut context_of_pipeline: BTreeMap = BTreeMap::new(); - for (pipeline, (context, _)) in ingestion { - context_of_pipeline.insert(pipeline, contexts.len()); - contexts.push(context); - } - let mut ingestion_context_of_shareable = BTreeMap::new(); - for context in shared { - if let ContextKind::Shared(class) = context.kind { - context_of_shareable.insert(class, contexts.len()); - if !context.at_query_time { - let pipeline = ingestion_pipeline(class, refresh_windows); - if let Some(&at) = context_of_pipeline.get(&pipeline) { - ingestion_context_of_shareable.insert(class, at); - } - } - } - contexts.push(context); - } - ExecutionContexts { - contexts, - context_of_shareable, - ingestion_context_of_shareable, - } -} - -// --------------------------------------------------------------------------- -// Per-candidate data -// --------------------------------------------------------------------------- - -/// Whether `node` outputs approximate values ([`CandidateData::approximate`]). -fn outputs_approximate_values(node: &OperatorNode) -> bool { - matches!( - node.asap(), - Some( - ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(..), - .. - } | ASAPOp::SummaryTumblingWindowChain { - family: FieldDataType::Sketch(..), - .. - } - ) - ) -} - -/// The physical-DAG node of `candidate`, with node id `candidate` -/// and each input's child id its first producer candidate (`usize::MAX` when it -/// has none), as Stage 2's realizer would export it before retention. -fn catalog_node( - physical: &PhysicalASAPMemoCandidates, - links: &CandidateLinks, - class: PhysicalCandidateId, -) -> PhysicalASAPDAGNode { - let candidate = &physical.candidates[class.0]; - let node = physical.physical_node(class); - let mut input = 0; - let payload = node.operator.map_children(|_| { - let child = links.producers_per_input[class.0] - .get(input) - .and_then(|producers| producers.first()) - .map_or(usize::MAX, |producer| producer.0); - input += 1; - child - }); - PhysicalASAPDAGNode { - id: class.0, - payload, - output_state: ExecutionDataState { - timing: candidate.timing, - primitive: if node.asap().and_then(ASAPOp::produced_state).is_some() { - DataPrimitive::SummaryState - } else { - DataPrimitive::Raw - }, - }, - materialization: candidate.materialization, - output_schema: node.schema.clone(), - guarantee: node.guarantee.clone(), - retention_ms: None, - } -} - -/// A DAG of `nodes` alone, without edges: what -/// [`asap_cost::tumbling_window_retention_ms`] reads (payload children and nodes). -fn nodes_dag(nodes: &[&PhysicalASAPDAGNode]) -> PhysicalASAPDAG { - PhysicalASAPDAG { - nodes: nodes.iter().map(|&node| node.clone()).collect(), - edges: Vec::new(), - roots: Vec::new(), - scalar_roots: Vec::new(), - } -} - -/// The window a scan read through `path` (time shifts, then the reader that -/// ends it) covers, as [`PhysicalASAPDAG::scan_window`] computes it for a -/// plan where `path` is the scan's only reader: the shifts' offsets add up, -/// and a `TimeRange` reads its range, a kept pane chain one pane, any other -/// pane chain its whole window; any other reader reads `Unbounded`. -fn window_read_through(path: &[&PhysicalASAPDAGNode]) -> ScanWindow { - let mut end_offset_ms: i64 = 0; - for node in path { - let width_ms: u128 = match &node.payload { - Operator::NonASAP(NonASAPOp::TimeShift { shift, .. }) if shift.at.is_none() => { - end_offset_ms = end_offset_ms.saturating_add(shift.offset_ms); - continue; - } - Operator::NonASAP(NonASAPOp::TimeRange { range, .. }) => range.as_millis(), - Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms, - .. - }) if matches!( - node.materialization, - Materialization::MatAtQueryKeptAcrossEvaluations(_) - ) => - { - u128::from(*tumbling_window_width_ms) - } - Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms, - tumbling_window_count, - .. - }) => u128::from(*tumbling_window_width_ms) * u128::from(*tumbling_window_count), - _ => return ScanWindow::Unbounded, - }; - return ScanWindow::Within { - end_offset_ms, - width_ms: i64::try_from(width_ms).unwrap_or(i64::MAX), - }; - } - ScanWindow::Unbounded -} - -/// The windows query-time scan `scan` may be read at: each reader's -/// windows (one per path through time shifts) and, per execution context, -/// the unions of the window kinds its readers there read (one scan run read -/// by several readers reads the union, `PhysicalASAPDAG::scan_window`): -/// every union of the kinds. -/// A read as an output or as a shareable candidate's own input is -/// `Unbounded`. Distinct; `Unbounded` alone when nothing reads it. -fn scan_window_options( - scan: PhysicalCandidateId, - windows_per_reader: &[(PhysicalCandidateId, Vec)], - contexts: &[ExecutionContext], - kind: &[RunKind], - producers_per_input: &[Vec>], - output_candidates: &[Vec], -) -> Vec { - use crate::milp::cost_terms::window_union; - let mut windows: Vec = Vec::new(); - let add = |window: ScanWindow, windows: &mut Vec| { - if !windows.contains(&window) { - windows.push(window); - } - }; - for (_, read) in windows_per_reader { - for &window in read { - add(window, &mut windows); - } - } - let window_of = |reader: PhysicalCandidateId| { - windows_per_reader - .iter() - .find(|(known, _)| *known == reader) - .map_or(ScanWindow::Unbounded, |(_, read)| { - window_union(read.iter().copied()) - }) - }; - let shareable = kind[scan.0] == RunKind::Shareable; - let mut all_kinds: Vec = Vec::new(); - for context in contexts { - if !context.options.contains(&scan) { - continue; - } - let mut kinds: Vec = Vec::new(); - for &reader in &context.options { - if context.computes(kind[reader.0]) - && producers_per_input[reader.0] - .iter() - .any(|producers| producers.contains(&scan)) - { - let window = window_of(reader); - if !kinds.contains(&window) { - kinds.push(window); - } - } - } - // A shareable candidate reading the scan as its own input reads it - // at its own window; a read as an output reads it whole. - if let ContextKind::Shared(own) = context.kind { - if producers_per_input[own.0] - .iter() - .any(|producers| producers.contains(&scan)) - { - let window = window_of(own); - if !kinds.contains(&window) { - kinds.push(window); - } - } - } - let read_whole = match context.kind { - ContextKind::Query(_) => output_candidates.iter().any(|out| out.contains(&scan)), - ContextKind::Shared(_) | ContextKind::Ingestion(_) => false, - }; - if read_whole && !kinds.contains(&ScanWindow::Unbounded) { - kinds.push(ScanWindow::Unbounded); - } - if shareable { - for window in kinds { - if !all_kinds.contains(&window) { - all_kinds.push(window); - } - } - continue; - } - add_unions(&kinds, &mut windows); - } - if shareable { - add_unions(&all_kinds, &mut windows); - } - if windows.is_empty() { - windows.push(ScanWindow::Unbounded); - } - windows -} - -/// Whether window `outer` contains window `inner`. -fn window_contains(outer: ScanWindow, inner: ScanWindow) -> bool { - match (outer, inner) { - (ScanWindow::Unbounded, _) => true, - (ScanWindow::Within { .. }, ScanWindow::Unbounded) => false, - ( - ScanWindow::Within { - end_offset_ms: outer_end, - width_ms: outer_width, - }, - ScanWindow::Within { - end_offset_ms: inner_end, - width_ms: inner_width, - }, - ) => { - outer_end <= inner_end - && outer_end.saturating_add(outer_width) >= inner_end.saturating_add(inner_width) - } - } -} - -/// Adds to `windows` every union of a nonempty subset of `kinds`. -fn add_unions(kinds: &[ScanWindow], windows: &mut Vec) { - use crate::milp::cost_terms::scan_window_read_at as window_union; - let mut push = |window: ScanWindow| { - if !windows.contains(&window) { - windows.push(window); - } - }; - // Every union of the kinds is enumerated (#1185: no cap); a shift past - // the word size would silently enumerate nothing. - assert!( - kinds.len() < usize::BITS as usize, - "a scan run with {} window kinds has too many unions to enumerate", - kinds.len() - ); - for set in 1usize..(1 << kinds.len()) { - push(window_union( - (0..kinds.len()) - .filter(|bit| set & (1 << bit) != 0) - .map(|bit| kinds[bit]), - )); - } -} - -/// Per class reading the query-time scan `scan`, the windows it may read it -/// at (through time shifts above it), distinct. -fn scan_windows_per_reader( - scan: PhysicalCandidateId, - nodes: &[PhysicalASAPDAGNode], - readers: &[Vec<(PhysicalCandidateId, usize)>], -) -> Vec<(PhysicalCandidateId, Vec)> { - /// Time shifts nested deeper than this are not followed. - const MAX_SHIFTS: usize = 4; - let mut per_reader: Vec<(PhysicalCandidateId, Vec)> = Vec::new(); - let mut paths: Vec> = vec![vec![scan]]; - while let Some(path) = paths.pop() { - let last = *path.last().expect("never empty"); - let follow = path.len() == 1 - || (matches!( - nodes[last.0].payload, - Operator::NonASAP(NonASAPOp::TimeShift { .. }) - ) && path.len() <= MAX_SHIFTS - && !readers[last.0].is_empty()); - if !follow { - // The path ends at a reader: what the scan reads under it. - let through: Vec<_> = path[1..].iter().map(|class| &nodes[class.0]).collect(); - let window = window_read_through(&through); - let Some(&reader) = path.get(1) else { - continue; - }; - match per_reader.iter_mut().find(|(known, _)| *known == reader) { - Some((_, windows)) if windows.contains(&window) => {} - Some((_, windows)) => windows.push(window), - None => per_reader.push((reader, vec![window])), - } - continue; - } - for &(reader, _) in readers[last.0].iter().rev() { - if !path.contains(&reader) { - let mut longer = path.clone(); - longer.push(reader); - paths.push(longer); - } - } - } - per_reader.sort_by_key(|(reader, _)| *reader); - per_reader -} - -/// The panes kept pane chain `chain` may keep, one per query that can reach -/// it (its range-evaluation span), in `reaching` order; its pane count for -/// any other node. -fn retained_pane_options( - chain: PhysicalCandidateId, - nodes: &[PhysicalASAPDAGNode], - first_producer: &[Option], - reaching: &[usize], - entries: &[QueryWorkloadEntry], -) -> Vec { - let node = &nodes[chain.0]; - let kept = matches!( - (&node.payload, node.materialization), - ( - Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }), - Materialization::MatAtIngestion(_) - | Materialization::MatAtQueryKeptAcrossEvaluations(_) - ) - ); - if !kept { - return vec![asap_cost::tumbling_windows_of(node)]; - } - // The chain over the time shifts below it, which set its input offset. - let mut below = Vec::new(); - let mut current = first_producer[chain.0]; - while let Some(producer) = current { - if !matches!( - nodes[producer.0].payload, - Operator::NonASAP(NonASAPOp::TimeShift { .. }) - ) || below.contains(&producer) - { - break; - } - below.push(producer); - current = first_producer[producer.0]; - } - let mut dag_nodes: Vec<&PhysicalASAPDAGNode> = - below.iter().rev().map(|class| &nodes[class.0]).collect(); - dag_nodes.push(node); - let dag = nodes_dag(&dag_nodes); - let mut options = Vec::new(); - let reach = vec![HashSet::from([chain.0])]; - let spans: Vec<_> = match reaching.is_empty() { - true => vec![None], - false => reaching - .iter() - .map(|&query| entries.get(query).and_then(|entry| entry.range_evaluation)) - .collect(), - }; - for range in spans { - let retention = asap_cost::tumbling_window_retention_ms(&dag, &reach, &[range]); - let panes = asap_cost::retained_tumbling_windows(node, &retention); - options.push(panes); - } - options -} - -/// Whether two producer outputs cost their readers the same. -fn same_producer_output(left: &ProducerOutput<'_>, right: &ProducerOutput<'_>) -> bool { - left.output.rows == right.output.rows - && left.output.bytes == right.output.bytes - && left.sources == right.sources - && left.family == right.family - && left.unchanged_columns == right.unchanged_columns - && (std::ptr::eq(left.payload, right.payload) - || left.payload.map_children(|_| 0usize) == right.payload.map_children(|_| 0usize)) - && (std::ptr::eq(left.output_schema, right.output_schema) - || left.output_schema == right.output_schema) -} - -/// `usage` as a reader of `node` sees it. -fn producer_output<'a>(node: &'a PhysicalASAPDAGNode, usage: &NodeUsage<'a>) -> ProducerOutput<'a> { - ProducerOutput { - payload: &node.payload, - output_schema: &node.output_schema, - output: usage.output, - sources: usage.sources.clone(), - family: usage.family, - unchanged_columns: usage.unchanged_columns.clone(), - } -} - -/// Per candidate, every `(reader candidate, input)` that may read it, ascending. -fn readers_of( - producer_candidates_per_input: &[Vec>], -) -> Vec> { - let mut readers = vec![Vec::new(); producer_candidates_per_input.len()]; - for (reader, inputs) in producer_candidates_per_input.iter().enumerate() { - for (input, producers) in inputs.iter().enumerate() { - for producer in producers { - readers[producer.0].push((PhysicalCandidateId(reader), input)); - } - } - } - readers -} - -/// Each candidate's [`CandidateData`] (module documentation, "Usage per execution -/// and its context"). `reaching[k]` are the queries that can reach candidate -/// `k` before pruning. -#[allow(clippy::too_many_arguments)] -fn candidate_data( - physical: &PhysicalASAPMemoCandidates, - links: &CandidateLinks, - entries: &[QueryWorkloadEntry], - reaching: &[Vec], - contexts: &[ExecutionContext], - output_candidates: &[Vec], - data: &DataWorkload, - cost_inputs: &CostInputs, -) -> Vec { - let count = physical.candidates.len(); - let formula_scale = cost_inputs.frozen_calibration_factor.unwrap_or(1.0); - let nodes: Vec = (0..count) - .map(|class| catalog_node(physical, links, PhysicalCandidateId(class))) - .collect(); - let node_by_id: asap_cost::NodeMap<'_> = nodes.iter().map(|node| (node.id, node)).collect(); - let first_producer: Vec> = links - .producers_per_input - .iter() - .map(|producers_per_input| { - producers_per_input - .first() - .and_then(|producers| producers.first().copied()) - }) - .collect(); - let first_producers_of: HashMap> = links - .producers_per_input - .iter() - .enumerate() - .map(|(class, producers_per_input)| { - ( - class, - producers_per_input - .iter() - .filter_map(|producers| producers.first().map(|producer| producer.0)) - .collect(), - ) - }) - .collect(); - let readers = readers_of(&links.producers_per_input); - // Per candidate, the distinct outputs it may give its readers; `None` until - // it is costed. - let mut outputs: Vec>, String>>> = vec![None; count]; - let mut result: Vec = (0..count) - .map(|class| { - let node = physical.physical_node(PhysicalCandidateId(class)); - let operator_node = physical.candidates[class].operator_node; - let accuracy_budget_parts = physical.operator_nodes_of_candidates_for_memo_groups - [operator_node.0] - .accuracy_budget_parts; - CandidateData { - approximate: outputs_approximate_values(node), - accuracy_budget_parts, - usage: Err("not costed".to_string()), - ..CandidateData::default() - } - }) - .collect(); - // Producers first (Kahn's order). Candidate links can close a cycle - // (e.g. a widened scan's `Project` that reads its own memo group): when no candidate - // is ready, the smallest one each of whose inputs has a costed producer - // is costed from those producers alone, and marked - // [`UsageContext::ProducerOnACycle`]. - let producer_sets: Vec> = links - .producers_per_input - .iter() - .map(|producers_per_input| producers_per_input.iter().flatten().map(|p| p.0).collect()) - .collect(); - let mut waiting: Vec = producer_sets.iter().map(BTreeSet::len).collect(); - let mut ready: std::collections::BinaryHeap> = (0..count) - .filter(|&class| waiting[class] == 0) - .map(std::cmp::Reverse) - .collect(); - let mut costed_classes = vec![false; count]; - loop { - let class = match ready.pop() { - Some(std::cmp::Reverse(class)) => class, - None => match (0..count).find(|&class| { - !costed_classes[class] - && links.producers_per_input[class] - .iter() - .all(|producers| producers.iter().any(|p| costed_classes[p.0])) - }) { - Some(class) => class, - None => break, - }, - }; - if std::mem::replace(&mut costed_classes[class], true) { - continue; - } - let mut readers_of_class: BTreeSet = BTreeSet::new(); - for &(reader, _) in &readers[class] { - readers_of_class.insert(reader.0); - } - for reader in readers_of_class { - waiting[reader] -= 1; - if waiting[reader] == 0 && !costed_classes[reader] { - ready.push(std::cmp::Reverse(reader)); - } - } - let class = PhysicalCandidateId(class); - let node = &nodes[class.0]; - let inputs = &links.producers_per_input[class.0]; - // Per input, the distinct outputs of its costed producer candidates. - let mut producer_options: Vec>> = Vec::with_capacity(inputs.len()); - let mut failed = None; - let mut on_a_cycle = false; - // A scan's outputs at windows that do not contain this reader's own - // window are never what it reads (the scan reads at least the - // union of its readers' windows). - let reader = class; - let reads_output = - |result: &[CandidateData], producer: PhysicalCandidateId, at: usize| -> bool { - let windowed = &result[producer.0].usage_per_window; - if windowed.is_empty() { - return true; - } - let Some((_, own)) = result[producer.0] - .windows_per_reader - .iter() - .find(|(known, _)| *known == reader) - else { - return true; - }; - let own = crate::milp::cost_terms::window_union(own.iter().copied()); - // An unbounded read is priced at the other readers' widest - // window (`plan_cost`), whatever it is. - if own == ScanWindow::Unbounded { - return true; - } - windowed - .iter() - .any(|&(window, _, output)| output == at && window_contains(window, own)) - }; - for producers in inputs { - let mut options: Vec> = Vec::new(); - for producer in producers { - match &outputs[producer.0] { - Some(Ok(produced)) => { - for (at, output) in produced.iter().enumerate() { - if !reads_output(&result, *producer, at) { - continue; - } - if !options - .iter() - .any(|known| same_producer_output(known, output)) - { - options.push(output.clone()); - } - } - } - // An uncostable producer is one no plan uses: the - // others still serve this input. - Some(Err(_)) => {} - None => on_a_cycle = true, - } - } - if options.is_empty() { - failed.get_or_insert_with(|| { - format!( - "input {} has no producer with a usage (producers {:?})", - producer_options.len(), - producers.iter().map(|p| p.0).collect::>() - ) - }); - } - producer_options.push(options); - } - if let Some(reason) = failed { - outputs[class.0] = Some(Err(reason.clone())); - result[class.0].usage = Err(reason); - continue; - } - let windows = match (&node.payload, node.output_state.timing) { - (Operator::NonASAP(NonASAPOp::Scan { .. }), ExecutionTiming::QueryTime) => { - result[class.0].windows_per_reader = - scan_windows_per_reader(class, &nodes, &readers); - scan_window_options( - class, - &result[class.0].windows_per_reader, - contexts, - &links.kind, - &links.producers_per_input, - output_candidates, - ) - } - _ => vec![ScanWindow::Unbounded], - }; - let retained_per_query = - retained_pane_options(class, &nodes, &first_producer, &reaching[class.0], entries); - let mut retained: Vec = Vec::new(); - for &panes in &retained_per_query { - if !retained.contains(&panes) { - retained.push(panes); - } - } - if retained.len() > 1 { - result[class.0].retained_panes_per_query = reaching[class.0] - .iter() - .copied() - .zip(retained_per_query.iter().copied()) - .collect(); - } - let ingestion_window_ms = (node.output_state.timing == ExecutionTiming::IngestionTime) - .then(|| asap_cost::maintained_window_ms(class.0, &node_by_id, &first_producers_of)) - .flatten(); - // Every combination of the options. - let combinations = producer_options - .iter() - .map(Vec::len) - .chain([windows.len(), retained.len()]) - .try_fold(1usize, |total, options| total.checked_mul(options)) - .expect("the number of context combinations fits in usize"); - // Each costed combination: its usage, its output, and its picks - // (the option per input, the window and the retained panes). - let mut costed: Vec<(CandidateUsage, ProducerOutput<'_>, Picks)> = Vec::new(); - let mut error = None; - for index in 0..combinations { - let mut rest = index; - let mut pick = |options: usize| { - let picked = rest % options; - rest /= options; - picked - }; - let retained_pick = pick(retained.len()); - let retained_tumbling_windows = retained[retained_pick]; - let window_pick = pick(windows.len()); - let window = windows[window_pick]; - let mut option_picks: Vec = producer_options - .iter() - .rev() - .map(|options| pick(options.len())) - .collect(); - option_picks.reverse(); - let producers: Vec> = producer_options - .iter() - .zip(&option_picks) - .map(|(options, &picked)| options[picked].clone()) - .collect(); - let usage_at = |per_second: f64| { - asap_cost::node_usage( - node, - &producers, - (1, per_second), - window, - retained_tumbling_windows, - ingestion_window_ms, - data, - cost_inputs, - formula_scale, - ) - }; - match (usage_at(1.0), usage_at(0.0)) { - (Ok(executed), Ok(standing)) => { - let usage = candidate_usage(&executed, &standing, node.output_state.timing); - costed.push(( - usage, - producer_output(node, &executed), - Picks { - options: option_picks, - window: window_pick, - retained: retained_pick, - }, - )); - } - (Err(e), _) | (_, Err(e)) => { - error.get_or_insert(e.to_string()); - } - } - } - // A context `asap-cost` cannot cost is one no plan can use. - let uncostable_context = error.is_some(); - if costed.is_empty() { - let error = error.unwrap_or_else(|| "no context".into()); - outputs[class.0] = Some(Err(error.clone())); - result[class.0].usage = Err(error); - continue; - } - let first = costed[0].0; - let mut depends_on = Vec::new(); - // Its distinct outputs, and the one each costed combination gives. - let mut produced: Vec> = Vec::new(); - let mut output_of_combination = Vec::with_capacity(costed.len()); - for (_, output, _) in &costed { - let at = match produced - .iter() - .position(|known| same_producer_output(known, output)) - { - Some(at) => at, - None => { - produced.push(output.clone()); - produced.len() - 1 - } - }; - output_of_combination.push(at); - } - // What a combination gives its plan: its usage and its output. - let given: Vec<(CandidateUsage, usize)> = costed - .iter() - .zip(&output_of_combination) - .map(|((usage, _, _), &output)| (*usage, output)) - .collect(); - // The inputs whose option changes the usage or the output, and - // whether the window does: two costed combinations differing only - // there give different usages or outputs. - let changes = |differs: &dyn Fn(&Picks, &Picks) -> bool| { - costed.iter().zip(&given).any(|((_, _, picks), gives)| { - costed - .iter() - .zip(&given) - .any(|((_, _, others), other)| differs(picks, others) && gives != other) - }) - }; - let inputs_changing_usage: Vec = (0..producer_options.len()) - .filter(|&input| { - changes(&|a: &Picks, b: &Picks| { - a.window == b.window - && a.retained == b.retained - && a.options[input] != b.options[input] - && (0..a.options.len()) - .all(|other| other == input || a.options[other] == b.options[other]) - }) - }) - .collect(); - let window_changes_usage = changes(&|a: &Picks, b: &Picks| { - a.window != b.window && a.retained == b.retained && a.options == b.options - }); - if !inputs_changing_usage.is_empty() { - // Per changing input, each producer and the options it gives: - // one choice per output of a producer with several. - let options_of_producers: Vec)>> = - inputs_changing_usage - .iter() - .map(|&input| { - let mut choices = Vec::new(); - for &producer in &inputs[input] { - let Some(Ok(produced)) = &outputs[producer.0] else { - continue; - }; - let option_of = |output: &ProducerOutput<'_>| { - producer_options[input] - .iter() - .position(|known| same_producer_output(known, output)) - }; - if produced.len() == 1 { - choices.push(( - ProducerChoice { - class: producer, - output: None, - }, - produced.iter().filter_map(option_of).collect(), - )); - continue; - } - for (at, output) in produced.iter().enumerate() { - if !reads_output(&result, producer, at) { - continue; - } - choices.push(( - ProducerChoice { - class: producer, - output: Some(at), - }, - option_of(output).into_iter().collect(), - )); - } - } - choices - }) - .collect(); - let choices = options_of_producers - .iter() - .try_fold(1usize, |total, producers| { - total.checked_mul(producers.len()) - }) - .expect("the number of producer choices fits in usize"); - for index in 0..choices { - let mut rest = index; - let chosen: Vec<&(ProducerChoice, Vec)> = options_of_producers - .iter() - .map(|producers| { - let picked = &producers[rest % producers.len()]; - rest /= producers.len(); - picked - }) - .collect(); - let matching: Vec<&(CandidateUsage, usize)> = costed - .iter() - .zip(&given) - .filter(|((_, _, picks), _)| { - inputs_changing_usage - .iter() - .zip(&chosen) - .all(|(&input, (_, options))| options.contains(&picks.options[input])) - }) - .map(|(_, gives)| gives) - .collect(); - let costliest = matching.iter().copied().max_by(|a, b| { - let total = |usage: &CandidateUsage| { - usage.cost_per_execution.unwrap_or(0.0) - + usage.standing_cost_per_second.unwrap_or(0.0) - + usage.work_ms_per_execution - }; - total(&a.0).total_cmp(&total(&b.0)) - }); - if matching.iter().any(|gives| Some(*gives) != costliest) { - result[class.0].producer_usage_ambiguous = true; - } - result[class.0].usage_per_producers.push(( - chosen.iter().map(|(choice, _)| *choice).collect(), - costliest.copied(), - )); - } - result[class.0].inputs_changing_usage = inputs_changing_usage; - } - if retained.len() > 1 { - let retained_changes_usage = changes(&|a: &Picks, b: &Picks| { - a.retained != b.retained && a.window == b.window && a.options == b.options - }); - if retained_changes_usage { - for (pick, &panes) in retained.iter().enumerate() { - if let Some(at) = costed - .iter() - .position(|(_, _, picks)| picks.retained == pick) - { - result[class.0] - .usage_per_retained_panes - .push((panes, given[at].0)); - } - } - } else { - result[class.0].retained_panes_per_query.clear(); - } - } - // A query-time scan keeps its usage per window even when every - // costed window gives the same: a window `asap-cost` cannot cost - // (an unbounded read with no `rows_held`) must stay unusable. - let query_time_scan = matches!( - (&node.payload, node.output_state.timing), - ( - Operator::NonASAP(NonASAPOp::Scan { .. }), - ExecutionTiming::QueryTime - ) - ); - if window_changes_usage || query_time_scan { - for (pick, window) in windows.iter().enumerate() { - if let Some(at) = costed.iter().position(|(_, _, picks)| picks.window == pick) { - let (usage, output) = given[at]; - result[class.0] - .usage_per_window - .push((*window, usage, output)); - } - } - } - if given.iter().any(|gives| gives.0 != first) { - if producer_options.iter().any(|options| options.len() > 1) { - depends_on.push(UsageContext::ProducerOutput); - } - if windows.len() > 1 { - depends_on.push(UsageContext::ScanWindowOfReader); - } - if retained.len() > 1 { - depends_on.push(UsageContext::PaneRetentionOfReachingQueries); - } - } - if on_a_cycle { - depends_on.push(UsageContext::ProducerOnACycle); - } - if uncostable_context { - depends_on.push(UsageContext::UncostableContext); - } - result[class.0].output_count = produced.len(); - outputs[class.0] = Some(Ok(produced)); - result[class.0].usage = Ok(first); - result[class.0].usage_depends_on = depends_on; - } - for (class, costed) in costed_classes.iter().enumerate() { - if !costed { - result[class].usage = Err("its producer links form a cycle".into()); - } - } - result -} - -/// One costed combination of a class's context options, by index: the -/// option per input, the window and the retained panes. -#[derive(Debug, Clone, PartialEq, Eq)] -struct Picks { - options: Vec, - window: usize, - retained: usize, -} - -/// The numbers of one execution (`executed`, at one execution per second) -/// and of none (`standing`) of a node at `timing`. -fn candidate_usage( - executed: &NodeUsage<'_>, - standing: &NodeUsage<'_>, - timing: ExecutionTiming, -) -> CandidateUsage { - let standing_cost = standing.cost.cost_per_second; - CandidateUsage { - cpu_ops_per_execution: executed.cpu_ops_per_second, - scanned_bytes_per_execution: executed.scanned_bytes_per_second, - work_ms_per_execution: executed.work_ms_per_execution, - own_latency_ms: match timing { - ExecutionTiming::QueryTime => executed.work_ms_per_execution, - ExecutionTiming::IngestionTime => 0.0, - }, - cost_per_execution: executed - .cost - .cost_per_second - .zip(standing_cost) - .map(|(total, standing)| total - standing), - standing_cost_per_second: standing_cost, - average_memory_bytes: executed.average_memory_bytes, - peak_memory_bytes: executed.peak_memory_bytes, - disk_bytes: executed.disk_bytes, - held_bytes: executed.held.map_or(0, |(bytes, _, _)| bytes), - raw_data_kept_bytes: executed.raw_scan.map_or(0, |(_, bytes)| bytes), - maintained_window_bytes: executed.maintained.unwrap_or(0), - output_rows: executed.output.rows, - output_bytes: executed.output.bytes, - } -} - -// --------------------------------------------------------------------------- -// Dominance -// --------------------------------------------------------------------------- - -/// What one candidate is compared on (module documentation, "Dominance"). -#[derive(Debug, Clone)] -pub(crate) struct DominanceProfile<'a> { - /// Must be equal for one candidate to dominate another. - pub(crate) same: DominanceKey<'a>, - pub(crate) guarantee: Option<&'a AccuracyGuarantee>, -} - -/// The parts of a [`DominanceProfile`] that must be equal. -#[derive(Debug, Clone, PartialEq)] -pub(crate) struct DominanceKey<'a> { - pub(crate) group: MemoGroupId, - pub(crate) timing: ExecutionTiming, - pub(crate) materialization: Materialization, - pub(crate) shareable: bool, - pub(crate) producer_candidates_per_input: &'a [Vec], - pub(crate) readers: Vec<(PhysicalCandidateId, usize)>, - pub(crate) output_references: Vec, - pub(crate) reader_constraint_ids: Vec, - pub(crate) may_keep_across_evaluations_per_sole_reader_query: &'a [bool], - pub(crate) output_schema: &'a Schema, - pub(crate) operator_kind: OperatorKindForReaders, -} - -/// The parts of a node's operator its readers' costs read: its kind, and a -/// pane chain's panes. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(crate) enum OperatorKindForReaders { - SummaryTumblingWindowChain { - tumbling_window_width_ms: u64, - tumbling_window_count: u32, - }, - Other(std::mem::Discriminant, &'static str), -} - -/// Each candidate's [`DominanceProfile`]. `output_candidates` are `Out(r)` before -/// pruning. -fn dominance_profiles<'a>( - physical: &'a PhysicalASAPMemoCandidates, - links: &'a CandidateLinks, - output_candidates: &[Vec], -) -> Vec> { - let constraint_ids = reader_constraint_ids_per_candidate(physical); - let mut readers = readers_of(&links.producers_per_input); - for list in &mut readers { - list.sort_unstable(); - } - let mut output_refs_of = vec![Vec::new(); physical.candidates.len()]; - for (reference, out) in output_candidates.iter().enumerate() { - for class in out { - output_refs_of[class.0].push(reference); - } - } - links - .producers_per_input - .iter() - .enumerate() - .map(|(class, producers_per_input)| { - let copy = PhysicalCandidateId(class); - let candidate = &physical.candidates[copy.0]; - let node = physical.physical_node(copy); - DominanceProfile { - same: DominanceKey { - group: physical.memo_group_of(copy), - timing: candidate.timing, - materialization: candidate.materialization, - shareable: links.is_shareable(copy), - producer_candidates_per_input: producers_per_input, - readers: std::mem::take(&mut readers[class]), - output_references: std::mem::take(&mut output_refs_of[class]), - reader_constraint_ids: constraint_ids[copy.0].clone(), - may_keep_across_evaluations_per_sole_reader_query: &candidate - .may_keep_across_evaluations_per_sole_reader_query, - output_schema: &node.schema, - operator_kind: match &node.operator { - Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms, - tumbling_window_count, - .. - }) => OperatorKindForReaders::SummaryTumblingWindowChain { - tumbling_window_width_ms: *tumbling_window_width_ms, - tumbling_window_count: *tumbling_window_count, - }, - operator => OperatorKindForReaders::Other( - std::mem::discriminant(operator), - node.operator.kind_name(), - ), - }, - }, - guarantee: node.guarantee.as_ref(), - } - }) - .collect() -} - -/// Whether `stronger` is at least as strong as `weaker`: equal, or `stronger` -/// exact, or both in one metric and scope with constant bounds and failure -/// probabilities no larger. Anything else (an unknown statistic, different -/// metrics) is not comparable. -pub(crate) fn guarantee_at_least_as_strong( - stronger: Option<&AccuracyGuarantee>, - weaker: Option<&AccuracyGuarantee>, -) -> bool { - match (stronger, weaker) { - (None, None) => true, - (Some(stronger), Some(weaker)) => { - if stronger.spec == weaker.spec || stronger.is_exact() { - return true; - } - stronger.spec.accuracy_metric == weaker.spec.accuracy_metric - && stronger.spec.accuracy_scope == weaker.spec.accuracy_scope - && matches!( - (stronger.spec.error.evaluate(), weaker.spec.error.evaluate()), - (Some(s), Some(w)) if s <= w - ) - && matches!( - ( - stronger.spec.failure_probability.evaluate(), - weaker.spec.failure_probability.evaluate() - ), - (Some(s), Some(w)) if s <= w - ) - } - _ => false, - } -} - -/// Every number of `usage` that a candidate it dominates must not beat, in a -/// fixed order. -fn compared_numbers(usage: &CandidateUsage) -> [f64; 12] { - [ - usage.cpu_ops_per_execution, - usage.scanned_bytes_per_execution, - usage.work_ms_per_execution, - usage.own_latency_ms, - usage.average_memory_bytes, - usage.peak_memory_bytes as f64, - usage.disk_bytes as f64, - usage.held_bytes as f64, - usage.raw_data_kept_bytes as f64, - usage.maintained_window_bytes as f64, - usage.output_rows as f64, - usage.output_bytes as f64, - ] -} - -/// Whether candidate `by` dominates candidate `candidate` (module documentation). -pub(crate) fn dominates( - by: (&DominanceProfile<'_>, &CandidateData), - class: (&DominanceProfile<'_>, &CandidateData), -) -> bool { - let ((by_profile, by_data), (profile, data)) = (by, class); - let (Ok(by_usage), Ok(usage)) = (&by_data.usage, &data.usage) else { - return false; - }; - by_profile.same == profile.same - && by_data.usage_depends_on.is_empty() - && data.usage_depends_on.is_empty() - && by_data.approximate == data.approximate - && by_data.accuracy_budget_parts == data.accuracy_budget_parts - && guarantee_at_least_as_strong(by_profile.guarantee, profile.guarantee) - && compared_numbers(by_usage) - .iter() - .zip(compared_numbers(usage)) - .all(|(by, class)| *by <= class) -} - -/// The candidates another candidate dominates (module documentation), each with -/// the smallest dominating candidate that is kept. Among candidates that dominate -/// each other, the smallest id is kept. -pub(crate) fn prune_dominated_candidates( - profiles: &[DominanceProfile<'_>], - candidate_data: &[CandidateData], -) -> Vec { - // Only candidates with equal keys can dominate each other. - let mut buckets: Vec> = Vec::new(); - let mut bucket_of_key: HashMap<(MemoGroupId, ExecutionTiming, Materialization), Vec> = - HashMap::new(); - for (class, profile) in profiles.iter().enumerate() { - let candidates = bucket_of_key - .entry(( - profile.same.group, - profile.same.timing, - profile.same.materialization, - )) - .or_default(); - match candidates - .iter() - .find(|&&bucket| profiles[buckets[bucket][0]].same == profile.same) - { - Some(&bucket) => buckets[bucket].push(class), - None => { - candidates.push(buckets.len()); - buckets.push(vec![class]); - } - } - } - let mut pruned = Vec::new(); - for bucket in buckets.iter().filter(|bucket| bucket.len() > 1) { - for &class in bucket { - let dominated_by = bucket.iter().copied().find(|&other| { - other != class - && dominates( - (&profiles[other], &candidate_data[other]), - (&profiles[class], &candidate_data[class]), - ) - && (other < class - || !dominates( - (&profiles[class], &candidate_data[class]), - (&profiles[other], &candidate_data[other]), - )) - }); - if let Some(other) = dominated_by { - pruned.push(DominatedCandidate { - class: PhysicalCandidateId(class), - dominated_by: PhysicalCandidateId(other), - reason: format!( - "candidate {other} has the same memo group, producers, readers and \ - (approx, parts), a guarantee at least as strong, and no larger \ - usage, latency, output or held state" - ), - }); - } - } - } - pruned.sort_by_key(|pruned| pruned.class); - pruned -} - -mod accuracy_positions; - -use accuracy_positions::AccuracyPositions; - -#[cfg(test)] -mod tests; diff --git a/crates/plan-selection/src/groups/accuracy_positions.rs b/crates/plan-selection/src/groups/accuracy_positions.rs deleted file mode 100644 index fa81af61f..000000000 --- a/crates/plan-selection/src/groups/accuracy_positions.rs +++ /dev/null @@ -1,777 +0,0 @@ -//! Accuracy weights and pair exclusions per query (module documentation of -//! [`super`], "Accuracy weights"), each from `asap-accuracy`'s composition -//! and Stage 3's own check ([`guarantee_check`]) of an output built over -//! the candidates. -//! -//! For query `q` and a set `S` of approximate candidates (one candidate, or -//! a pair), the outputs `q` can have with exactly `S` approximate are built -//! from the candidates' operator nodes: a candidate in `S` is its own node; -//! a candidate with no member of `S` below it stands in as its node with an -//! exact guarantee; any other candidate is its node rebuilt over each -//! combination of its producers' outputs, with its guarantee composed again -//! ([`composed_guarantee`]), as the realizer recomposes a selected plan. -//! Outputs over which readers compose the same guarantee are kept once -//! ([`reader_key`]). Each output is then checked as -//! the selected plan is: [`guarantee_check`] with the deployment's model. -//! `S` is usable for `q` only when every output passes: a candidate that -//! fails through one reader is unusable through every reader (the largest -//! weight counts, as before). The candidates of one operator node (other -//! timings or materializations) build the same node, so they are checked -//! once, together. What a check builds is kept for the later checks -//! ([`BuiltBefore`]). -use std::cell::OnceCell; -use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; -use std::hash::{Hash, Hasher}; -use std::rc::Rc; - -use asap_accuracy::propagation::{composed_guarantee, root_guarantee, with_guarantees}; -use asap_types::ir::properties::AccuracyGuarantee; -use asap_types::ir::{OperatorNode, OperatorResultKind, QueryRoot}; -use asap_types::stages::{output_references, PhysicalASAPMemoCandidates, PhysicalCandidateId}; -use asap_types::workload::{DataWorkload, QueryWorkloadEntry}; - -use super::{AccuracyWeight, CandidateData, UnusableReason}; -use crate::checks::guarantee_check; -use crate::report::RejectionCause; -use crate::SelectionModels; - -/// What the accuracy rows (#1095 §2.0 row 9) are built from. -#[derive(Debug, Clone, Default)] -pub(crate) struct AccuracyPositions { - /// Per query: `w(q, k)` for each approximate candidate `k` whose values - /// reach `q`'s output. - pub(crate) weights: Vec>, - /// Per query: the pairs `(k1, k2)`, `k1 < k2`, each usable alone, whose - /// outputs miss `q`'s target when both are approximate: - /// `a[q, k1] + a[q, k2] ≤ 1`. - pub(crate) excluded_pairs: Vec>, -} - -/// The outcome of one check of a query with a set of approximate candidates. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum Outcome { - /// Every output is exact: no value of the set reaches the output. - Exact, - /// Every output meets the target. - Meets, - Fails(UnusableReason), -} - -/// [`AccuracyPositions`] of `physical`'s queries over the candidate links -/// left (`producer_candidates_per_input`, `output_candidates`). -pub(crate) fn accuracy_positions( - physical: &PhysicalASAPMemoCandidates, - candidate_data: &[CandidateData], - producer_candidates_per_input: &[Vec>], - output_candidates: &[Vec], - entries: &[QueryWorkloadEntry], - data: &DataWorkload, - models: &SelectionModels<'_>, -) -> AccuracyPositions { - let below = approximate_below(candidate_data, producer_candidates_per_input); - let output_refs = output_references(&physical.query_roots); - let exact_stand_ins: Vec>> = (0..physical.candidates.len()) - .map(|_| OnceCell::new()) - .collect(); - let mut built_before = BuiltBefore::default(); - let mut positions = AccuracyPositions::default(); - for (query, root) in physical.query_roots.iter().enumerate() { - let references: Vec = (0..output_refs.len()) - .filter(|&reference| output_refs[reference].0 == query) - .collect(); - let reachable: BTreeSet = references - .iter() - .flat_map(|&reference| &output_candidates[reference]) - .flat_map(|class| below[class.0].iter().copied()) - .collect(); - let Some(entry) = entries.get(query) else { - positions.weights.push(BTreeMap::new()); - positions.excluded_pairs.push(Vec::new()); - continue; - }; - let checker = Checker { - physical, - exact_stand_ins: &exact_stand_ins, - producer_candidates_per_input, - output_candidates, - below: &below, - query, - root, - references: &references, - entry, - data, - models, - }; - // Candidates of one operator node (other timings or - // materializations) build the same node: checked once, together. - let mut groups: BTreeMap> = BTreeMap::new(); - for &candidate in &reachable { - groups - .entry(physical.candidates[candidate].operator_node.0) - .or_default() - .push(candidate); - } - let mut weights = BTreeMap::new(); - let mut usable: Vec<&[usize]> = Vec::new(); - for group in groups.values() { - let weight = match checker.check(group, &mut built_before) { - Outcome::Exact => continue, - Outcome::Meets => { - usable.push(group); - AccuracyWeight::One - } - Outcome::Fails(reason) => AccuracyWeight::Unusable(reason), - }; - for &candidate in group { - weights.insert(PhysicalCandidateId(candidate), weight); - } - } - let usable_candidates: Vec = usable - .iter() - .flat_map(|group| group.iter().copied()) - .collect(); - let group_of: HashMap = usable - .iter() - .enumerate() - .flat_map(|(index, group)| group.iter().map(move |&candidate| (candidate, index))) - .collect(); - let group_pairs: BTreeSet<(usize, usize)> = pairs_that_meet( - &usable_candidates, - &references, - physical, - producer_candidates_per_input, - output_candidates, - &below, - ) - .into_iter() - .map(|(first, second)| (group_of[&first], group_of[&second])) - .filter(|(first, second)| first != second) - .map(|(first, second)| (first.min(second), first.max(second))) - // A pair with a member sized for fewer than two shares is never - // read together: the shares row already excludes it. - .filter(|&(first, second)| { - [first, second] - .iter() - .all(|&group| candidate_data[usable[group][0]].accuracy_budget_parts >= 2) - }) - .collect(); - let mut excluded = Vec::new(); - for (first, second) in group_pairs { - let both: Vec = usable[first] - .iter() - .chain(usable[second]) - .copied() - .collect(); - if let Outcome::Fails(_) = checker.check(&both, &mut built_before) { - for &one in usable[first] { - for &other in usable[second] { - excluded.push(( - PhysicalCandidateId(one.min(other)), - PhysicalCandidateId(one.max(other)), - )); - } - } - } - } - excluded.sort_unstable(); - positions.weights.push(weights); - positions.excluded_pairs.push(excluded); - } - positions -} - -/// Per candidate, the approximate candidates at or below it through the -/// links, ascending. A fixpoint, so cycles of candidate links are covered. -fn approximate_below( - candidate_data: &[CandidateData], - producer_candidates_per_input: &[Vec>], -) -> Vec> { - let mut below: Vec> = candidate_data - .iter() - .enumerate() - .map(|(class, data)| match data.approximate { - true => BTreeSet::from([class]), - false => BTreeSet::new(), - }) - .collect(); - loop { - let mut changed = false; - for class in 0..below.len() { - let mut grown = below[class].clone(); - for producer in producer_candidates_per_input[class].iter().flatten() { - grown.extend(below[producer.0].iter().copied()); - } - if grown.len() != below[class].len() { - below[class] = grown; - changed = true; - } - } - if !changed { - return below; - } - } -} - -/// The pairs of `usable` (ascending, `first < second`) one plan of the -/// query can read together where their values meet: under two different -/// inputs of one candidate, under two output references of a scalar root, -/// or one below the other. -fn pairs_that_meet( - usable: &[usize], - references: &[usize], - physical: &PhysicalASAPMemoCandidates, - producer_candidates_per_input: &[Vec>], - output_candidates: &[Vec], - below: &[BTreeSet], -) -> BTreeSet<(usize, usize)> { - let usable: BTreeSet = usable.iter().copied().collect(); - let mut pairs = BTreeSet::new(); - let mut add_across = |sides: &[BTreeSet]| { - for (index, left) in sides.iter().enumerate() { - for right in &sides[index + 1..] { - for &a in left { - for &b in right { - if a != b { - pairs.insert((a.min(b), a.max(b))); - } - } - } - } - } - }; - let side = |producers: &[PhysicalCandidateId]| -> BTreeSet { - producers - .iter() - .flat_map(|producer| below[producer.0].iter().copied()) - .filter(|candidate| usable.contains(candidate)) - .collect() - }; - for producers_per_input in producer_candidates_per_input - .iter() - .take(physical.candidates.len()) - { - if producers_per_input.len() < 2 { - continue; - } - let sides: Vec> = producers_per_input - .iter() - .map(|producers| side(producers)) - .collect(); - add_across(&sides); - } - let sides: Vec> = references - .iter() - .map(|&reference| side(&output_candidates[reference])) - .collect(); - add_across(&sides); - for &upper in &usable { - for &lower in &below[upper] { - if lower != upper && usable.contains(&lower) { - pairs.insert((lower.min(upper), lower.max(upper))); - } - } - } - pairs -} - -/// One query's checks. -struct Checker<'a> { - physical: &'a PhysicalASAPMemoCandidates, - /// Per candidate, its node with an exact guarantee, built once. - exact_stand_ins: &'a [OnceCell>], - producer_candidates_per_input: &'a [Vec>], - output_candidates: &'a [Vec], - below: &'a [BTreeSet], - query: usize, - root: &'a QueryRoot, - references: &'a [usize], - entry: &'a QueryWorkloadEntry, - data: &'a DataWorkload, - models: &'a SelectionModels<'a>, -} - -/// The outputs one candidate is built as, or why they are not checked. -type Built = Result>, UnusableReason>; - -/// What the checks built, kept across checks and queries. A candidate's -/// outputs depend only on the members of the approximate set at or below -/// it, so a pair check rebuilds only the candidates above both members -/// and reuses the rest from the single checks. Outputs built across a -/// cycle of links depend on where the walk entered the cycle and are not -/// kept. -#[derive(Default)] -struct BuiltBefore { - /// Per (candidate, approximate members at or below it, ascending): its - /// outputs. - outputs: HashMap<(usize, Vec), Built>, - /// The [`reader_key`]s of every node built so far. - keys: Keys, -} - -#[cfg(test)] -thread_local! { - /// Set by a test to check with no kept builds: every check builds - /// from scratch ([`BuiltBefore`] cleared) and builds every output - /// candidate (no exact stand-in skipped). - pub(crate) static FRESH_BUILDS: std::cell::Cell = const { std::cell::Cell::new(false) }; -} - -/// One check's walk. -#[derive(Default)] -struct Walk { - /// Per candidate built in this check. - built: HashMap, - /// The candidates being built. - visiting: Vec, - /// How many reads of a candidate still being built the walk has made. - cycle_reads: usize, -} - -/// A candidate's outputs in one check, and whether they were built across -/// a cycle of links (then they are not kept in [`BuiltBefore`]). -struct BuiltInWalk { - outputs: Built, - across_cycle: bool, -} - -impl Checker<'_> { - /// Checks the query's outputs with exactly `approximate` approximate. - /// `before` holds what earlier checks built, for every query. - fn check(&self, approximate: &[usize], before: &mut BuiltBefore) -> Outcome { - #[cfg(test)] - let fresh = FRESH_BUILDS.with(std::cell::Cell::get); - #[cfg(not(test))] - let fresh = false; - if fresh { - *before = BuiltBefore::default(); - } - let mut walk = Walk::default(); - let mut per_reference: Vec>> = Vec::new(); - for &reference in self.references { - let mut outputs: Vec> = Vec::new(); - // Exact stand-ins that are not summary states are the same - // for readers ([`reader_key`]): only the first is kept. - let mut exact_value_stand_in = false; - for &class in &self.output_candidates[reference] { - let reaches = approximate - .iter() - .any(|member| self.below[class.0].contains(member)); - if !fresh - && !reaches - && self.physical.physical_node(class).result_kind != OperatorResultKind::State - && std::mem::replace(&mut exact_value_stand_in, true) - { - continue; - } - match self.built(class.0, approximate, &mut walk, before) { - Ok(built) => outputs.extend(built), - Err(reason) => return Outcome::Fails(reason), - } - } - per_reference.push(distinct_for_readers(outputs, &mut before.keys)); - } - let roots = self.roots(&per_reference); - let mut outcome = Outcome::Exact; - for root in roots { - if root_guarantee(&root).is_some_and(|g| g.is_exact()) { - continue; - } - match guarantee_check( - self.query + 1, - &root, - &self.entry.requirements.accuracy, - self.data, - self.models, - ) { - Ok(_) => outcome = Outcome::Meets, - Err(failure) => { - return Outcome::Fails(match failure.cause { - RejectionCause::NoAccuracyGuarantee => UnusableReason::NoAccuracyGuarantee, - _ => UnusableReason::AccuracyTargetMissed, - }) - } - } - } - outcome - } - - /// The query roots over each combination of `per_reference`'s outputs. - fn roots(&self, per_reference: &[Vec>]) -> Vec { - combinations(per_reference) - .into_iter() - .map(|chosen| match self.root { - QueryRoot::Operator(_) => QueryRoot::Operator(chosen[0].clone()), - QueryRoot::Scalar(expr) => { - let mut next = chosen.into_iter(); - QueryRoot::Scalar( - expr.map_operator_refs(&mut |_| { - next.next().expect("one output per reference") - }), - ) - } - }) - .collect() - } - - /// `class`'s node with an exact guarantee, built once. - fn exact_stand_in(&self, class: usize) -> Rc { - self.exact_stand_ins[class] - .get_or_init(|| { - let node = self.physical.physical_node(PhysicalCandidateId(class)); - Rc::new( - (**node) - .clone() - .with_guarantee(Some(AccuracyGuarantee::exact("an exact stand-in"))), - ) - }) - .clone() - } - - /// The outputs `class` is built as with `approximate` approximate - /// (module documentation). `walk.visiting` holds the candidates being - /// built, so a cycle of links builds nothing. - fn built( - &self, - class: usize, - approximate: &[usize], - walk: &mut Walk, - before: &mut BuiltBefore, - ) -> Built { - if let Some(done) = walk.built.get(&class) { - walk.cycle_reads += usize::from(done.across_cycle); - return done.outputs.clone(); - } - if walk.visiting.contains(&class) { - walk.cycle_reads += 1; - return Ok(Vec::new()); - } - let reaches = |candidate: usize| { - approximate - .iter() - .any(|member| self.below[candidate].contains(member)) - }; - if !reaches(class) { - return Ok(vec![self.exact_stand_in(class)]); - } - let mut members: Vec = approximate - .iter() - .copied() - .filter(|member| self.below[class].contains(member)) - .collect(); - members.sort_unstable(); - let key = (class, members); - if let Some(done) = before.outputs.get(&key) { - walk.built.insert( - class, - BuiltInWalk { - outputs: done.clone(), - across_cycle: false, - }, - ); - return done.clone(); - } - let cycle_reads = walk.cycle_reads; - let node = self.physical.physical_node(PhysicalCandidateId(class)); - let producers = &self.producer_candidates_per_input[class]; - let below_inputs = producers - .iter() - .flatten() - .any(|producer| reaches(producer.0)); - let built = if !below_inputs { - // `class` is in `approximate`, over exact inputs: its node with - // every guarantee below it composed, as the realizer does. - with_guarantees(node, &mut HashMap::new()) - .map(|node| vec![node]) - .map_err(|_| UnusableReason::NotChecked) - } else { - walk.visiting.push(class); - let mut per_input = Vec::with_capacity(producers.len()); - let mut failed = None; - for input in producers { - let mut outputs = Vec::new(); - for producer in input.iter().filter(|producer| reaches(producer.0)) { - match self.built(producer.0, approximate, walk, before) { - Ok(built) => outputs.extend(built), - Err(reason) => failed = Some(reason), - } - } - // One exact stand-in for the producers with no member below: - // one with no approximate candidate below, when there is one. - let exact = input - .iter() - .filter(|producer| !reaches(producer.0)) - .min_by_key(|producer| !self.below[producer.0].is_empty()); - if let Some(exact) = exact { - outputs.push(self.exact_stand_in(exact.0)); - } - per_input.push(distinct_for_readers(outputs, &mut before.keys)); - } - walk.visiting.pop(); - match failed { - Some(reason) => Err(reason), - None => rebuilt(node, &per_input, &mut before.keys), - } - }; - let across_cycle = walk.cycle_reads != cycle_reads; - if !across_cycle { - before.outputs.insert(key, built.clone()); - } - walk.built.insert( - class, - BuiltInWalk { - outputs: built.clone(), - across_cycle, - }, - ); - built - } -} - -/// `node` rebuilt over each combination of `per_input`'s outputs, with its -/// timing and materialization and its guarantee composed again; outputs the -/// same for their readers kept once. -fn rebuilt(node: &Rc, per_input: &[Vec>], keys: &mut Keys) -> Built { - let combinations = combinations(per_input); - let mut outputs = Vec::with_capacity(combinations.len()); - for chosen in combinations { - let mut next = chosen.into_iter(); - let rebuilt = node - .with_new_children(|_| next.next().expect("one output per input")) - .map_err(|_| UnusableReason::NotChecked)?; - let guarantee = composed_guarantee(&rebuilt); - outputs.push(Rc::new( - rebuilt - .with_guarantee(guarantee) - .with_timing(node.timing) - .with_materialization(node.materialization), - )); - } - Ok(distinct_for_readers(outputs, keys)) -} - -/// Every choice of one entry per list, in order. One empty list gives none. -fn combinations(lists: &[Vec]) -> Vec> { - let mut combinations: Vec> = vec![Vec::new()]; - for list in lists { - combinations = combinations - .into_iter() - .flat_map(|prefix| { - list.iter().map(move |item| { - let mut next = prefix.clone(); - next.push(item.clone()); - next - }) - }) - .collect(); - } - combinations -} - -/// `outputs` with outputs the same for their readers kept once (the -/// first): equal [`reader_key`]s. -fn distinct_for_readers(outputs: Vec>, keys: &mut Keys) -> Vec> { - let mut seen = HashSet::new(); - outputs - .into_iter() - .filter(|output| seen.insert(reader_key(output, keys))) - .collect() -} - -/// Per node (by address): its [`reader_key`], and whether binding -/// statistics leaves its guarantee as it is; the node is held so its -/// address is not reused during the check. -type Keys = HashMap<*const OperatorNode, (u64, bool, Rc)>; - -/// A key equal for outputs over which readers compose the same guarantee: -/// an exact node as one key (an exact summary state, a stand-in, by its -/// address); a node with a guarantee (not a summary state) by its bound and failure -/// probability (binding keeps a node's guarantee and ignores its inputs; -/// provenance is not read); a node with none that binding cannot give one -/// as one key; any other node (a summary state, whose readers read its -/// inputs' guarantees, or a node with none that binding may still give -/// one) by its operator kind and its inputs' keys. -fn reader_key(node: &Rc, keys: &mut Keys) -> u64 { - key_and_settled(node, keys).0 -} - -/// [`reader_key`], and whether binding statistics leaves `node`'s -/// guarantee as it is: a guarantee with no unknown statistic, or none over -/// inputs that are all settled. -fn key_and_settled(node: &Rc, keys: &mut Keys) -> (u64, bool) { - if let Some(&(key, settled, _)) = keys.get(&Rc::as_ptr(node)) { - return (key, settled); - } - let mut hasher = std::collections::hash_map::DefaultHasher::new(); - // Exact whatever its inputs (an exact stand-in among them). - if node - .guarantee - .as_ref() - .is_some_and(|guarantee| guarantee.is_exact()) - { - "exact".hash(&mut hasher); - if node.result_kind == OperatorResultKind::State { - Rc::as_ptr(node).hash(&mut hasher); - } - let known = (hasher.finish(), true); - keys.insert(Rc::as_ptr(node), (known.0, known.1, node.clone())); - return known; - } - let spec = node.guarantee.as_ref().map(|guarantee| &guarantee.spec); - format!("{spec:?}").hash(&mut hasher); - let state = node.result_kind == OperatorResultKind::State; - let known = match &node.guarantee { - Some(guarantee) if !state => ( - hasher.finish(), - guarantee.spec.error.evaluate().is_some() - && guarantee.spec.failure_probability.evaluate().is_some(), - ), - _ => { - let children: Vec<(u64, bool)> = node - .children() - .into_iter() - .map(|child| key_and_settled(child, keys)) - .collect(); - let settled = node.guarantee.is_none() && children.iter().all(|(_, settled)| *settled); - if !(settled && !state) { - node.operator.kind_name().hash(&mut hasher); - state.hash(&mut hasher); - for (key, _) in &children { - key.hash(&mut hasher); - } - } - (hasher.finish(), settled) - } - }; - keys.insert(Rc::as_ptr(node), (known.0, known.1, node.clone())); - known -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::groups::tests::{candidate, catalog, reader_of, relative, root, values_node}; - use asap_logical_asap_optimization::test_support::entry_with; - use asap_types::ir::properties::AccuracyTarget; - use asap_types::workload::{QueryRecurrence, RepeatedDemand, RepetitionInterval}; - - /// The depth of each output of `built`. - fn depths(built: &Built) -> Vec { - fn depth(node: &OperatorNode) -> usize { - 1 + node - .children() - .into_iter() - .map(|child| depth(child)) - .max() - .unwrap_or(0) - } - built - .as_ref() - .expect("built") - .iter() - .map(|node| depth(node)) - .collect() - } - - // A two-candidate link cycle above an approximate member: `A` reads - // the leaf or `B`; `B` reads `A` or a second leaf. Walking from the - // top enters the cycle at `A`, so `B` is built there with `A` cut off - // (a cycle read), over the second leaf only. Such outputs are not - // kept: a later walk that starts at `B` builds it in full, over `A`, - // as with no kept builds, for one member and for a pair. - #[test] - fn outputs_built_across_a_cycle_are_not_kept() { - let approximate_leaf = |value: f64, error: f64| { - Rc::new( - (*values_node(value)) - .clone() - .with_guarantee(Some(relative(error, 0.01))), - ) - }; - let leaf = approximate_leaf(1.0, 0.01); - let other_leaf = approximate_leaf(2.0, 0.02); - let a = reader_of(&leaf); - let b = reader_of(&leaf); - let top = reader_of(&a); - let physical = catalog( - vec![(leaf, 0), (a, 1), (b, 2), (top, 3), (other_leaf, 4)], - vec![ - candidate(0, &[], false), - candidate(1, &[&[0, 2]], false), - candidate(2, &[&[1, 4]], false), - candidate(3, &[&[1]], true), - candidate(4, &[], false), - ], - vec![root(3)], - ); - let producers: Vec>> = physical - .candidates - .iter() - .map(|candidate| { - candidate - .producer_candidates_per_input - .iter() - .map(|producers| producers.iter().copied().collect()) - .collect() - }) - .collect(); - // Leaves 0 and 4 stand for approximate candidates. - let below: Vec> = vec![ - BTreeSet::from([0]), - BTreeSet::from([0, 4]), - BTreeSet::from([0, 4]), - BTreeSet::from([0, 4]), - BTreeSet::from([4]), - ]; - let exact_stand_ins: Vec>> = (0..physical.candidates.len()) - .map(|_| OnceCell::new()) - .collect(); - let entry = entry_with( - AccuracyTarget::max_error(0.01).failure_probability(0.01), - QueryRecurrence::Repeated(RepeatedDemand::FixedInterval(RepetitionInterval(60_000))), - ); - let data = DataWorkload::default(); - let cost_inputs = asap_cost::CostInputs::default(); - let models = SelectionModels { - accuracy: &asap_accuracy::BuiltinAccuracyModel, - evidence: &asap_accuracy::NoAccuracyEvidence, - cost_inputs: &cost_inputs, - }; - let output_candidates = vec![vec![PhysicalCandidateId(3)]]; - let checker = Checker { - physical: &physical, - exact_stand_ins: &exact_stand_ins, - producer_candidates_per_input: &producers, - output_candidates: &output_candidates, - below: &below, - query: 0, - root: &physical.query_roots[0], - references: &[0], - entry: &entry, - data: &data, - models: &models, - }; - for approximate in [&[0][..], &[0, 4]] { - let mut kept = BuiltBefore::default(); - // From the top: `B` is built with `A` cut off, over the second - // leaf alone (depth 2). - let mut walk = Walk::default(); - checker - .built(3, approximate, &mut walk, &mut kept) - .expect("built"); - assert!(walk.cycle_reads > 0); - assert!(walk.built[&2].across_cycle); - assert_eq!(depths(&walk.built[&2].outputs)[0], 2, "{approximate:?}"); - // From `B`: in full, its first output reads `A` over the leaf - // (depth 3), with kept builds as with none. - let from_b = checker.built(2, approximate, &mut Walk::default(), &mut kept); - let fresh = checker.built( - 2, - approximate, - &mut Walk::default(), - &mut BuiltBefore::default(), - ); - assert_eq!(depths(&from_b), depths(&fresh), "{approximate:?}"); - assert_eq!(depths(&fresh)[0], 3, "{approximate:?}"); - } - } -} diff --git a/crates/plan-selection/src/groups/tests.rs b/crates/plan-selection/src/groups/tests.rs deleted file mode 100644 index f9c2ce4ee..000000000 --- a/crates/plan-selection/src/groups/tests.rs +++ /dev/null @@ -1,1085 +0,0 @@ -use std::rc::Rc; - -use super::*; -use asap_types::ir::properties::{AccuracyMetric, BoundExpr, ProbabilityExpr}; -use asap_types::ir::scalar::expr_ir::ScalarValue; -use asap_types::ir::schema::SketchAlgorithm; -use asap_types::ir::schema::{DataType, Field}; -use asap_types::ir::{QueryRoot, ScalarExpr}; -use asap_types::physical::Medium; -use asap_types::stages::{ - CandidateForMemoGroupId, OperatorNodeOfCandidateForMemoGroup, - OperatorNodeOfCandidateForMemoGroupId, PhysicalCandidate, ProducerCandidates, -}; - -// --------------------------------------------------------------------------- -// Hand-built catalogs -// --------------------------------------------------------------------------- - -/// A one-row `VALUES` node of `value`. -pub(super) fn values_node(value: f64) -> Rc { - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Values { - rows: vec![vec![ScalarExpr::Literal(ScalarValue::Float64(value))]], - schema: Schema::lifted(vec![Field::plain("value", DataType::Float64, false)], None), - })) - .unwrap() -} - -/// A `VALUES` node of `rows` one-column rows. -fn rows_node(rows: usize) -> Rc { - let schema = Schema::lifted(vec![Field::plain("value", DataType::Float64, false)], None); - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Values { - rows: vec![vec![ScalarExpr::Literal(ScalarValue::Float64(1.0))]; rows], - schema, - })) - .unwrap() -} - -/// A stateless one-input reader of `child`. -pub(super) fn reader_of(child: &Rc) -> Rc { - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Dedup { - cols: Vec::new(), - child: Rc::clone(child), - })) - .unwrap() -} - -/// A catalog of `nodes`, node `i` in memo group `nodes[i].1` (each node its -/// own candidate of that group, all implementation roots), with -/// `candidates` and one query root per entry of `roots`. -pub(super) fn catalog( - nodes: Vec<(Rc, u32)>, - candidates: Vec, - roots: Vec>, -) -> PhysicalASAPMemoCandidates { - let mut index_in_group: HashMap = HashMap::new(); - PhysicalASAPMemoCandidates::new( - nodes - .into_iter() - .map(|(physical_node, group)| { - let index = index_in_group.entry(group).or_default(); - *index += 1; - OperatorNodeOfCandidateForMemoGroup { - candidates_for_memo_group: vec![CandidateForMemoGroupId { - group: MemoGroupId(group), - index: *index - 1, - }], - is_implementation_root: true, - accuracy_budget_parts: 1, - physical_node, - } - }) - .collect(), - candidates, - roots, - Vec::new(), - ) -} - -/// One candidate of operator node `node` at query time, not materialized, -/// reading `producers` per input, that may produce a query's output when -/// `output`. -pub(super) fn candidate(node: usize, producers: &[&[usize]], output: bool) -> PhysicalCandidate { - PhysicalCandidate { - operator_node: OperatorNodeOfCandidateForMemoGroupId(node), - timing: ExecutionTiming::QueryTime, - materialization: Materialization::NotMaterialized, - producer_candidates_per_input: producers - .iter() - .map(|input| { - input - .iter() - .map(|&id| PhysicalCandidateId(id)) - .collect::() - }) - .collect(), - may_produce_query_output: output, - may_keep_across_evaluations_per_sole_reader_query: Vec::new(), - } -} - -pub(super) fn root(group: u32) -> QueryRoot { - QueryRoot::Operator(MemoGroupId(group)) -} - -fn prepared(physical: &PhysicalASAPMemoCandidates) -> MilpInputs { - prepare_milp_inputs( - physical, - &[], - &DataWorkload::default(), - &CostInputs::default(), - ) -} - -fn id(candidate: usize) -> PhysicalCandidateId { - PhysicalCandidateId(candidate) -} - -// --------------------------------------------------------------------------- -// Dominance -// --------------------------------------------------------------------------- - -fn usage() -> CandidateUsage { - CandidateUsage { - cpu_ops_per_execution: 10.0, - scanned_bytes_per_execution: 10.0, - work_ms_per_execution: 10.0, - own_latency_ms: 10.0, - cost_per_execution: Some(1.0), - standing_cost_per_second: Some(1.0), - average_memory_bytes: 10.0, - peak_memory_bytes: 10, - disk_bytes: 10, - held_bytes: 10, - raw_data_kept_bytes: 10, - maintained_window_bytes: 10, - output_rows: 10, - output_bytes: 10, - } -} - -fn data_with(usage: CandidateUsage) -> CandidateData { - CandidateData { - approximate: true, - accuracy_budget_parts: 1, - usage: Ok(usage), - ..CandidateData::default() - } -} - -pub(super) fn relative(error: f64, failure_probability: f64) -> AccuracyGuarantee { - AccuracyGuarantee::new( - AccuracyMetric::Relative, - BoundExpr::Constant { value: error }, - ProbabilityExpr::Constant { - value: failure_probability, - }, - Vec::new(), - ) -} - -// Candidate dominance needs every condition: a candidate with equal keys, a -// guarantee at least as strong and no larger number dominates; changing any one condition stops it. -#[test] -fn dominance_needs_every_condition() { - let producers = vec![vec![PhysicalCandidateId(7)]]; - let other_producers = vec![vec![PhysicalCandidateId(8)]]; - let keep = vec![false]; - let other_keep = vec![true]; - let schema = rows_node(1).schema.clone(); - let other_schema = Schema::lifted(vec![Field::plain("v", DataType::Int64, false)], None); - let strong = relative(0.01, 0.01); - let weak = relative(0.02, 0.01); - let values_kind = - OperatorKindForReaders::Other(std::mem::discriminant(&rows_node(1).operator), "Values"); - let key = DominanceKey { - group: MemoGroupId(3), - timing: ExecutionTiming::QueryTime, - materialization: Materialization::NotMaterialized, - shareable: false, - producer_candidates_per_input: &producers, - readers: vec![(PhysicalCandidateId(9), 0)], - output_references: vec![0], - reader_constraint_ids: vec![1], - may_keep_across_evaluations_per_sole_reader_query: &keep, - output_schema: &schema, - operator_kind: values_kind, - }; - let profile = |guarantee| DominanceProfile { - same: key.clone(), - guarantee, - }; - let better = profile(Some(&strong)); - let worse = profile(Some(&weak)); - let mut worse_usage = usage(); - worse_usage.cpu_ops_per_execution = 20.0; - let (better_data, worse_data) = (data_with(usage()), data_with(worse_usage)); - assert!(dominates((&better, &better_data), (&worse, &worse_data))); - assert!(!dominates((&worse, &worse_data), (&better, &better_data))); - - // Each key field. - let changed_keys = [ - ( - "group", - DominanceKey { - group: MemoGroupId(4), - ..key.clone() - }, - ), - ( - "timing", - DominanceKey { - timing: ExecutionTiming::IngestionTime, - ..key.clone() - }, - ), - ( - "materialization", - DominanceKey { - materialization: Materialization::MatAtQuery(Medium::Memory), - ..key.clone() - }, - ), - ( - "shareable", - DominanceKey { - shareable: true, - ..key.clone() - }, - ), - ( - "producers", - DominanceKey { - producer_candidates_per_input: &other_producers, - ..key.clone() - }, - ), - ( - "readers", - DominanceKey { - readers: vec![(PhysicalCandidateId(10), 0)], - ..key.clone() - }, - ), - ( - "outputs", - DominanceKey { - output_references: vec![1], - ..key.clone() - }, - ), - ( - "reader constraints", - DominanceKey { - reader_constraint_ids: vec![2], - ..key.clone() - }, - ), - ( - "keep flags", - DominanceKey { - may_keep_across_evaluations_per_sole_reader_query: &other_keep, - ..key.clone() - }, - ), - ( - "schema", - DominanceKey { - output_schema: &other_schema, - ..key.clone() - }, - ), - ( - "operator kind", - DominanceKey { - operator_kind: OperatorKindForReaders::SummaryTumblingWindowChain { - tumbling_window_width_ms: 60_000, - tumbling_window_count: 5, - }, - ..key.clone() - }, - ), - ]; - for (name, changed) in changed_keys { - let by = DominanceProfile { - same: changed, - ..better.clone() - }; - assert!( - !dominates((&by, &better_data), (&worse, &worse_data)), - "{name}" - ); - } - // (approx, parts), guarantee. - let mut data = better_data.clone(); - data.approximate = false; - assert!( - !dominates((&better, &data), (&worse, &worse_data)), - "approx" - ); - let mut data = better_data.clone(); - data.accuracy_budget_parts = 2; - assert!(!dominates((&better, &data), (&worse, &worse_data)), "parts"); - let by = profile(Some(&weak)); - let class = profile(Some(&strong)); - assert!( - !dominates((&by, &better_data), (&class, &worse_data)), - "guarantee" - ); - // Each number. - for number in 0..compared_numbers(&usage()).len() { - let mut larger = usage(); - match number { - 0 => larger.cpu_ops_per_execution = 30.0, - 1 => larger.scanned_bytes_per_execution = 30.0, - 2 => larger.work_ms_per_execution = 30.0, - 3 => larger.own_latency_ms = 30.0, - 4 => larger.average_memory_bytes = 30.0, - 5 => larger.peak_memory_bytes = 30, - 6 => larger.disk_bytes = 30, - 7 => larger.held_bytes = 30, - 8 => larger.raw_data_kept_bytes = 30, - 9 => larger.maintained_window_bytes = 30, - 10 => larger.output_rows = 30, - _ => larger.output_bytes = 30, - } - assert!( - !dominates((&better, &data_with(larger)), (&worse, &worse_data)), - "number {number}" - ); - } - // A plan-dependent or uncosted usage is never compared. - let mut data = better_data.clone(); - data.usage_depends_on = vec![UsageContext::ProducerOutput]; - assert!( - !dominates((&better, &data), (&worse, &worse_data)), - "context" - ); - let mut data = better_data.clone(); - data.usage = Err("uncosted".into()); - assert!(!dominates((&better, &data), (&worse, &worse_data)), "error"); -} - -// A guarantee is at least as strong when equal, exact, or no larger in one -// metric and scope; different metrics and unknown bounds do not compare. -#[test] -fn guarantee_strength_compares_constant_bounds_in_one_metric() { - let strong = relative(0.01, 0.01); - let weak = relative(0.02, 0.02); - let exact = AccuracyGuarantee::exact("exact"); - let rank = AccuracyGuarantee::new( - AccuracyMetric::Rank, - BoundExpr::Constant { value: 0.001 }, - ProbabilityExpr::Zero, - Vec::new(), - ); - assert!(guarantee_at_least_as_strong(Some(&strong), Some(&weak))); - assert!(!guarantee_at_least_as_strong(Some(&weak), Some(&strong))); - assert!(guarantee_at_least_as_strong(Some(&exact), Some(&weak))); - assert!(guarantee_at_least_as_strong(None, None)); - assert!(!guarantee_at_least_as_strong(Some(&strong), None)); - assert!(!guarantee_at_least_as_strong(Some(&rank), Some(&weak))); -} - -// Among dominated candidates the dominating one is reported; of two equal -// candidates the smaller id is kept; a candidate better in one number and worse in -// another is kept. -#[test] -fn pruning_keeps_the_smallest_equal_class_and_incomparable_ones() { - let producers = vec![vec![PhysicalCandidateId(9)]]; - let keep = Vec::new(); - let schema = rows_node(1).schema.clone(); - let key = DominanceKey { - group: MemoGroupId(0), - timing: ExecutionTiming::QueryTime, - materialization: Materialization::NotMaterialized, - shareable: false, - producer_candidates_per_input: &producers, - readers: Vec::new(), - output_references: vec![0], - reader_constraint_ids: Vec::new(), - may_keep_across_evaluations_per_sole_reader_query: &keep, - output_schema: &schema, - operator_kind: OperatorKindForReaders::Other( - std::mem::discriminant(&rows_node(1).operator), - "Values", - ), - }; - let profiles = vec![ - DominanceProfile { - same: key.clone(), - guarantee: None, - }; - 4 - ]; - let mut worse = usage(); - worse.output_rows = 20; - let mut incomparable = usage(); - incomparable.cpu_ops_per_execution = 5.0; - incomparable.held_bytes = 20; - let data = vec![ - data_with(usage()), - data_with(usage()), - data_with(worse), - data_with(incomparable), - ]; - let pruned = prune_dominated_candidates(&profiles, &data); - let summary: Vec<_> = pruned - .iter() - .map(|pruned| (pruned.class.0, pruned.dominated_by.0)) - .collect(); - assert_eq!(summary, vec![(1, 0), (2, 0)]); -} - -// Through `prepare_milp_inputs`: of two deduplications in one memo group -// over one leaf, read by one reader and costing the same, the later one is -// deleted with its links, and the reader reads the other. -#[test] -fn a_dominated_class_is_deleted_with_its_links() { - let schema = Schema::lifted( - vec![ - Field::plain("a", DataType::Float64, false), - Field::plain("b", DataType::Float64, false), - ], - None, - ); - let leaf = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Values { - rows: vec![vec![ScalarExpr::Literal(ScalarValue::Float64(1.0)); 2]; 4], - schema, - })) - .unwrap(); - let dedup = |cols: Vec| { - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Dedup { - cols, - child: Rc::clone(&leaf), - })) - .unwrap() - }; - // Two equal nodes: equal schemas and costs. - let (first, second) = (dedup(vec![0]), dedup(vec![0])); - let reader = reader_of(&first); - let physical = catalog( - vec![(leaf, 0), (first, 1), (second, 1), (reader, 2)], - vec![ - candidate(0, &[], false), - candidate(1, &[&[0]], false), - candidate(2, &[&[0]], false), - candidate(3, &[&[1, 2]], true), - ], - vec![root(2)], - ); - let inputs = prepared(&physical); - let (leaf, kept, deleted, top) = (id(0), id(1), id(2), id(3)); - assert_eq!( - inputs.candidate_data[kept.0].usage, - inputs.candidate_data[deleted.0].usage - ); - assert_eq!( - inputs - .pruned_candidates - .iter() - .map(|pruned| (pruned.class, pruned.dominated_by)) - .collect::>(), - vec![(deleted, kept)] - ); - assert_eq!( - inputs.producer_candidates_per_input[top.0], - vec![vec![kept]] - ); - assert!(inputs.producer_candidates_per_input[deleted.0].is_empty()); - assert_eq!(inputs.contexts[0].options, vec![leaf, kept, top]); -} - -// --------------------------------------------------------------------------- -// Execution contexts -// --------------------------------------------------------------------------- - -/// A leaf, a materialized (shareable) middle, and two queries each reading -/// the middle through its own stateless top; a third query reads only its -/// own leaf. Candidate `i` runs operator node `i`. -fn shared_middle() -> PhysicalASAPMemoCandidates { - let leaf = values_node(1.0); - let middle = reader_of(&leaf); - let top_0 = reader_of(&middle); - let top_1 = reader_of(&middle); - let lone = values_node(2.0); - let candidates = vec![ - candidate(0, &[], false), - PhysicalCandidate { - materialization: Materialization::MatAtQuery(Medium::Memory), - ..candidate(1, &[&[0]], false) - }, - candidate(2, &[&[1]], true), - candidate(3, &[&[1]], true), - candidate(4, &[], true), - ]; - catalog( - vec![(leaf, 0), (middle, 1), (top_0, 2), (top_1, 3), (lone, 4)], - candidates, - vec![root(2), root(3), root(4)], - ) -} - -// Each query context holds its own top and the shared middle as a read -// across its boundary; the middle's own context holds the leaf below it; -// no `I_r` is read, so there is none; a query that cannot reach the middle holds only its -// own candidate. -#[test] -fn execution_contexts_on_a_hand_built_catalog() { - let physical = shared_middle(); - let inputs = prepared(&physical); - let (leaf, middle, top_0, top_1, lone) = (id(0), id(1), id(2), id(3), id(4)); - assert!(CandidateLinks::new(&physical).is_shareable(middle)); - assert!(inputs.pruned_candidates.is_empty()); - let contexts: Vec<(ContextKind, Vec)> = inputs - .contexts - .iter() - .map(|context| (context.kind, context.options.clone())) - .collect(); - assert_eq!( - contexts, - vec![ - (ContextKind::Query(0), vec![middle, top_0]), - (ContextKind::Query(1), vec![middle, top_1]), - (ContextKind::Query(2), vec![lone]), - (ContextKind::Shared(middle), vec![leaf]), - ] - ); - assert_eq!(inputs.context_of_shareable, BTreeMap::from([(middle, 3)])); - assert!(inputs.contexts[3].computes(RunKind::PerContext)); - assert!(!inputs.contexts[3].computes(RunKind::Shareable)); - assert_eq!( - inputs.output_candidates, - vec![vec![top_0], vec![top_1], vec![lone]] - ); -} - -/// Stage 2's output for `queries` and what Stage 3 also reads. -struct ThroughStages1And2 { - physical: PhysicalASAPMemoCandidates, - entries: Vec, - data: DataWorkload, -} - -/// `queries` (PromQL over metric `m`, each predictable and repeating every -/// minute with a 1 % error target, over metric `m` with 100 series) -/// through Stage 1 and Stage 2 as the facade runs them, over the complete pane-width domain, with raw data and -/// query-time results retained. -fn through_stages_1_and_2(queries: &[&str]) -> ThroughStages1And2 { - use asap_logical_asap_optimization::test_support::lower_promql; - let roots: Vec = queries - .iter() - .map(|query| QueryRoot::Operator(lower_promql(query, target()))) - .collect(); - through_stages_1_and_2_of(roots) -} - -/// The accuracy target of [`through_stages_1_and_2`]'s queries. -fn target() -> asap_types::ir::properties::AccuracyTarget { - asap_types::ir::properties::AccuracyTarget::max_error(0.01).failure_probability(0.01) -} - -/// [`through_stages_1_and_2`] for query roots `roots`. -fn through_stages_1_and_2_of(roots: Vec) -> ThroughStages1And2 { - use asap_logical_asap_optimization::generate_logical_asap_memo_candidates; - use asap_logical_asap_optimization::generate_per_computation_logical_memo_candidates; - use asap_logical_asap_optimization::test_support::entry_with; - use asap_types::workload::{ - DurationMs, Predictability, QueryRecurrence, RepeatedDemand, RepetitionInterval, - }; - let target = target(); - let every_minute = - QueryRecurrence::Repeated(RepeatedDemand::FixedInterval(RepetitionInterval(60_000))); - let entries: Vec<_> = roots - .iter() - .map(|_| QueryWorkloadEntry { - predictability: Predictability::Predictable { known_at: None }, - ..entry_with(target.clone(), every_minute.clone()) - }) - .collect(); - let mut data = DataWorkload::sampled_every(["m".to_string()], DurationMs(1_000)); - let source = data.sources.get_mut("m").expect("m"); - source.series_count = Some(100); - source.values_non_negative = true; - let targets = vec![target.clone(); roots.len()]; - let pass1 = generate_per_computation_logical_memo_candidates( - asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&roots), - &targets, - &asap_accuracy::BuiltinAccuracyModel, - &asap_accuracy::NoAccuracyEvidence, - &data, - None, - ) - .unwrap(); - let logical = generate_logical_asap_memo_candidates(pass1, &entries).unwrap(); - let physical = asap_physical_asap_optimization::generate_physical_asap_memo_candidates( - &logical, &entries, &data, true, true, - ) - .unwrap(); - // Stage 2 copies each node's `accuracy_budget_parts` from its memo - // candidates, the largest for a factored node. - for node in &physical.operator_nodes_of_candidates_for_memo_groups { - let largest = node - .candidates_for_memo_group - .iter() - .map(|id| logical.memo.candidates[id.group.0 as usize][id.index].accuracy_budget_parts) - .max(); - assert_eq!(Some(node.accuracy_budget_parts), largest); - } - ThroughStages1And2 { - physical, - entries, - data, - } -} - -// Through the real Stages 1 and 2: two quantile queries over 1m and 5m -// windows of one metric can share a pane chain. Both queries reach it, it -// is an option of both query contexts with a context of its own, and every -// context computes only what runs there. -// No quotient: every approximate candidate weighs 1. -#[test] -fn two_queries_share_a_pane_chain_through_the_real_pipeline() { - let stages = through_stages_1_and_2(&[ - "quantile_over_time(0.9, m[15m])", - "quantile_over_time(0.9, m[1h])", - ]); - let physical = &stages.physical; - let links = CandidateLinks::new(physical); - let inputs = prepare_milp_inputs( - physical, - &stages.entries, - &stages.data, - &CostInputs::default(), - ); - let shared_chains: Vec = (0..physical.candidates.len()) - .map(PhysicalCandidateId) - .filter(|candidate| { - matches!( - physical.physical_node(*candidate).asap(), - Some(ASAPOp::SummaryTumblingWindowChain { .. }) - ) && links.is_shareable(*candidate) - && inputs.contexts[0].options.contains(candidate) - && inputs.contexts[1].options.contains(candidate) - }) - .collect(); - assert!(!shared_chains.is_empty()); - for &chain in &shared_chains { - let context = inputs.context_of_shareable[&chain]; - assert_eq!(inputs.contexts[context].kind, ContextKind::Shared(chain)); - } - // Every context computes only candidates of its own kind and reads the - // others across its boundary. - for context in &inputs.contexts { - for &option in &context.options { - let kind = inputs.kind[option.0]; - assert!( - context.computes(kind) - || kind == RunKind::Shareable - || (kind == RunKind::Ingestion - && !matches!(context.kind, ContextKind::Ingestion(_))) - ); - } - } - for weights in &inputs.accuracy_weights { - assert!(!weights.is_empty()); - assert!( - weights - .values() - .all(|&weight| weight == AccuracyWeight::One), - "{weights:?}" - ); - } -} - -// --------------------------------------------------------------------------- -// Accuracy weights -// --------------------------------------------------------------------------- - -/// The sketch algorithm and `parts` of each approximate candidate of -/// `physical`, with its weight for query 0 (`None`: no weight). -fn sketch_weights( - physical: &PhysicalASAPMemoCandidates, - inputs: &MilpInputs, -) -> Vec<(SketchAlgorithm, usize, Option)> { - (0..physical.candidates.len()) - .filter(|&class| inputs.candidate_data[class].approximate) - .map(|class| { - let id = PhysicalCandidateId(class); - let algorithm = match physical.physical_node(id).asap() { - Some( - ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), - .. - } - | ASAPOp::SummaryTumblingWindowChain { - family: FieldDataType::Sketch(kind, _), - .. - }, - ) => *kind.algorithm(), - _ => unreachable!("an approximate candidate is a sketch"), - }; - ( - algorithm, - inputs.candidate_data[class].accuracy_budget_parts, - inputs.accuracy_weights[0].get(&id).copied(), - ) - }) - .collect() -} - -// `q0.5 / q0.9` through the real stages, weights from the composition: a -// KLL rank error has no rule through a quotient, so Pass 1 prunes every KLL -// candidate (#1129); a DDSketch relative -// error composes, but a lone denominator sized for the whole target -// (`parts = 1`) misses it (`ε/(1 − ε) > ε`) while one sized for two parts -// meets it. -#[test] -fn ratio_weights_come_from_the_composition() { - let stages = through_stages_1_and_2(&[ - "quantile_over_time(0.5, m[5m]) / quantile_over_time(0.9, m[5m])", - ]); - let inputs = prepare_milp_inputs( - &stages.physical, - &stages.entries, - &stages.data, - &CostInputs::default(), - ); - let weights = sketch_weights(&stages.physical, &inputs); - let of = |algorithm: SketchAlgorithm, parts: usize| -> BTreeSet> { - weights - .iter() - .filter(|(a, p, _)| *a == algorithm && *p == parts) - .map(|(_, _, weight)| *weight) - .collect() - }; - let missed = Some(AccuracyWeight::Unusable( - UnusableReason::AccuracyTargetMissed, - )); - // Pass 1 prunes every KLL: its rank error composes no guarantee - // through the quotient. - for parts in [1, 2] { - assert_eq!( - of(SketchAlgorithm::Kll, parts), - BTreeSet::new(), - "{weights:?}" - ); - } - // At one part: numerators meet the target, denominators miss it. - assert_eq!( - of(SketchAlgorithm::DDSketch, 1), - BTreeSet::from([Some(AccuracyWeight::One), missed]), - "{weights:?}" - ); - assert_eq!( - of(SketchAlgorithm::DDSketch, 2), - BTreeSet::from([Some(AccuracyWeight::One)]), - "{weights:?}" - ); - // Two DDSketches at two parts compose: no pair is excluded. - assert!(inputs.accuracy_excluded_pairs[0].is_empty()); -} - -// RangeHeavy T8+T5's shape: `q0.9 / q0.5` with both operands also queried -// alone. One DDSketch pane chain sized for two shares of the budget serves -// both quotient operands and both lone quantiles, as the old Stage 3 -// engine planned it: the check accepts the quotient over it (`(α + α)/(1 − α) ≤ ε` at -// `α = 1 − (1 + ε)^{−1/2}`), so it weighs 1 for every query. A chain sized -// for the whole budget (`α = ε`) never serves the quotient's denominator. -#[test] -fn one_pane_chain_serves_a_ratio_and_its_operands() { - let stages = through_stages_1_and_2(&[ - "quantile_over_time(0.9, m[5m]) / quantile_over_time(0.5, m[5m])", - "quantile_over_time(0.9, m[5m])", - "quantile_over_time(0.5, m[5m])", - ]); - let inputs = prepare_milp_inputs( - &stages.physical, - &stages.entries, - &stages.data, - &CostInputs::default(), - ); - let alpha = |class: usize| match stages - .physical - .physical_node(PhysicalCandidateId(class)) - .asap() - { - Some(ASAPOp::SummaryTumblingWindowChain { - family: FieldDataType::Sketch(kind, _), - .. - }) => match kind.params() { - asap_types::ir::schema::SketchParams::DDSketch { alpha } => Some(*alpha), - _ => None, - }, - _ => None, - }; - let weight = |query: usize, class: usize| { - inputs.accuracy_weights[query] - .get(&PhysicalCandidateId(class)) - .copied() - }; - let two_shares = 1.0 - 1.01f64.powf(-0.5); - let shared: Vec = (0..stages.physical.candidates.len()) - .filter(|&class| alpha(class).is_some()) - .filter(|&class| (0..3).all(|query| weight(query, class) == Some(AccuracyWeight::One))) - .collect(); - assert!(!shared.is_empty()); - for &class in &shared { - assert!( - (alpha(class).unwrap() - two_shares).abs() < 1e-9, - "{:?}", - alpha(class) - ); - } - // A chain sized for the whole budget weighs 1 for the ratio only where - // its values are the numerator's alone, so it never also serves both - // lone quantiles. - for class in (0..stages.physical.candidates.len()).filter(|&class| alpha(class) == Some(0.01)) { - assert!( - !(0..3).all(|query| weight(query, class) == Some(AccuracyWeight::One)), - "{class}" - ); - } -} - -// The checks keep what they build for later checks and skip repeated -// exact stand-ins: on RangeHeavy T8+T5's shape (a quotient and both -// operands alone), on a quotient of three quantiles (where pairs are -// excluded) and on a ratio with one approximate operand, the weights and -// excluded pairs equal those with no kept builds and no skipped stand-in. -#[test] -fn kept_builds_give_the_weights_of_fresh_builds() { - kept_builds_match_fresh_builds(&[ - "quantile_over_time(0.9, m[5m]) / quantile_over_time(0.5, m[5m])", - "quantile_over_time(0.9, m[5m])", - "quantile_over_time(0.5, m[5m])", - ]); - let three = kept_builds_match_fresh_builds(&[ - "quantile_over_time(0.9, m[5m]) / quantile_over_time(0.5, m[5m]) / quantile_over_time(0.1, m[5m])", - ]); - assert!(!three.accuracy_excluded_pairs[0].is_empty()); - // A scalar ratio of two output references whose approximate values sit - // under one operand only: the other reference's output candidates - // reach no member, and its first exact stand-in must still be built, - // or the ratio has no root to check and every approximate candidate - // would weigh nothing. - use asap_logical_asap_optimization::test_support::lower_promql; - use asap_types::ir::scalar::{ArithmeticOpKind, ExprSemantics}; - let operand = |query: &str| { - Box::new(ScalarExpr::PromqlScalarFromVector(lower_promql( - query, - target(), - ))) - }; - let ratio = QueryRoot::Scalar(ScalarExpr::Arithmetic { - op: ArithmeticOpKind::Div, - left: operand("avg_over_time(m[5m])"), - right: operand("quantile_over_time(0.5, m[5m])"), - semantics: ExprSemantics::Promql, - }); - let ratio = kept_builds_match_fresh_builds_of(vec![ratio]); - assert!(!ratio.accuracy_weights[0].is_empty()); -} - -/// Asserts that `queries`' accuracy weights and excluded pairs are the -/// same with kept builds and with none ([`accuracy_positions::FRESH_BUILDS`]); -/// returns the former. -fn kept_builds_match_fresh_builds(queries: &[&str]) -> MilpInputs { - use asap_logical_asap_optimization::test_support::lower_promql; - kept_builds_match_fresh_builds_of( - queries - .iter() - .map(|query| QueryRoot::Operator(lower_promql(query, target()))) - .collect(), - ) -} - -/// [`kept_builds_match_fresh_builds`] for query roots `roots`. -fn kept_builds_match_fresh_builds_of(roots: Vec) -> MilpInputs { - let stages = through_stages_1_and_2_of(roots); - let prepare = |fresh: bool| { - accuracy_positions::FRESH_BUILDS.with(|cell| cell.set(fresh)); - let inputs = prepare_milp_inputs( - &stages.physical, - &stages.entries, - &stages.data, - &CostInputs::default(), - ); - accuracy_positions::FRESH_BUILDS.with(|cell| cell.set(false)); - inputs - }; - let (kept, fresh) = (prepare(false), prepare(true)); - assert!(kept - .accuracy_weights - .iter() - .any(|weights| !weights.is_empty())); - assert_eq!(kept.accuracy_weights, fresh.accuracy_weights); - assert_eq!(kept.accuracy_excluded_pairs, fresh.accuracy_excluded_pairs); - kept -} - -// --------------------------------------------------------------------------- -// Ingestion pipelines per refresh window -// --------------------------------------------------------------------------- - -/// Ingestion-time views of metric `m`, each read by its own query: -/// - candidates 2 and 3: a KLL and a DDSketch over `TimeRange(10 s)` (one -/// refresh window, (10 s, 0)); -/// - 5: a KLL over `TimeRange(60 s)`, window (60 s, 0); -/// - 8: a KLL over `TimeRange(10 s)` over `TimeShift(5 s)`, window -/// (10 s, 5 s), through two `NotMaterialized` ingestion-time nodes; -/// - 9: a KLL over the whole scan, an unbounded window. -/// -/// Candidates 0, 1, 4, 6 and 7 (the scan, ranges and shift) are -/// `NotMaterialized` at ingestion time; 10–14 are the query-time readers. -fn ingestion_views_of_three_windows_and_an_unbounded_one() -> PhysicalASAPMemoCandidates { - use asap_types::ir::operator::{Reduction, Source, TimeShift}; - use asap_types::ir::scalar::ColumnRef; - use asap_types::ir::schema::{GroupingStrategy, SketchKind, SketchParams, SummaryUpdate}; - use asap_types::ir::TimeRangeKind; - use std::time::Duration; - let scan = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Scan { - source: Source::TimeSeries { metric: "m".into() }, - predicates: vec![], - schema: Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("value", DataType::Float64, false), - Field::plain("job", DataType::Utf8, true), - ], - 0, - vec![], - ), - })) - .unwrap(); - let range = |seconds: u64, child: &Rc| { - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeRange { - range: Duration::from_secs(seconds), - kind: TimeRangeKind::Range, - child: Rc::clone(child), - })) - .unwrap() - }; - let sketch = |algorithm: SketchAlgorithm, child: &Rc| { - let params = match algorithm { - SketchAlgorithm::Kll => SketchParams::Kll { - k: 200, - failure_probability: None, - }, - _ => SketchParams::DDSketch { alpha: 0.01 }, - }; - OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryAgg { - child: Rc::clone(child), - family: FieldDataType::Sketch(SketchKind::new(algorithm, params), Default::default()), - input: SummaryUpdate::column(ColumnRef::Named("value".into())), - reduction: Reduction::by(vec![2]), - grouping: GroupingStrategy::default(), - filter: None, - })) - .unwrap() - }; - let range_10s = range(10, &scan); - let range_60s = range(60, &scan); - let shifted = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeShift { - shift: TimeShift { - offset_ms: 5_000, - at: None, - }, - child: Rc::clone(&scan), - })) - .unwrap(); - let range_10s_shifted = range(10, &shifted); - let views = [ - sketch(SketchAlgorithm::Kll, &range_10s), - sketch(SketchAlgorithm::DDSketch, &range_10s), - sketch(SketchAlgorithm::Kll, &range_60s), - sketch(SketchAlgorithm::Kll, &range_10s_shifted), - sketch(SketchAlgorithm::Kll, &scan), - ]; - let mut nodes = vec![ - (scan, 0), - (range_10s, 1), - (views[0].clone(), 2), - (views[1].clone(), 3), - (range_60s, 4), - (views[2].clone(), 5), - (shifted, 6), - (range_10s_shifted, 7), - (views[3].clone(), 8), - (views[4].clone(), 9), - ]; - let at_ingestion = |node: usize, producers: &[&[usize]], view: bool| PhysicalCandidate { - timing: ExecutionTiming::IngestionTime, - materialization: if view { - Materialization::MatAtIngestion(Medium::Memory) - } else { - Materialization::NotMaterialized - }, - ..candidate(node, producers, false) - }; - let mut candidates = vec![ - at_ingestion(0, &[], false), - at_ingestion(1, &[&[0]], false), - at_ingestion(2, &[&[1]], true), - at_ingestion(3, &[&[1]], true), - at_ingestion(4, &[&[0]], false), - at_ingestion(5, &[&[4]], true), - at_ingestion(6, &[&[0]], false), - at_ingestion(7, &[&[6]], false), - at_ingestion(8, &[&[7]], true), - at_ingestion(9, &[&[0]], true), - ]; - let mut roots = Vec::new(); - for (index, view) in [2, 3, 5, 8, 9].into_iter().enumerate() { - let group = 10 + index as u32; - nodes.push((reader_of(&nodes[view].0), group)); - candidates.push(candidate(10 + index, &[&[view]], true)); - roots.push(root(group)); - } - catalog(nodes, candidates, roots) -} - -// One `I_r` per refresh window: the two views over `TimeRange(10 s)` share -// one; the 60 s view and the shifted 10 s view (window (10 s, 5 s), through -// a `NotMaterialized` shift and range) have their own; the unbounded view -// gets an `Alone` one. Each `I_r` holds the scan and the stateless nodes -// below its views, and each view's context reads across into its `I_r`. -#[test] -fn execution_contexts_have_one_ingestion_pipeline_per_refresh_window() { - let physical = ingestion_views_of_three_windows_and_an_unbounded_one(); - let links = CandidateLinks::new(&physical); - let alive = vec![true; physical.candidates.len()]; - let windows = refresh_windows(&physical, &links); - let window = |width_ms, phase_ms| RefreshWindow { width_ms, phase_ms }; - assert_eq!( - windows, - HashMap::from([ - (id(2), Some(window(10_000, 0))), - (id(3), Some(window(10_000, 0))), - (id(5), Some(window(60_000, 0))), - (id(8), Some(window(10_000, 5_000))), - (id(9), None), - ]) - ); - let contexts = execution_contexts( - &physical, - &links.kind, - &producer_candidates_left(&links, &alive), - &output_candidates(&physical, &alive), - &windows, - |_| true, - ); - let ingestion: Vec<(ContextKind, Vec)> = contexts.contexts[5..9] - .iter() - .map(|context| (context.kind, context.options.clone())) - .collect(); - let pipeline = |width_ms, phase_ms| { - ContextKind::Ingestion(IngestionPipeline::Window(window(width_ms, phase_ms))) - }; - assert_eq!( - ingestion, - vec![ - (pipeline(10_000, 0), vec![id(0), id(1)]), - (pipeline(10_000, 5_000), vec![id(0), id(6), id(7)]), - (pipeline(60_000, 0), vec![id(0), id(4)]), - ( - ContextKind::Ingestion(IngestionPipeline::Alone(id(9))), - vec![id(0)] - ), - ] - ); - assert_eq!( - contexts.ingestion_context_of_shareable, - BTreeMap::from([(id(2), 5), (id(3), 5), (id(5), 7), (id(8), 6), (id(9), 8)]) - ); - let shared = contexts.context_of_shareable[&id(8)]; - assert_eq!(contexts.contexts[shared].options, vec![id(7)]); -} - -// Every producer candidate of every input reads one view window, the -// assumption behind computing a refresh window from first producers. -#[test] -fn producers_of_an_input_read_one_view_window() { - let physical = ingestion_views_of_three_windows_and_an_unbounded_one(); - assert!(producer_window_disagreements_of_all(&physical).is_empty()); -} diff --git a/crates/plan-selection/src/lib.rs b/crates/plan-selection/src/lib.rs index 77100f7df..a4168acc7 100644 --- a/crates/plan-selection/src/lib.rs +++ b/crates/plan-selection/src/lib.rs @@ -1,5858 +1,27 @@ -//! #509 Stage 3, plan selection (MVP): the only stage that computes cost. -//! -//! Each CandidatePhysicalASAPDAG from Stage 2 is checked against every -//! query's accuracy target: the query's output meets it when its root -//! guarantee does (theoretical: Stage 1 composes it with the built-in -//! analytical rules, and this stage binds the statistics it left symbolic -//! from the data workload and the deployment's operand-domain contracts), -//! or when the deployment's empirical evidence shows this -//! candidate's output does (`AccuracyEvidenceProvider::output_guarantee`, -//! read only here). The deployment's accuracy model may instead accept the -//! output directly, whatever its guarantee says -//! (`AccuracyModel::accepts_output_regardless_of_guarantee`); the -//! accept-all model the facade passes to ignore accuracy guarantees -//! (`asap_accuracy::IgnoreAccuracyGuarantees`) accepts every output so, and -//! the report records each one the guarantee check would have rejected. Before that, for -//! a query with a target, Count-Min is admitted only over weights proven -//! non-negative, which the executor needs to run it; a query without one -//! constrains nothing. -//! A miss rejects the candidate as invalid, with a reason; only this stage -//! discards valid candidates. -//! A candidate whose evaluation of some query takes longer than that -//! query's latency bound, or than its repeat interval, is rejected as -//! invalid too ("Latency" below). -//! Every valid candidate's resource usage is measured node by node over its -//! physical DAG ([`ResourceUsage`](asap_cost::ResourceUsage)), and the deployment's [`CostModel`](asap_cost::CostModel) -//! chooses among them ("Cost models" below). The rest are reported valid -//! but costlier (or slower, or larger). -//! The group selection MILP ([`select_plan`], #1095) selects one candidate -//! per memo group and execution context without constructing the Cartesian -//! product of local candidates; it prices cost model 1 only (#888). -//! -//! ## Cost model: cost per second of wall time -//! -//! A candidate's cost is what running it costs per second ([`COST_UNIT`](asap_cost::COST_UNIT)), -//! so queries with different recurrences add up. A node here is one -//! operator node of the candidate's physical plan DAG (`PhysicalASAPDAG`), -//! not a machine. Each node's price for one -//! computation is multiplied by how often it is computed per second -//! ([`NodeCost::cost_per_second`](asap_cost::NodeCost::cost_per_second)): -//! -//! * A `NotMaterialized` node is recomputed for each consumer that reads it -//! and is not stored: it is charged once per consumer -//! ([`NodeCost::multiplier`](asap_cost::NodeCost::multiplier)) at the sum of their rates. Its consumers are -//! the queries whose sub-DAG reaches it and the materialized nodes above -//! it, counted without looking through a materialized node: a node below -//! one is computed when that node is. The `MatAtIngestion` consumers of -//! one refresh window (`PhysicalASAPDAG::refresh_window`: for a tumbling window -//! chain, its tumbling-window width) are refreshed in one ingestion run and count as -//! one consumer: a `Scan` under a KLL chain and a DDSketch chain of 10 s -//! tumbling windows is computed once per second of arriving data, under chains of -//! 10 s and 60 s tumbling windows twice. Stage 2 keeps a shared node one -//! node (it copies it only for readers at two timings). A shared -//! `NotMaterialized` input costs what reading it separately does (#509 -//! Example 4) only when every consumer reads the same window: a query-time `Scan` is priced over the longest window -//! read above it, so each consumer pays for that window, and a scan a 1 m -//! and a 5 m window share costs more than two separate scans. Stage 2 -//! offers the highest shared node built once per evaluation (`MatAtQuery`) -//! as another candidate. -//! * A `MatAtQuery` node is computed once for all the queries that reach -//! it, at their combined rate: queries repeating at the same interval are -//! evaluated together, so each distinct interval counts once. -//! * A `MatAtIngestion` node runs as data arrives: it is priced over one -//! second of arriving rows, once per second, whatever its readers' rates. -//! A tumbling-window chain maintained this way updates only its newest -//! tumbling window: each arriving row is placed in it and updates it. -//! * A query repeating every `I` ms runs `1000 / I` times per second, and an -//! estimated rate runs at that rate. One-off, scheduled and unknown -//! queries are spread over the settings' `non_repeating_query_horizon_seconds` -//! (default 3600 s): `invocations / horizon` per second. -//! -//! Under cost model 1, memory is charged as bytes held per second -//! ([`ResourceUsage::average_memory_bytes`](asap_cost::ResourceUsage::average_memory_bytes)), at -//! `cost_per_memory_byte_second` per byte per second. A `MatAtIngestion` -//! node keeps its state every second: twice for a whole window (the window -//! being built and the completed one queries read) and `N + 1` tumbling windows for a -//! chain of `N` (the `N` the query merges and the one being built). A -//! `MatAtQuery` node's state is reused only within one evaluation, so it is -//! held only while that evaluation runs: its bytes times the latency of the -//! longest query reading it, once per evaluation. A query-time -//! `NotMaterialized` node's working memory (a sketch being built, an exact -//! aggregate's values) is charged once per consumer that recomputes it: -//! for a query, its bytes times that query's latency, at its rate; for a -//! materialized node reading it (directly or through other -//! `NotMaterialized` nodes), its bytes times the latency of the longest -//! query reading that node, at the node's rate, as a `MatAtQuery` state is. -//! A `MatAtQueryKeptAcrossEvaluations` tumbling-window chain (#509 Example -//! 4's B3, offered by Stage 2 when the deployment keeps query-time results, -//! `DeploymentCapabilities::query_time_results_retained`) is kept across -//! evaluations: each evaluation scans one tumbling window of raw data and builds the -//! newest tumbling window, and the `N − 1` older tumbling windows, which complete the readers' -//! look-back, are kept every second (in memory, all `N` at the peak). When -//! Stage 2 offers both query-time options for a chain, plan selection -//! prices each and the cheaper wins. -//! -//! A query whose workload entry declares a range evaluation -//! ([`RangeEvaluation`](asap_types::workload::RangeEvaluation): each request evaluates it at every `step_ms` over -//! the last `span_ms`, as a dashboard's `query_range` does) is priced as -//! `S = span_ms / step_ms + 1` evaluations per request (61 for a 1 h span -//! at 1 min steps): -//! -//! * every query-time computation that is not kept across evaluations -//! (`NotMaterialized`, `MatAtQuery`) runs once per step, `S` times per -//! request; a `MatAtIngestion` state is maintained as before, and the -//! merge and estimate over it run once per step; -//! * a chain kept across evaluations is built once per request: Stage 2 -//! offers it only when the requests repeat at its tumbling-window width, so in the -//! steady state each request builds its newest tumbling window and every other tumbling window -//! any of its steps reads was built by an earlier request or step; -//! * a tumbling-window chain kept beyond one evaluation (ingestion time or kept -//! across evaluations) retains its input's offset plus its window plus -//! the longest `span_ms` among its readers (`RealizedPlan::tumbling_window_retention_ms`), -//! so the earliest step of the next request still finds its tumbling windows: that -//! many more tumbling windows are kept every second. -//! -//! Latency stays per evaluation (one step). Without a range evaluation -//! `S = 1` and the retention is the window, so every price is as above. -//! Memory a computation holds only while it runs counts in -//! [`ResourceUsage::peak_memory_bytes`](asap_cost::ResourceUsage::peak_memory_bytes), which cost models 2 and 3 read, and -//! under cost model 1 for the time it is held, as above. (A `NotMaterialized` -//! node computed at ingestion time is a streaming operator holding one row, -//! and is not charged.) -//! -//! Prices come from `asap-cost`'s analytical model (`analytical::estimate_operator`) over edge -//! statistics derived from the [`DataWorkload`](asap_types::workload::DataWorkload) and a fixed default group -//! count; summary build and estimation are priced as rows × sketch depth and -//! groups × k. A query-time scan reads the longest window above it (a time -//! shift moves a window without widening it); a time shift is free, and a -//! time range is priced by the rows it keeps. A tumbling-window chain is -//! priced as one build over its whole window plus one operation per row to -//! place it in its tumbling window, and keeps one state per group and tumbling window; a merge -//! costs one operation per state it reads. Kept state is sized by its bytes: a sketch by its parameters, a -//! fixed-size exact accumulator (sum, count, min, max, rate, ...) as one -//! value per group, and exact state that grows with the data (quantile, -//! top-k, distinct count, L2, entropy keep every value or item) as one value -//! per input row of its group, assuming rows spread evenly over groups and, -//! for distinct items, that every row may be distinct. These numbers are -//! illustrative, not calibrated: a binary, join or concat node is priced from -//! its first input only (MVP; follow-up). -//! A cost error on any candidate aborts selection; it is reachable only from -//! bad declared rates or bad [`CostInputs`](asap_cost::CostInputs). -//! -//! ## Measured operation costs -//! -//! A deployment whose runtime measured its own costs gives them per -//! operator ([`CostInputs::measured_operation_costs`](asap_cost::CostInputs::measured_operation_costs), -//! [`MeasuredOperationCosts`](asap_cost::MeasuredOperationCosts)): for an operator kind at ingestion or query -//! time (and, for a summary operator, its sketch or exact accumulator; for -//! an aggregate, each measure), CPU ns per unit and per computation. A node -//! with an entry takes `units × ns_per_unit + ns_per_invocation` per -//! computation, its scanning included; the units are what the operator -//! processes ([`OperatorKind`](asap_cost::OperatorKind): samples a scan reads, rows a build inserts, -//! states a merge reads, groups an estimate reads, ...). A query-time scan -//! is the fetch and decode of raw data samples from the database holding them, -//! so a plan that reads raw data pays for it. A sketch whose parameters -//! no entry has exactly is priced between the two measured configurations -//! of the same sketch nearest in state size, on a log scale, or from the -//! nearest one outside their range. A kept state's bytes are the entry's -//! measured `memory_bytes_per_state` when it has one. The newest window's -//! compaction (see "Latency") merges with the state's query-time merge -//! entry too, and each query request pays the table's `ns_per_evaluation` -//! once, on the node it reads first and in its latency. -//! -//! A node without an entry is priced from the formulas above, its CPU -//! operations times one calibration factor: the measured time of every -//! node the table prices, over all candidates, divided by what the -//! formulas give for those nodes. Without it a table that measures some -//! operators (a slow sketch) and not others (the exact plan's scan and -//! aggregate) makes the unmeasured plan look cheap only because its nodes -//! keep the formulas' scale. The factor is one global heuristic, not a -//! measurement: an unmeasured operator whose real cost is far from the -//! measured ones' ratio is still mispriced, so a partial table can still -//! mis-rank plans (e.g. a cheap raw scan and exact quantile priced at the -//! slow sketch's ratio make the exact rebuild look costlier than it is). -//! Only a table covering every operator kind the candidates use makes the -//! choice the measured one. Each node reports how it was priced -//! ([`NodePriceSource`](asap_cost::NodePriceSource)). -//! Measured times count as CPU operations of [`ResourceSpeed`](asap_cost::ResourceSpeed), so latency -//! and all three cost models read them like any other operation. -//! -//! ## Cost models -//! -//! [`CostInputs::cost_model`](asap_cost::CostInputs::cost_model) (the cost inputs [`select_plan`] receives) chooses how -//! usage becomes a choice; the default is cost model 1 with -//! [`ResourceUnitCost::illustrative`](asap_cost::ResourceUnitCost::illustrative). -//! [`select_plan`] searches under cost model 1 only; cost models 2 and 3 -//! are an `UnsupportedCostModel` error until #888 (the whole-plan reference -//! pricer still ranks plans under all three): -//! -//! * [`CostModel::UsageOverTime`](asap_cost::CostModel::UsageOverTime) (cost model 1, pay-as-you-go): each -//! node's CPU operations, scanned bytes and memory byte-seconds per second -//! (and disk bytes, unpriced by default) times their prices; the cheapest -//! candidate wins. -//! * [`CostModel::ProvisionedCapacity`](asap_cost::CostModel::ProvisionedCapacity) (cost model 2): the candidate's peak -//! memory, peak CPU cores and disk times their prices per second; the -//! cheapest wins. Peak memory assumes every evaluation overlaps: kept -//! in-memory state plus each computation's working memory, once per -//! consumer that recomputes it. Peak cores are the ingestion work's cores -//! plus one per query evaluating at once. -//! * [`CostModel::ClusterCapacity`](asap_cost::CostModel::ClusterCapacity) (cost model 3, own cluster): no prices. -//! A candidate over the cluster's memory, disk or cores is invalid, and -//! [`ClusterObjective`](asap_cost::ClusterObjective) chooses among the rest (lowest worst-query -//! latency, or least peak memory). -//! -//! Disk holds state materialized on `Medium::Disk` (Stage 2 places none -//! there yet) and the raw data query-time scans read: per source, the -//! longest window a scan reads. A deployment that does not retain raw data -//! (`DeploymentCapabilities::raw_data_retained`) keeps no raw data samples -//! after they arrive, so Stage 2 generates no query-time scan of raw data. -//! -//! ## Latency -//! -//! Following the sketch-bench RQE cost model (§5, batch latency), a query's -//! latency per evaluation is its longest chain of work, each step on a -//! full core of elastic CPU (no queueing): the compaction of the newest -//! window of an ingestion-time state it reads, then its own query-time -//! work. Its own work is the sum, over every query-time node it reaches -//! (shared ones included, each once), of one computation's CPU operations -//! and scanned bytes timed by [`ResourceSpeed`](asap_cost::ResourceSpeed)'s `ms_per_cpu_op` and -//! `ms_per_scanned_byte`. A state whose ingestion work needs `k` cores keeps -//! `k` partial copies of its newest window, and compacting them takes -//! `(k − 1)` merges for each of its states per window, each one operation -//! (times the calibration factor), or the state's measured merge time when -//! the measured table has one; with -//! one core there is nothing to compact. A query with -//! `Some(bound)` whose latency is above the -//! bound makes the candidate invalid, with the latency as the reason -//! ([`RejectionCause::LatencyBoundExceeded`]). With or without a bound, a -//! query repeating every `I` ms whose latency is above `I` makes the -//! candidate invalid too ([`RejectionCause::LatencyOverRepeatInterval`]): -//! each evaluation starts before the last one ends, so it falls further -//! behind with every evaluation. Only a fixed interval is checked: an -//! estimated rate, a schedule or a one-time query has none. A range -//! evaluation is checked by its latency per step, as priced: the latency -//! model gives each step a full core, so a request's steps may run in -//! parallel. Run one after another they take `S` times as long, which this -//! check does not assume. -//! [`ResourceUsage::latency_per_query`](asap_cost::ResourceUsage::latency_per_query) reports every query's latency. -//! -//! No memory limit applies outside cost model 3, where the cluster's memory -//! is declared ([`ClusterLimits::memory_bytes`](asap_cost::ClusterLimits::memory_bytes)); without one, a selected -//! plan whose peak memory is over `LARGE_PEAK_MEMORY_BYTES` (10^12) gets a -//! warning, which `asap_planner` reports ([`large_peak_memory_warning`]). - -#[cfg(test)] -use asap_types::workload::RepeatedDemand; +//! #509 Stage 3, plan selection (concepts doc +//! `memo-groups-properties-requirements.md`, section 3, "Stage 3: baseline +//! selection and realization", and sections 4 and 5). +//! +//! [`select_plan`] reads Stage 2's physical candidates $\mathcal C$ and the +//! admissible part of the physical edges $\mathcal E$, selects with a MILP +//! over $x_v$, $y_{u,c,i}$, $z_{q,v}$ and $t_v$ (constraints (A)–(I); +//! (G) and (H) hold because $y$ exists only on admissible edges), and +//! decodes $\{x = 1\}$ and $\{y = 1\}$ into a +//! `DagOfSelectedCandidates`, which the injected realizer builds. The +//! extensions add execution contexts $\kappa$ with $x^{\mathrm{mat}}_v$, +//! $x_{\kappa,v}$, $y_{\kappa,u,c,i}$ and the retention amount $h_v$ +//! (section 5), and accuracy at each query root, constraint (K), with +//! per-candidate error bounds $E_c$ and $D_c$ (section 4). Costs come from +//! `asap-cost`. -mod accuracy_binding; -mod checks; -mod models; -// #1095 §2.0: the group selection MILP's inputs (dominance, execution -// contexts, accuracy weights). -mod groups; -// #1095 §2.0: the group selection MILP and the decode of its solution. mod milp; +mod models; mod report; -// Stage 3's entry, `select_plan`, over the group selection MILP (#1095). mod select; -// The whole-plan selector behind `reference_pricer` and the tests. -#[cfg(any(test, feature = "reference-pricer"))] -mod selector; + pub use models::AccuracyCheckInputs; -pub(crate) use models::SelectionModels; pub use report::{ large_peak_memory_warning, AcceptedAccuracy, ErrorSelectingPlan, InexactObjective, RejectionCause, SearchReport, SearchStatus, Selection, }; pub use select::select_plan; -#[cfg(test)] -pub(crate) use selector::select_among_plans; - -/// The whole-workload reference pricer (feature `reference-pricer`): the -/// selector Stage 3 used before the memo search, over whole physical plans -/// a developer tool enumerates (`asap_devtools::memo_pricing`, -/// `planner_fuzz`). Not part of Stage 3's interface. -#[cfg(feature = "reference-pricer")] -pub mod reference_pricer { - pub use crate::report::{Rejection, WholePlanSelection}; - use crate::{AccuracyCheckInputs, ErrorSelectingPlan, SelectionModels}; - use asap_cost::CostInputs; - use asap_types::physical::PhysicalWorkloadPlan; - use asap_types::workload::{DataWorkload, QueryWorkloadEntry}; - - /// Check, price and rank every plan of `candidates`, keeping every - /// candidate: the selected one, each valid one's cost and accepted - /// accuracy, and why each other one lost. - /// - /// Note: with measured operation costs and no frozen calibration factor, - /// the formulas are calibrated over `candidates` alone, so costs are not - /// on the scale of a facade run's selection (`--price-enumerated`). - pub fn select_among_plans( - candidates: &[PhysicalWorkloadPlan], - entries: &[QueryWorkloadEntry], - data: &DataWorkload, - accuracy_checks: &AccuracyCheckInputs<'_>, - cost_inputs: &CostInputs, - ) -> Result { - crate::selector::select_among_plans( - candidates, - entries, - data, - SelectionModels::new(accuracy_checks, cost_inputs), - ) - } -} - -/// `roots` sized as Pass 1 sizes them for `targets` over `data` -/// ([`with_summaries_sized_for_targets`](asap_accuracy::sizing::with_summaries_sized_for_targets)). -#[cfg(test)] -pub(crate) fn sized_as_pass1( - roots: &[asap_types::ir::QueryRoot], - targets: &[asap_types::ir::properties::AccuracyTarget], - data: &asap_types::workload::DataWorkload, -) -> Vec { - asap_accuracy::sizing::with_summaries_sized_for_targets(roots, targets, data) - .expect("sized summaries rebuild") -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::accuracy_binding::bound_root_guarantee; - use crate::accuracy_binding::{bind_statistics, workload_statistic}; - use crate::checks::guarantee_check; - use crate::checks::{accuracy_check, CheckFailure}; - use crate::report::WholePlanSelection; - use crate::selector::PlanSelector; - use asap_accuracy::composition::INPUT_ROW_COUNT; - use asap_accuracy::estimators::hll; - use asap_accuracy::propagation::root_guarantee; - use asap_cost::analytical::AnalyticalCostError; - use asap_cost::execution_demand::evaluation_rate; - use asap_cost::state_bytes::{sketch_shape, summary_shape, StateInputRows}; - use asap_cost::{kept_tumbling_window_chains, node_map, plan_cost, NodeMap}; - use asap_cost::{lookup, StateFamily}; - use asap_cost::{ - CalibrationFit, ClusterLimits, ClusterObjective, CostInputs, CostModel, NodePriceSource, - OperatorKind, PlanCost, ResourceUsage, COST_SOURCE, MEASURED_COST_SOURCE, - MIXED_COST_SOURCE, - }; - use asap_cost::{MeasuredOperationCosts, OperatorCost, ResourceUnitCost}; - use asap_types::ir::operator::AggIntent; - use asap_types::ir::physical_export::{ - PhysicalASAPDAG, PhysicalASAPNodeId, PhysicalASAPOperatorPayload as Payload, - }; - use asap_types::ir::properties::{AccuracyGuarantee, AccuracyScope, ExecutionTiming}; - use asap_types::ir::scalar::ScalarValue; - use asap_types::ir::schema::DataType; - use asap_types::ir::schema::{ - FieldDataType, Schema, SketchAlgorithm, SketchParams, WeightDomain, - }; - use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode, QueryRoot}; - use asap_types::physical::PhysicalWorkloadPlan; - use asap_types::physical::{Materialization, Medium}; - use asap_types::workload::bounds::with_confirmed_weights; - use asap_types::workload::bounds::{declared_series, series_read, values_per_result}; - use asap_types::workload::{ - DataWorkload, QueryRecurrence, QueryWorkloadEntry, RangeEvaluation, - }; - use std::collections::{BTreeMap, BTreeSet}; - use std::rc::Rc; - - /// [`plan_cost`] of one candidate alone, its calibration factor fitted - /// over its own measured nodes (as [`select_plan`] fits it over all). - fn self_calibrated_cost( - dag: &PhysicalASAPDAG, - nodes: &NodeMap<'_>, - kept_chains: &std::collections::HashSet, - recurrence: &[QueryRecurrence], - range_evaluations: &[Option], - data: &DataWorkload, - settings: &CostInputs, - ) -> Result { - let at = |formula_scale| { - plan_cost( - dag, - nodes, - kept_chains, - recurrence, - range_evaluations, - data, - settings, - formula_scale, - ) - }; - let (_, fit) = at(1.0)?; - Ok(at(fit.factor())?.0) - } - - /// [`self_calibrated_cost`] with `dag`'s node map and kept tumbling-window chains. - fn price( - dag: &PhysicalASAPDAG, - recurrence: &[QueryRecurrence], - data: &DataWorkload, - settings: &CostInputs, - ) -> Result { - self_calibrated_cost( - dag, - &node_map(dag), - &kept_tumbling_window_chains(dag), - recurrence, - &[], - data, - settings, - ) - } - use asap_accuracy::{ - AccuracyEvidenceProvider, AccuracyModel, BuiltinAccuracyModel, NoAccuracyEvidence, - }; - use asap_logical_asap_optimization::test_support::{entry, entry_with, lower_promql}; - use asap_logical_asap_optimization::DeploymentCapabilities; - use asap_physical_asap_optimization::optimize_physical_asap; - use asap_types::ir::properties::AccuracyTarget; - use asap_types::ir::properties::{AccuracyMetric, BoundExpr, ProbabilityExpr}; - use asap_types::ir::scalar::expr_ir::ArithmeticOpKind; - use asap_types::ir::schema::{GroupingStrategy, SketchKind}; - use asap_types::ir::{ExprSemantics, ScalarExpr}; - use asap_types::workload::{ - DataArrival, LabelSetDistinctValues, PrometheusMetricType, SourceFacts, - }; - - /// `G` of `root` over `data` ([`values_per_result`]). - fn values_of(root: &QueryRoot, data: &DataWorkload) -> Option { - values_per_result(root, series_read(root, data), data) - } - - /// [`bound_root_guarantee`] of `root` over `data`. - fn bound_of( - root: &QueryRoot, - data: &DataWorkload, - models: &SelectionModels<'_>, - ) -> Option { - bound_root_guarantee(root, series_read(root, data), data, models) - } - - /// `n` queries repeating every 10 s, as Example 1's panels do. - fn recurrences(n: usize) -> Vec { - vec![repeating(10_000); n] - } - - /// A query repeating every `interval_ms`. - fn repeating(interval_ms: u32) -> QueryRecurrence { - QueryRecurrence::Repeated(RepeatedDemand::FixedInterval( - asap_types::workload::RepetitionInterval(interval_ms), - )) - } - - /// The metrics and tables the tests' queries read. - const TEST_SOURCES: [&str; 5] = ["m", "x", "a", "b", "t"]; - - /// Data arriving at 10 000 rows per second, each test source holding - /// 10 000 series. - fn data() -> DataWorkload { - with_series(Some(10_000)) - } - - /// [`data`] with every test source holding `series` (`None`: unknown). - fn with_series(series: Option) -> DataWorkload { - DataWorkload { - arrival: DataArrival::Ingesting { - rows_per_second: 10_000.0, - }, - sources: TEST_SOURCES - .iter() - .map(|source| { - let facts = SourceFacts { - series_count: series, - ..SourceFacts::default() - }; - (source.to_string(), facts) - }) - .collect(), - } - } - - /// Explicit domain assumption needed for interpolated DDS quantiles. - fn non_negative_data() -> DataWorkload { - let mut data = data(); - for facts in data.sources.values_mut() { - facts.values_non_negative = true; - } - data - } - - /// `data` with each of `types`' metrics declared of its type. - fn with_metric_types( - mut data: DataWorkload, - types: &[(&str, PrometheusMetricType)], - ) -> DataWorkload { - for (metric, metric_type) in types { - data.sources - .entry(metric.to_string()) - .or_default() - .metric_type = Some(*metric_type); - } - data - } - - /// Classic HLL's failure probability over a source binds only that - /// source's distinct bound: a bound keyed to another source leaves it - /// unknown. - #[test] - fn classic_hll_failure_probability_binds_only_its_own_sources_bound() { - let statistic = asap_logical_asap_optimization::test_support::hll_classic_failure_statistic( - 15, 0.05, "a", - ); - let bounded = |source: &str| { - let mut data = data(); - data.sources - .get_mut(source) - .unwrap() - .max_distinct_per_evaluation = Some(100); - data - }; - assert_eq!(workload_statistic(&statistic, None, &bounded("b")), None); - assert_eq!( - workload_statistic(&statistic, None, &bounded("a")), - Some(hll::classic_failure_probability(15, 0.05, 100)) - ); - assert!(hll::classic_failure_probability(15, 0.05, 100) <= 0.01); - } - - /// Exact (P1), whole-expression CMS+heap (P2) and CountSketch+heap (P3) realizations of - /// one approximate top-k query, in Pass 1 catalog order. - fn candidates() -> Vec { - let target = AccuracyTarget::max_error(0.01).failure_probability(0.001); - let root = lower_promql("topk by (job) (10, sum_over_time(m[1m]))", target.clone()); - let inventory = - asap_logical_asap_optimization::test_support::generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(root), - ]), - ) - .unwrap(); - let topk = inventory - .targets - .iter() - .position(|t| t.choice_count() > 2) - .unwrap(); - [0, 1, 2] - .into_iter() - .map(|index| { - let mut choice = vec![0; inventory.targets.len()]; - choice[topk] = index; - let roots: Vec<_> = - asap_logical_asap_optimization::test_support::compose_logical_candidate( - &inventory, &choice, - ) - .unwrap(); - let roots = sized_as_pass1(&roots, std::slice::from_ref(&target), &data()); - let mut candidate = optimize_physical_asap("L", &roots, &[], &data(), false) - .unwrap() - .remove(0); - candidate.id = format!("P{}", index + 1); - candidate - }) - .collect() - } - - /// A summary whose analytical guarantee misses the target is rejected as - /// invalid with a reason; the exact plan is then selected. (Top-K - /// estimates carry no guarantee since #575, so a quantile summary is used.) - #[test] - fn accuracy_failing_candidate_is_rejected_as_invalid() { - let planned = AccuracyTarget::max_error(0.01).failure_probability(0.001); - let candidates = all_candidates("quantile_over_time(0.99, m[1m])", &planned); - let strict = AccuracyTarget::max_error(1e-6).failure_probability(1e-9); - let summaries: Vec<_> = candidates - .iter() - .filter(|c| has_summary_estimate(c)) - .map(|c| c.id.as_str()) - .collect(); - assert!(!summaries.is_empty()); - let selection = select_among_plans( - &candidates, - &[entry_with(strict, repeating(10_000))], - &data(), - SelectionModels::builtin(), - ) - .unwrap(); - assert!(!summaries.contains(&selection.selected.as_str())); - for id in &summaries { - let rejection = selection.rejected.iter().find(|r| r.id == *id).unwrap(); - assert_eq!(rejection.cause, RejectionCause::AccuracyTargetMissed); - assert!( - rejection.reason.contains("misses target"), - "{}", - rejection.reason - ); - assert!(!selection.costs.contains_key(*id)); - } - } - - /// For a query with a target, Count-Min over weights not proven - /// non-negative is invalid. - #[test] - fn count_min_over_signed_weights_is_rejected_as_invalid() { - let target = AccuracyTarget::max_error(0.01).failure_probability(0.001); - let selection = select_among_plans( - &candidates(), - &[entry_with( - target - .clone() - .accuracy_metric(AccuracyMetric::ABSOLUTE_STREAM_L1), - repeating(10_000), - )], - &data(), - SelectionModels::builtin(), - ) - .unwrap(); - let p2 = selection.rejected.iter().find(|r| r.id == "P2").unwrap(); - assert_eq!( - p2.cause, - RejectionCause::CountMinWeightsNotProvenNonNegative - ); - assert!(p2.reason.contains("non-negative"), "{}", p2.reason); - } - - /// Count-Min over raw data samples of `m` is admitted only when the data - /// workload declares `m` a counter: a gauge, an undeclared metric or a - /// counter under another name leaves the requirement unmet. Admitted, - /// the top-k still needs evidence about its output. - #[test] - fn count_min_over_raw_data_samples_needs_a_declared_counter() { - use asap_types::workload::PrometheusMetricType; - let target = AccuracyTarget::max_error(0.01).failure_probability(0.001); - let with_types = |types: &[(&str, PrometheusMetricType)]| with_metric_types(data(), types); - for (types, admitted) in [ - (vec![("m", PrometheusMetricType::Counter)], true), - (vec![("m", PrometheusMetricType::Gauge)], false), - (vec![("m_total", PrometheusMetricType::Counter)], false), - (vec![], false), - ] { - let data = with_types(&types); - let selection = select_among_plans( - &candidates(), - &[entry_with( - target - .clone() - .accuracy_metric(AccuracyMetric::ABSOLUTE_STREAM_L1), - repeating(10_000), - )], - &data, - SelectionModels::builtin(), - ) - .unwrap(); - let p2 = selection.rejected.iter().find(|r| r.id == "P2").unwrap(); - // Admitted weights leave the top-k output without a guarantee, - // which the built-in model does not accept; unproven weights - // reject the plan before any accuracy check. - let expected_cause = if admitted { - RejectionCause::NoAccuracyGuarantee - } else { - RejectionCause::CountMinWeightsNotProvenNonNegative - }; - assert_eq!(p2.cause, expected_cause); - assert_eq!( - !p2.reason.contains("non-negative"), - admitted, - "{types:?}: {}", - p2.reason - ); - if admitted { - assert!(p2.reason.contains("no guarantee"), "{}", p2.reason); - } else { - assert!( - p2.reason.contains("not declared a counter"), - "{}", - p2.reason - ); - } - } - } - - /// The Count-Min sketch algorithms (plain or with a heap) `candidate` - /// builds. - fn count_min_builds(candidate: &PhysicalWorkloadPlan) -> Vec { - candidate - .dag - .nodes - .iter() - .filter_map(|node| match &node.payload { - Payload::ASAP(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), - .. - }) if matches!( - kind.algorithm(), - SketchAlgorithm::Cms | SketchAlgorithm::CmsWithHeap - ) => - { - Some(*kind.algorithm()) - } - _ => None, - }) - .collect() - } - - /// `increase` (and `rate`, `irate`) over a counter is non-negative, since - /// resets are corrected: the Count-Min-with-heap top-k of - /// `topk(10, sum by (job) (increase(data_total[5m])))` passes the weight - /// check when `data_total` is declared a counter, and goes on to the - /// accuracy and cost checks. Over a gauge it is still rejected for its - /// weights (issue #773). - #[test] - fn count_min_over_a_counter_derivative_needs_a_declared_counter() { - let target = AccuracyTarget::max_error(0.01) - .failure_probability(0.001) - .accuracy_metric(AccuracyMetric::ABSOLUTE_STREAM_L1); - // An `offset` moves the window, not the samples' sign. - for query in [ - "topk(10, sum by (job) (increase(data_total[5m])))", - "topk(10, sum by (job) (rate(data_total[5m])))", - "topk(10, sum by (job) (irate(data_total[5m])))", - "topk(10, sum by (job) (rate(data_total[5m] offset 1h)))", - ] { - let candidates = all_candidates(query, &target); - let count_min: Vec<&str> = candidates - .iter() - .filter(|c| count_min_builds(c).contains(&SketchAlgorithm::CmsWithHeap)) - .map(|c| c.id.as_str()) - .collect(); - assert!(!count_min.is_empty(), "{query}"); - for (metric_type, admitted) in [ - (PrometheusMetricType::Counter, true), - (PrometheusMetricType::Gauge, false), - ] { - let data = with_metric_types(data(), &[("data_total", metric_type)]); - let rejected = match select_among_plans( - &candidates, - &[entry_with(target.clone(), repeating(10_000))], - &data, - SelectionModels::builtin(), - ) { - Ok(selection) => selection.rejected, - Err(ErrorSelectingPlan::NoValidCandidate(rejected)) => rejected, - Err(error) => panic!("{query}: {error}"), - }; - for id in &count_min { - let weights_rejected = rejected.iter().any(|r| { - &r.id == id - && r.cause == RejectionCause::CountMinWeightsNotProvenNonNegative - }); - assert_eq!( - weights_rejected, !admitted, - "{query} over a {metric_type:?}, {id}: {rejected:?}" - ); - } - } - } - } - - /// The `_sum` series of a declared histogram family is a counter: with - /// only the family `latency` declared a histogram, the Count-Min-with-heap - /// top-k of `topk(10, sum by (job) (increase(latency_sum[5m])))` passes - /// the weight check. Undeclared, or with `latency` a gauge, it does not - /// (issue #1165). - #[test] - fn count_min_over_a_declared_histograms_sum_series_is_admitted() { - let target = AccuracyTarget::max_error(0.01) - .failure_probability(0.001) - .accuracy_metric(AccuracyMetric::ABSOLUTE_STREAM_L1); - let query = "topk(10, sum by (job) (increase(latency_sum[5m])))"; - let candidates = all_candidates(query, &target); - let count_min: Vec<&str> = candidates - .iter() - .filter(|c| count_min_builds(c).contains(&SketchAlgorithm::CmsWithHeap)) - .map(|c| c.id.as_str()) - .collect(); - assert!(!count_min.is_empty()); - for (family_type, admitted) in [ - (Some(PrometheusMetricType::Histogram), true), - (Some(PrometheusMetricType::Gauge), false), - (None, false), - ] { - let mut data = data(); - data.sources.insert( - "latency".into(), - SourceFacts { - series_count: Some(10_000), - metric_type: family_type, - ..SourceFacts::default() - }, - ); - let rejected = match select_among_plans( - &candidates, - &[entry_with(target.clone(), repeating(10_000))], - &data, - SelectionModels::builtin(), - ) { - Ok(selection) => selection.rejected, - Err(ErrorSelectingPlan::NoValidCandidate(rejected)) => rejected, - Err(error) => panic!("{family_type:?}: {error}"), - }; - for id in &count_min { - let weights_rejected = rejected.iter().any(|r| { - &r.id == id && r.cause == RejectionCause::CountMinWeightsNotProvenNonNegative - }); - assert_eq!( - weights_rejected, !admitted, - "latency declared {family_type:?}, {id}: {rejected:?}" - ); - } - } - } - - /// A gauge declared non-negative - /// ([`SourceFacts::values_non_negative`]) proves its raw data samples, and - /// plain sums of them, non-negative: the Count-Min-with-heap top-k of - /// `topk by (job) (10, sum_over_time(m[1m]))` (the ASAPQuery workloads' - /// top-k-sum shape) passes the weight check, and the selected plan carries - /// [`NonNegativeWeightProof::DeclaredNonNegativeRawDataSamples`]. A rate or - /// increase of that gauge stays unproven: a reset correction over a - /// gauge can still give a negative value. - #[test] - fn count_min_over_raw_data_samples_of_a_gauge_declared_non_negative_is_admitted() { - use asap_types::ir::schema::NonNegativeWeightProof; - let target = AccuracyTarget::max_error(0.01) - .failure_probability(0.001) - .accuracy_metric(AccuracyMetric::ABSOLUTE_STREAM_L1); - let mut gauge = with_metric_types(data(), &[("m", PrometheusMetricType::Gauge)]); - gauge.sources.get_mut("m").unwrap().values_non_negative = true; - for (query, admitted) in [ - ("topk by (job) (10, sum_over_time(m[1m]))", true), - ("topk(10, sum by (job) (increase(m[5m])))", false), - ] { - let candidates = all_candidates(query, &target); - let count_min: Vec<&PhysicalWorkloadPlan> = candidates - .iter() - .filter(|c| count_min_builds(c).contains(&SketchAlgorithm::CmsWithHeap)) - .collect(); - assert!(!count_min.is_empty(), "{query}"); - let rejected = match select_among_plans( - &candidates, - &[entry_with(target.clone(), repeating(10_000))], - &gauge, - SelectionModels::builtin(), - ) { - Ok(selection) => selection.rejected, - Err(ErrorSelectingPlan::NoValidCandidate(rejected)) => rejected, - Err(error) => panic!("{query}: {error}"), - }; - for candidate in &count_min { - let weights_rejected = rejected.iter().any(|r| { - r.id == candidate.id - && r.cause == RejectionCause::CountMinWeightsNotProvenNonNegative - }); - assert_eq!( - weights_rejected, !admitted, - "{query}, {}: {rejected:?}", - candidate.id - ); - if !admitted { - continue; - } - // The proof the executor reads. - let proofs: Vec = with_confirmed_weights(&candidate.roots, &gauge) - .iter() - .flat_map(QueryRoot::operator_refs) - .flat_map(OperatorNode::reachable) - .filter_map(|n| match n.asap() { - Some(ASAPOp::SummaryAgg { - input, - family: FieldDataType::Sketch(kind, _), - .. - }) if *kind.algorithm() == SketchAlgorithm::CmsWithHeap => { - Some(input.weight_domain.clone()) - } - _ => None, - }) - .collect(); - assert!(!proofs.is_empty(), "{}", candidate.id); - assert!( - proofs.iter().all(|domain| *domain - == WeightDomain::NonNegative { - proof: NonNegativeWeightProof::DeclaredNonNegativeRawDataSamples - }), - "{}: {proofs:?}", - candidate.id - ); - } - } - } - - /// The sign gate is an indicator that only 1 (proven) opens: any other - /// value a binder supplies, 0 included, leaves it unbound rather than - /// reading as a perfect bound. Other statistics pass through. - #[test] - fn the_sign_gate_opens_only_at_one() { - use asap_accuracy::test_support::raw_data_samples_sign_statistic; - let gated = AccuracyGuarantee::new( - AccuracyMetric::Relative, - BoundExpr::Product { - factors: vec![ - BoundExpr::Constant { value: 0.01 }, - BoundExpr::Unknown { - statistic: raw_data_samples_sign_statistic("m"), - }, - ], - }, - ProbabilityExpr::Constant { value: 0.001 }, - vec![], - ); - for value in [0.0, 0.5, 2.0, -1.0] { - assert_eq!( - bind_statistics(&gated, &|_| Some(value)) - .spec - .error - .evaluate(), - None, - "{value}" - ); - } - assert_eq!( - bind_statistics(&gated, &|_| Some(1.0)) - .spec - .error - .evaluate(), - Some(0.01) - ); - let mut norm = gated; - norm.spec.error = BoundExpr::Unknown { - statistic: INPUT_ROW_COUNT.into(), - }; - assert_eq!( - bind_statistics(&norm, &|_| Some(0.0)).spec.error.evaluate(), - Some(0.0) - ); - } - - /// A relative bound gated on the sign of one metric's samples is bound - /// when the data workload declares that metric a counter, and stays - /// unknown for a gauge, an undeclared metric or a counter by another - /// name. - #[test] - fn sign_of_counter_raw_data_samples_is_bound_from_the_declared_type() { - use asap_accuracy::test_support::raw_data_samples_sign_statistic; - let gated = AccuracyGuarantee::new( - AccuracyMetric::Relative, - BoundExpr::Product { - factors: vec![ - BoundExpr::Constant { value: 0.01 }, - BoundExpr::Unknown { - statistic: raw_data_samples_sign_statistic("m"), - }, - ], - }, - ProbabilityExpr::Constant { value: 0.001 }, - vec![], - ); - let node = lower_promql("sum(m)", AccuracyTarget::exact()); - let root = QueryRoot::Operator(Rc::new((*node).clone().with_guarantee(Some(gated)))); - let with = |types: &[(&str, PrometheusMetricType)]| with_metric_types(data(), types); - let bound = |data: &DataWorkload| { - bound_of(&root, data, &SelectionModels::builtin()) - .unwrap() - .spec - .error - .evaluate() - }; - assert_eq!( - bound(&with(&[("m", PrometheusMetricType::Counter)])), - Some(0.01) - ); - for types in [ - vec![("m", PrometheusMetricType::Gauge)], - vec![("m_total", PrometheusMetricType::Counter)], - vec![], - ] { - assert_eq!(bound(&with(&types)), None, "{types:?}"); - } - } - - /// Every consumer of a run sees the selected plan with its weight - /// requirements answered: the logical plan (`selected_logical`) and the - /// Offered one at a time and keeping only the best so far, the selector - /// selects what `select_among_plans` selects over every candidate at - /// once, at the same cost, under every cost model. - #[test] - fn streaming_selection_selects_what_select_plan_selects() { - let target = AccuracyTarget::max_error(0.01).failure_probability(0.001); - let root = lower_promql("quantile_over_time(0.99, m[5m])", target.clone()); - let entries = [entry_with(target, repeating(60_000))]; - let data = with_series(Some(10_000)); - let candidates = &exhaustive_candidates(&[QueryRoot::Operator(root)], &entries, &data); - assert!(candidates.len() > 2); - let cluster = |objective| { - CostModel::ClusterCapacity(ClusterLimits { - memory_bytes: u64::MAX, - disk_bytes: u64::MAX, - cpu_cores: None, - objective, - }) - }; - for cost_model in [ - CostModel::UsageOverTime(ResourceUnitCost::illustrative()), - CostModel::ProvisionedCapacity(ResourceUnitCost::illustrative()), - cluster(ClusterObjective::LowestLatency), - cluster(ClusterObjective::LeastMemory), - ] { - let settings = CostInputs { - cost_model, - ..CostInputs::default() - }; - let models = SelectionModels::builtin().with_cost_inputs(&settings); - let all = select_among_plans(candidates, &entries, &data, models).unwrap(); - let mut selector = PlanSelector::new(&entries, &data, models, false).unwrap(); - for candidate in candidates { - selector.offer(candidate).unwrap(); - } - let (streamed, built) = selector.finish().unwrap(); - assert_eq!(streamed.selected, all.selected); - assert_eq!(built.id, all.selected); - assert_eq!(streamed.plan, all.plan); - assert_eq!(streamed.costs[&streamed.selected], all.costs[&all.selected]); - assert_eq!(streamed.candidates_checked, candidates.len() as u64); - // Only the selected candidate is kept. - assert_eq!(streamed.costs.len(), 1); - } - } - - /// The selected plan's open requirement becomes a proof when confirmed - /// and stays unproven otherwise; the executor reads no data workload. - #[test] - fn confirmed_weights_rewrite_only_the_summary_update() { - use asap_types::ir::schema::NonNegativeWeightProof; - use asap_types::workload::PrometheusMetricType; - let p2 = &candidates()[1]; - let domains = |roots: &[QueryRoot]| -> Vec { - roots - .iter() - .flat_map(QueryRoot::operator_refs) - .flat_map(OperatorNode::reachable) - .filter_map(|n| match n.asap() { - Some(ASAPOp::SummaryAgg { input, .. }) => Some(input.weight_domain.clone()), - _ => None, - }) - .collect() - }; - assert_eq!( - domains(&p2.roots), - [WeightDomain::RawDataSamplesOfMetric { metric: "m".into() }] - ); - let counter = with_metric_types(data(), &[("m", PrometheusMetricType::Counter)]); - let confirmed = with_confirmed_weights(&p2.roots, &counter); - assert_eq!( - domains(&confirmed), - [WeightDomain::NonNegative { - proof: NonNegativeWeightProof::CounterRawDataSamples - }] - ); - let mut gauge = with_metric_types(data(), &[("m", PrometheusMetricType::Gauge)]); - gauge.sources.get_mut("m").unwrap().values_non_negative = true; - assert_eq!( - domains(&with_confirmed_weights(&p2.roots, &gauge)), - [WeightDomain::NonNegative { - proof: NonNegativeWeightProof::DeclaredNonNegativeRawDataSamples - }] - ); - assert_eq!( - domains(&with_confirmed_weights(&p2.roots, &data())), - [WeightDomain::UnknownOrSigned] - ); - // Nothing open: the same roots. - let exact = &candidates()[0]; - let QueryRoot::Operator(before) = &exact.roots[0] else { - panic!() - }; - let QueryRoot::Operator(after) = &with_confirmed_weights(&exact.roots, &counter)[0] else { - panic!() - }; - assert!(Rc::ptr_eq(before, after)); - } - - /// Each node is priced under its DAG id, and the total is the sum; - /// every candidate is either selected or rejected as costlier. The - /// CountSketch+heap estimate has no analytical guarantee, so deployment - /// evidence admits it. - #[test] - fn per_node_costs_cover_the_dag_and_sum_to_the_total() { - let candidates = candidates(); - let target = AccuracyTarget::max_error(0.01).failure_probability(0.001); - let selection = select_among_plans( - &candidates, - &[entry_with( - target - .clone() - .accuracy_metric(AccuracyMetric::ABSOLUTE_STREAM_L1), - repeating(10_000), - )], - &data(), - SelectionModels::builtin().with_evidence(&EveryOutputMeasuredExact), - ) - .unwrap(); - for candidate in candidates.iter().filter(|c| c.id != "P2") { - let cost = &selection.costs[&candidate.id]; - let ids: Vec<_> = candidate.dag.nodes.iter().map(|n| n.id).collect(); - let mut keys: Vec<_> = cost.per_node.keys().copied().collect(); - keys.sort(); - let mut sorted = ids.clone(); - sorted.sort(); - assert_eq!(keys, sorted); - let sum: f64 = cost - .per_node - .values() - .map(|n| n.cost_per_second.unwrap()) - .sum(); - assert_eq!(cost.cost_per_second.unwrap(), sum); - assert!(cost.cost_per_second.unwrap() > 0.0); - } - let costlier: Vec<_> = selection - .rejected - .iter() - .filter(|r| r.cause == RejectionCause::Costlier) - .collect(); - assert_eq!(costlier.len(), 1); - assert_ne!(costlier[0].id, selection.selected); - } - - /// Query 1 is `sum(...)` over query 2's output. Every node reached by - /// both queries is `NotMaterialized`, so recomputed for each: charged - /// twice. Materializing query 2's root (`MatAtQuery`) charges it once, - /// and the nodes below it once, since they run only when it does. - #[test] - fn not_materialized_shared_nodes_are_charged_per_query() { - let outer = lower_promql("sum(sum_over_time(m[1m]))", AccuracyTarget::exact()); - let inner = outer.children()[0].clone(); - let mut candidate = optimize_physical_asap( - "L", - &[QueryRoot::Operator(outer), QueryRoot::Operator(inner)], - &[], - &data(), - false, - ) - .unwrap() - .remove(0); - let [outer, inner] = candidate.dag.roots[..] else { - panic!("two roots") - }; - // Kept state is priced apart; this compares computations only. - let settings = unit_costs(|c| c.cost_per_memory_byte_second = 0.0); - let cost = price(&candidate.dag, &recurrences(2), &data(), &settings).unwrap(); - let multipliers: BTreeMap<_, _> = cost - .per_node - .iter() - .map(|(id, c)| (*id, c.multiplier)) - .collect(); - assert_eq!(multipliers[&outer], 1); - assert_eq!(multipliers[&inner], 2); - assert!(multipliers.len() > 2, "nodes below the shared root"); - for (id, times) in &multipliers { - if *id != outer { - assert_eq!(*times, 2, "node {id} is read by both queries"); - } - } - - let node = candidate - .dag - .nodes - .iter_mut() - .find(|n| n.id == inner) - .unwrap(); - node.materialization = Materialization::MatAtQuery(asap_types::physical::Medium::Memory); - let stored = price(&candidate.dag, &recurrences(2), &data(), &settings).unwrap(); - for (id, c) in &stored.per_node { - assert_eq!(c.multiplier, 1, "node {id} is computed once"); - let once = cost.per_node[id].cost_per_second.unwrap() / multipliers[id] as f64; - assert!( - (c.cost_per_second.unwrap() - once).abs() <= 1e-12 * once, - "node {id}" - ); - } - assert!(stored.cost_per_second.unwrap() < cost.cost_per_second.unwrap()); - } - - /// A `NotMaterialized` node two queries read is computed for each, so - /// its working memory is held once per query, each time for that - /// query's latency: `bytes · Σ rate_q · latency_q`, not the longest - /// latency at both rates. Query 1 is `sum(...)` over query 2's output, - /// so it runs longer; they repeat every 10 s and every 20 s. - #[test] - fn shared_not_materialized_memory_is_held_for_each_reading_query() { - let outer = lower_promql("sum(sum_over_time(m[1m]))", AccuracyTarget::exact()); - let inner = outer.children()[0].clone(); - let candidate = optimize_physical_asap( - "L", - &[QueryRoot::Operator(outer), QueryRoot::Operator(inner)], - &[], - &data(), - false, - ) - .unwrap() - .remove(0); - let recurrence = [repeating(10_000), repeating(20_000)]; - let cost = |price_per_byte_second| { - let settings = unit_costs(|c| c.cost_per_memory_byte_second = price_per_byte_second); - price(&candidate.dag, &recurrence, &data(), &settings).unwrap() - }; - let (with, without) = (cost(1.0), cost(0.0)); - let [outer_ms, inner_ms] = [0, 1].map(|q| with.usage.latency_per_query[q].total_ms()); - assert!(outer_ms > inner_ms, "{outer_ms} vs {inner_ms}"); - let mut checked = 0; - for (id, node) in &with.per_node { - let Some(held) = node.detail.split("; holds ").nth(1) else { - continue; - }; - if node.multiplier != 2 { - continue; - } - let bytes: f64 = held.split(' ').next().unwrap().parse().unwrap(); - let term = - node.cost_per_second.unwrap() - without.per_node[id].cost_per_second.unwrap(); - let expected = bytes * (0.1 * outer_ms + 0.05 * inner_ms) / 1_000.0; - assert!(close(term, expected), "node {id}: {term} vs {expected}"); - checked += 1; - } - assert!(checked > 0, "a shared node holds memory"); - } - - /// Every combination of Pass 1 candidates of one query, as Stage 2 - /// candidates `P1`, `P2`, ... - fn all_candidates(query: &str, target: &AccuracyTarget) -> Vec { - all_candidates_sized(query, target, &|roots| roots) - } - - /// [`all_candidates`] with each candidate's logical roots passed through - /// `size` before Stage 2. - fn all_candidates_sized( - query: &str, - target: &AccuracyTarget, - size: &dyn Fn(Vec) -> Vec, - ) -> Vec { - let root = lower_promql(query, target.clone()); - let inventory = - asap_logical_asap_optimization::test_support::generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(root), - ]), - ) - .unwrap(); - let mut choices = vec![vec![]]; - for target in &inventory.targets { - choices = choices - .into_iter() - .flat_map(|choice: Vec| { - (0..target.choice_count()).map(move |index| { - let mut choice = choice.clone(); - choice.push(index); - choice - }) - }) - .collect(); - } - choices - .iter() - .enumerate() - .map(|(index, choice)| { - let roots: Vec<_> = - asap_logical_asap_optimization::test_support::compose_logical_candidate( - &inventory, choice, - ) - .unwrap(); - let roots = size(roots); - let mut candidate = optimize_physical_asap("L", &roots, &[], &data(), false) - .unwrap() - .remove(0); - candidate.id = format!("P{}", index + 1); - candidate - }) - .collect() - } - - fn has_summary_estimate(candidate: &PhysicalWorkloadPlan) -> bool { - candidate - .dag - .nodes - .iter() - .any(|n| matches!(n.payload, Payload::ASAP(ASAPOp::SummaryEstimate { .. }))) - } - - /// [`per_series_kll`]'s candidates with their summaries sized as Pass 1 - /// sizes them for the target over `data` ([`sized_as_pass1`]). - fn per_series_kll_sized_as_pass1( - scope: AccuracyScope, - data: &DataWorkload, - ) -> (Vec, AccuracyTarget) { - let (_, target) = per_series_kll(scope); - let requested = target.clone().accuracy_scope(AccuracyScope::PerValue); - let candidates = - all_candidates_sized("quantile_over_time(0.99, x[5m])", &requested, &|roots| { - sized_as_pass1(&roots, std::slice::from_ref(&target), data) - }) - .into_iter() - .filter(|c| !kll_params(&c.roots).is_empty()) - .collect(); - (candidates, target) - } - - /// The KLL candidates of `quantile_over_time(0.99, x[5m])`, per series, - /// within rank error 0.01 at δ = 0.01 over `scope`. - fn per_series_kll(scope: AccuracyScope) -> (Vec, AccuracyTarget) { - let requested = AccuracyTarget::max_error(0.01) - .failure_probability(0.01) - .accuracy_metric(AccuracyMetric::Rank); - let candidates = all_candidates("quantile_over_time(0.99, x[5m])", &requested) - .into_iter() - .filter(|c| !kll_params(&c.roots).is_empty()) - .collect::>(); - assert!(!candidates.is_empty()); - let target = requested - .clone() - .accuracy_metric(AccuracyMetric::Rank) - .accuracy_scope(scope); - (candidates, target) - } - - /// Bad input: a workload with a different number of entries than a - /// candidate has query roots is an error, not a rejection. - #[test] - fn an_entry_count_mismatch_is_an_error() { - let target = AccuracyTarget::max_error(0.01).failure_probability(0.001); - let entries = vec![entry_with(target, repeating(10_000)); 2]; - let Err(ErrorSelectingPlan::EntryCount { - candidate, - roots, - entries, - }) = select_among_plans(&candidates(), &entries, &data(), SelectionModels::builtin()) - else { - panic!("an entry count mismatch is an error") - }; - assert_eq!((candidate.as_str(), roots, entries), ("P1", 1, 2)); - } - - fn kll_params(roots: &[QueryRoot]) -> Vec { - roots - .iter() - .flat_map(QueryRoot::operator_refs) - .flat_map(OperatorNode::reachable) - .filter_map(|node| match node.asap() { - Some(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), - .. - }) if *kind.algorithm() == SketchAlgorithm::Kll => Some(kind.params().clone()), - _ => None, - }) - .collect() - } - - /// Stage 3 only checks: a KLL Pass 1 sized per value (δ) misses a - /// whole-result target over `G` = 10 000 series and is rejected, not - /// enlarged; sized as Pass 1 sizes it for that target (δ / G = 1e-6, a - /// larger `k`), it is valid, and the selected plan runs that size. - #[test] - fn a_whole_result_target_is_checked_not_resized() { - let (per_value_sized, whole) = per_series_kll(AccuracyScope::WholeResult); - let entries = [entry_with(whole.clone(), repeating(10_000))]; - let Err(ErrorSelectingPlan::NoValidCandidate(rejected)) = select_among_plans( - &per_value_sized, - &entries, - &data(), - SelectionModels::builtin(), - ) else { - panic!("a per-value size misses the whole-result target") - }; - assert!(rejected - .iter() - .all(|r| r.cause == RejectionCause::AccuracyTargetMissed)); - let (sized, _) = per_series_kll_sized_as_pass1(AccuracyScope::WholeResult, &data()); - let selection = - select_among_plans(&sized, &entries, &data(), SelectionModels::builtin()).unwrap(); - let ( - SketchParams::Kll { k: per_value_k, .. }, - SketchParams::Kll { - k, - failure_probability, - }, - ) = ( - &kll_params(&per_value_sized[0].roots)[0], - &kll_params(&selection.plan)[0], - ) - else { - panic!("KLL builds") - }; - assert_eq!(*failure_probability, Some(0.01 / 10_000.0)); - assert!(k > per_value_k, "{k} vs {per_value_k}"); - let selected = sized.iter().find(|c| c.id == selection.selected).unwrap(); - assert_eq!(kll_params(&selection.plan), kll_params(&selected.roots)); - } - - /// With its metric declared a counter, the whole-expression CMS+heap - /// candidate (P2) proves its weights non-negative, is valid and is - /// priced: its per-node costs cover its DAG and sum to its total. - #[test] - fn a_declared_counter_prices_the_count_min_heap_candidate() { - use asap_types::workload::PrometheusMetricType; - let candidates = candidates(); - let target = AccuracyTarget::max_error(0.01) - .failure_probability(0.001) - .accuracy_metric(AccuracyMetric::ABSOLUTE_STREAM_L1); - let counter = with_metric_types(data(), &[("m", PrometheusMetricType::Counter)]); - let selection = select_among_plans( - &candidates, - &[entry_with(target, repeating(10_000))], - &counter, - SelectionModels::builtin().with_evidence(&EveryOutputMeasuredExact), - ) - .unwrap(); - let p2 = candidates.iter().find(|c| c.id == "P2").unwrap(); - let cost = &selection.costs["P2"]; - let mut ids: Vec<_> = p2.dag.nodes.iter().map(|n| n.id).collect(); - ids.sort(); - let mut keys: Vec<_> = cost.per_node.keys().copied().collect(); - keys.sort(); - assert_eq!(keys, ids); - let sum: f64 = cost - .per_node - .values() - .map(|n| n.cost_per_second.unwrap()) - .sum(); - assert_eq!(cost.cost_per_second.unwrap(), sum); - } - - /// The KLL failure probability each selected sketch is sized for. - fn kll_failure_probabilities(roots: &[QueryRoot]) -> Vec> { - kll_params(roots) - .into_iter() - .map(|params| match params { - SketchParams::Kll { - failure_probability, - .. - } => failure_probability, - other => panic!("{other:?}"), - }) - .collect() - } - - /// A target on the groups holding at least 5% of the records covers at - /// most 20 of them, so Pass 1 sizes each sketch for δ / 20 and the target is - /// met even when the number of values the result returns is unbound - /// (here a source with no declared series count), where a whole-result - /// target stays unmet. With 10 000 series declared it still needs only - /// δ / 20, not the whole result's δ / 10 000. - #[test] - fn a_record_share_target_sizes_each_sketch_for_one_over_share_groups() { - for data in [with_series(None), data()] { - let (candidates, five_percent) = per_series_kll_sized_as_pass1( - AccuracyScope::GroupsWithRecordShareAtLeast(0.05), - &data, - ); - let selection = select_among_plans( - &candidates, - &[entry_with(five_percent.clone(), repeating(10_000))], - &data, - SelectionModels::builtin(), - ) - .expect("a KLL candidate meets the 5% target"); - assert_eq!( - kll_failure_probabilities(&selection.plan), - vec![Some(0.01 / 20.0)] - ); - } - } - - /// Without a bound on the values one evaluation returns, a per-value - /// guarantee does not meet a whole-result target: a PromQL vector whose - /// source declares no series count, and a SQL relation (until a bound on - /// its groups is an input). A scalar returns one value. - #[test] - fn an_unbound_number_of_values_leaves_a_whole_result_target_unmet() { - let (candidates, whole) = per_series_kll(AccuracyScope::WholeResult); - let no_series_count = with_series(None); - let Err(ErrorSelectingPlan::NoValidCandidate(rejected)) = select_among_plans( - &candidates, - &[entry_with(whole, repeating(10_000))], - &no_series_count, - SelectionModels::builtin(), - ) else { - panic!("the KLL candidates are invalid") - }; - assert!( - rejected[0].reason.contains("is not bound"), - "{}", - rejected[0].reason - ); - let table = asap_logical_asap_optimization::test_support::scan_from( - asap_types::ir::operator::Source::Table { - table_ref: "t".into(), - }, - Schema::new(vec![asap_types::ir::schema::Field::plain( - "v", - DataType::Float64, - false, - )]), - ); - assert_eq!(values_of(&QueryRoot::Operator(table), &data()), None); - assert_eq!( - values_of( - &QueryRoot::Scalar(ScalarExpr::literal_f64(1.0)), - &DataWorkload::default() - ), - Some(1) - ); - } - - /// A root that returns at most one row has `G` = 1, provable from the - /// query with no data workload: a global aggregate, PromQL or SQL, and - /// what selects, renames or joins single rows above it. A grouped SQL - /// aggregate stays unbound, and a grouped PromQL one, or a global top-k - /// (k rows), is bound by the series count of the source it reads. - #[test] - fn a_root_of_at_most_one_row_returns_one_value() { - let none = DataWorkload::default(); - let root = - |query: &str| QueryRoot::Operator(lower_promql(query, AccuracyTarget::max_error(0.01))); - assert_eq!(values_of(&root("sum(rate(x[5m]))"), &none), Some(1)); - assert_eq!(values_of(&root("topk(5, rate(x[5m]))"), &none), None); - assert_eq!(values_of(&root("sum by (job) (rate(x[5m]))"), &none), None); - assert_eq!( - values_of(&root("sum by (job) (rate(x[5m]))"), &data()), - Some(10_000) - ); - let table = || { - asap_logical_asap_optimization::test_support::scan_from( - asap_types::ir::operator::Source::Table { - table_ref: "t".into(), - }, - Schema::new(vec![ - asap_types::ir::schema::Field::plain("k", DataType::Utf8, false), - asap_types::ir::schema::Field::plain("v", DataType::Float64, false), - ]), - ) - }; - let count = || AggIntent::Count { - accuracy: AccuracyTarget::exact(), - }; - let global = asap_logical_asap_optimization::test_support::agg(vec![], count(), table()); - let grouped = asap_logical_asap_optimization::test_support::agg(vec![0], count(), table()); - assert_eq!( - values_of(&QueryRoot::Operator(global.clone()), &none), - Some(1) - ); - assert_eq!(values_of(&QueryRoot::Operator(grouped), &data()), None); - let cross = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Join { - kind: asap_types::ir::operator::operator_properties::JoinKind::Cross, - pred: asap_types::ir::Predicate(ScalarExpr::Literal(ScalarValue::Boolean(true))), - left: global.clone(), - right: global, - })) - .unwrap(); - assert_eq!(values_of(&QueryRoot::Operator(cross), &none), Some(1)); - } - - /// `G` and the input row count are bound per query from the sources - /// it reads: another source's series do not count, a query over two - /// sources is bound by their sum, and one source without a series - /// count leaves it unbound. Pricing reads the same per-source series, with - /// the default for an undeclared source. - #[test] - fn values_per_result_is_bound_by_the_series_of_the_sources_read() { - let root = - |query: &str| QueryRoot::Operator(lower_promql(query, AccuracyTarget::max_error(0.01))); - let series = |declared: &[(&str, u64)]| DataWorkload { - sources: declared - .iter() - .map(|(source, series)| { - let facts = SourceFacts { - series_count: Some(*series), - ..SourceFacts::default() - }; - (source.to_string(), facts) - }) - .collect(), - ..DataWorkload::default() - }; - let x = root("sum by (job) (rate(x[5m]))"); - let both = root("sum by (job) (rate(x[5m])) + sum by (job) (rate(m[5m]))"); - assert_eq!(values_of(&x, &series(&[("x", 10)])), Some(10)); - assert_eq!(values_of(&x, &series(&[("m", 10)])), None); - let declared = series(&[("x", 10), ("m", 20)]); - assert_eq!(values_of(&x, &declared), Some(10)); - assert_eq!(values_of(&both, &declared), Some(30)); - assert_eq!(series_read(&both, &declared), Some(30)); - assert_eq!(values_of(&both, &series(&[("x", 10)])), None); - // Pricing sums the series of the sources a node reads; an - // undeclared source, or none, leaves them unknown. - assert_eq!(declared_series(&["x", "m"].into(), &declared), Some(30)); - assert_eq!(declared_series(&["x", "y"].into(), &declared), None); - assert_eq!(declared_series(&BTreeSet::new(), &declared), None); - } - - /// The exact physical DAG of `queries` planned together, over `data`. - fn exact_dag(queries: &[&str], data: &DataWorkload) -> PhysicalASAPDAG { - let roots: Vec<_> = queries - .iter() - .map(|query| { - let root = lower_promql(query, AccuracyTarget::exact()); - QueryRoot::Operator(root) - }) - .collect(); - let inventory = - asap_logical_asap_optimization::test_support::generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&roots), - ) - .unwrap(); - let choice = vec![0; inventory.targets.len()]; - let roots: Vec<_> = - asap_logical_asap_optimization::test_support::compose_logical_candidate( - &inventory, &choice, - ) - .unwrap(); - optimize_physical_asap("L1", &roots, &[], data, false) - .unwrap() - .remove(0) - .dag - } - - /// A node's operator, timing, work and price, to compare nodes across - /// DAGs whose ids differ. - fn node_prices(cost: &PlanCost) -> Vec<(OperatorKind, ExecutionTiming, String, f64)> { - cost.per_node - .values() - .map(|node| { - ( - node.operator, - node.timing, - node.detail.clone(), - node.cost_per_second.unwrap(), - ) - }) - .collect() - } - - /// No fact bounds the rows of a scan whose source declares neither an - /// arrival rate (`rows_per_second`, or `series_count` with - /// `sample_interval`) nor `rows_held`: pricing refuses, naming the - /// source and the missing fact, instead of assuming a rate. - #[test] - fn a_scan_without_a_row_fact_is_refused_naming_its_source() { - let data = DataWorkload::default(); - let error = price( - &exact_dag(&["sum(rate(m[5m]))"], &data), - &recurrences(1), - &data, - &CostInputs::default(), - ) - .unwrap_err() - .1; - assert!( - matches!(&error, AnalyticalCostError::MissingSourceFact { source_name, .. } if source_name == "m"), - "{error}" - ); - assert!(error.to_string().contains("rows_held"), "{error}"); - } - - // #955 example 1: `rate(m[5m] offset 1h)` at query time reads the 5 - // minutes `(T - 65m, T - 60m]`, so it scans the samples `rate(m[5m])` - // scans but keeps and prices the 65 minutes of raw `m` `rate(m[65m])` - // keeps. - #[test] - fn an_offset_window_at_query_time_keeps_its_whole_raw_lookback() { - // The scan's samples read and raw data bytes kept, from its detail. - let scan_of = |query: &str| { - let dag = exact_dag(&[query], &data()); - let scan = node_of(&dag, is_scan); - let cost = price(&dag, &recurrences(1), &data(), &CostInputs::default()).unwrap(); - let detail = &cost.per_node[&scan].detail; - let kept_bytes: u64 = detail - .split("; keeps ") - .find_map(|part| { - part.split(" bytes of raw data on disk") - .next()? - .parse() - .ok() - }) - .unwrap_or_else(|| panic!("no raw data kept: {detail}")); - (scanned_samples(&cost, scan), kept_bytes) - }; - let (offset_samples, offset_kept) = scan_of("sum(rate(m[5m] offset 1h))"); - let (five_minutes_samples, five_minutes_kept) = scan_of("sum(rate(m[5m]))"); - let (_, sixty_five_minutes_kept) = scan_of("sum(rate(m[65m]))"); - assert_eq!(offset_samples, five_minutes_samples); - assert_eq!(offset_kept, sixty_five_minutes_kept); - assert_eq!(offset_kept, 13 * five_minutes_kept); - } - - /// Data at rest is bounded by the rows its source holds: a 5-minute - /// window of `m`, which holds 5,000 rows and declares no arrival rate, - /// reads at most those 5,000. - #[test] - fn rows_held_bound_a_scan_of_data_at_rest() { - let data = DataWorkload { - arrival: DataArrival::AtRest, - sources: [( - "m".to_string(), - SourceFacts { - rows_held: Some(5_000), - ..SourceFacts::default() - }, - )] - .into(), - }; - let cost = price( - &exact_dag(&["sum(rate(m[5m]))"], &data), - &recurrences(1), - &data, - &CostInputs::default(), - ) - .unwrap(); - let details: Vec = node_prices(&cost) - .into_iter() - .map(|(_, _, d, _)| d) - .collect(); - assert!( - details.iter().any(|d| d.starts_with("scan 5000 samples;")), - "{details:?}" - ); - } - - /// With no declared `series_count`, a per-series reduction is priced at - /// one state per row read (each series has at least one), not at an - /// assumed series count: 1,000 rows per second over 5 minutes. - #[test] - fn undeclared_series_are_bound_by_the_rows_read() { - let data = DataWorkload { - arrival: DataArrival::Ingesting { - rows_per_second: 1_000.0, - }, - sources: [("m".to_string(), SourceFacts::default())].into(), - }; - let cost = price( - &exact_dag(&["sum(rate(m[5m]))"], &data), - &recurrences(1), - &data, - &CostInputs::default(), - ) - .unwrap(); - let details: Vec = node_prices(&cost) - .into_iter() - .map(|(_, _, d, _)| d) - .collect(); - assert!( - details.iter().any(|d| d.contains("into 300000 states")), - "{details:?}" - ); - } - - /// Each scan is priced at its own source's rate, `series_count` series - /// each sampled every `sample_interval`, not at the whole workload's - /// `rows_per_second`: adding a query on a second metric, which a - /// runtime declares by raising `rows_per_second` by that metric's rate, - /// leaves the first metric's node prices unchanged. - #[test] - fn a_query_on_another_metric_leaves_a_metrics_node_prices_unchanged() { - // `a` and `b`: 10 000 series sampled every 100 ms, 100 000 rows/s each. - let ingesting = |rows_per_second| DataWorkload { - arrival: DataArrival::Ingesting { rows_per_second }, - sources: ["a", "b"] - .into_iter() - .map(|metric| { - let facts = SourceFacts { - series_count: Some(10_000), - sample_interval: Some(asap_types::workload::DurationMs(100)), - ..SourceFacts::default() - }; - (metric.to_string(), facts) - }) - .collect(), - }; - let settings = CostInputs::default(); - let on_a = "sum(sum_over_time(a[1m]))"; - let alone = ingesting(100_000.0); - let alone = price( - &exact_dag(&[on_a], &alone), - &recurrences(1), - &alone, - &settings, - ) - .unwrap(); - let joint = ingesting(200_000.0); - let joint = price( - &exact_dag(&[on_a, "sum(sum_over_time(b[1m]))"], &joint), - &recurrences(2), - &joint, - &settings, - ) - .unwrap(); - let mut joint_nodes = node_prices(&joint); - for node in node_prices(&alone) { - let found = joint_nodes.iter().position(|other| *other == node); - assert!(found.is_some(), "{node:?} not in {joint_nodes:#?}"); - joint_nodes.remove(found.unwrap()); - } - // Each metric's scan reads its own 100 000 rows per second. - assert_eq!( - joint.usage.scanned_bytes_per_second, - 2.0 * alone.usage.scanned_bytes_per_second - ); - } - - /// A partitioned sort and limit (`topk by (job)`) take their partition - /// count from `job`'s declared distinct values. With `job` undeclared, - /// no count is invented: each series is in one partition and each row - /// read in at most one, so the partitions are at most the series and - /// at most the rows the sort reads. - #[test] - fn a_partitioned_topk_has_one_partition_per_declared_label_value() { - let details = |declared: &[(&str, u64)], series: u64| { - let mut data = with_series(Some(series)); - data.sources.get_mut("m").unwrap().distinct_values_per_label = declared - .iter() - .map(|(label, count)| (label.to_string(), *count)) - .collect(); - let cost = price( - &exact_dag(&["topk by (job) (3, m)"], &data), - &recurrences(1), - &data, - &CostInputs::default(), - ) - .unwrap(); - node_prices(&cost) - .into_iter() - .filter(|(operator, ..)| { - matches!(operator, OperatorKind::Sort | OperatorKind::Limit) - }) - .map(|(_, _, detail, _)| detail) - .collect::>() - }; - let declared = details(&[("job", 10)], 10_000); - assert!( - declared.iter().any(|d| d.contains(" in 10 partitions;")), - "{declared:?}" - ); - assert!( - declared.iter().any(|d| d.starts_with("limit to 30 rows;")), - "{declared:?}" - ); - // Undeclared: the 2,000 rows read, fewer than the 10,000 series. - let undeclared = details(&[("instance", 10)], 10_000); - assert!( - undeclared - .iter() - .any(|d| d.starts_with("sort 2000 rows in 2000 partitions;")), - "{undeclared:?}" - ); - assert!( - undeclared - .iter() - .any(|d| d.starts_with("limit to 2000 rows;")), - "{undeclared:?}" - ); - // 50 series hold at most 50 groups, declared or not. - let few = details(&[("job", 10_000)], 50); - assert!( - few.iter().any(|d| d.contains(" in 50 partitions;")), - "{few:?}" - ); - let few = details(&[], 50); - assert!( - few.iter().any(|d| d.contains(" in 50 partitions;")), - "{few:?}" - ); - } - - /// `x` holding 10,000 series over 4 regions, 25 services and 625 - /// endpoints, each endpoint in one service (the eval's RollupHeavy - /// `http`), with `(service, endpoint)`'s 625 combinations declared when - /// `declared_set`. - fn hierarchical_labels(declared_set: bool) -> DataWorkload { - let mut data = with_series(Some(10_000)); - let facts = data.sources.get_mut("x").unwrap(); - facts.distinct_values_per_label = [ - ("region".to_string(), 4), - ("service".to_string(), 25), - ("endpoint".to_string(), 625), - ] - .into(); - if declared_set { - facts.distinct_values_per_label_set = vec![LabelSetDistinctValues { - labels: vec!["service".to_string(), "endpoint".to_string()], - distinct_values: 625, - }]; - } - data - } - - /// A grouping is priced at the smallest bound the declared facts give: - /// the declared `(service, endpoint)` set's 625 combinations, not - /// 25 x 625 capped at the 10,000 series; a superset grouping at the set - /// times `region`'s 4; without the set, at the per-label product capped - /// at the series. A partitioned top-k takes the same count. - #[test] - fn a_grouping_is_priced_at_its_declared_label_set_count() { - let details = |query: &str, declared_set: bool| { - let data = hierarchical_labels(declared_set); - let cost = price( - &exact_dag(&[query], &data), - &recurrences(1), - &data, - &CostInputs::default(), - ) - .unwrap(); - node_prices(&cost) - .into_iter() - .map(|(_, _, detail, _)| detail) - .collect::>() - }; - let sum = |keys: &str| format!("sum by ({keys}) (sum_over_time(x[15m]))"); - for (query, declared_set, expected) in [ - (sum("service, endpoint"), true, "into 625 states"), - (sum("region, service, endpoint"), true, "into 2500 states"), - ( - sum("service, endpoint"), - false, - "Sum accumulator into 10000 states", - ), - (sum("region, service"), true, "into 100 states"), - ( - "topk by (service, endpoint) (3, sum_over_time(x[15m]))".to_string(), - true, - " in 625 partitions;", - ), - // A relabeled key no longer holds the declared set's values: - // `endpoint` now joins `region` and `endpoint` (up to 4 x 625). - ( - r#"sum by (service, endpoint) (label_join(sum_over_time(x[15m]), "endpoint", "-", "region", "endpoint"))"#.to_string(), - true, - "into 10000 states", - ), - // A binary operator pairs the labels of two rows: per-label only. - ( - "sum by (service, endpoint) (sum_over_time(x[15m]) / on (service) group_left (endpoint) sum_over_time(x[15m]))".to_string(), - true, - "into 10000 states", - ), - ] { - let details = details(&query, declared_set); - assert!( - details.iter().any(|d| d.contains(expected)), - "{query} (set declared: {declared_set}): {details:#?}" - ); - } - } - - /// A grouped root's `G` takes the declared label set's count too: the - /// set's 625 for `(service, endpoint)`, times `region`'s 4 for a - /// superset; without the set, the per-label product capped at the - /// series. - #[test] - fn values_per_result_is_bound_by_a_declared_label_set() { - let root = - |query: &str| QueryRoot::Operator(lower_promql(query, AccuracyTarget::max_error(0.01))); - for (query, declared_set, values) in [ - ("sum by (service, endpoint) (rate(x[5m]))", true, 625), - ( - "sum by (region, service, endpoint) (rate(x[5m]))", - true, - 2_500, - ), - ("sum by (service, endpoint) (rate(x[5m]))", false, 10_000), - ] { - assert_eq!( - values_of(&root(query), &hierarchical_labels(declared_set)), - Some(values), - "{query} (set declared: {declared_set})" - ); - } - } - - /// A grouped root's `G` is also bound by its keys' declared distinct - /// values, at most the series read: `by (job)` with 10 jobs returns at - /// most 10 values, also below a sort; a key without a declared count, a - /// relabeled key, `without (...)` and a top-k (several rows per group) - /// keep the series bound. - #[test] - fn values_per_result_is_bound_by_the_declared_distinct_values_of_its_keys() { - let root = - |query: &str| QueryRoot::Operator(lower_promql(query, AccuracyTarget::max_error(0.01))); - let mut data = with_series(Some(10_000)); - data.sources.get_mut("x").unwrap().distinct_values_per_label = - [("job".to_string(), 10), ("instance".to_string(), 5_000)].into(); - for (query, values) in [ - ("sum by (job) (rate(x[5m]))", 10), - ("sum by (job, instance) (rate(x[5m]))", 10_000), - ("sum by (job, region) (rate(x[5m]))", 10_000), - ("sum without (instance) (rate(x[5m]))", 10_000), - ("topk by (job) (3, rate(x[5m]))", 10_000), - ("sum by (job) (rate(x[5m]) + rate(m[5m]))", 20_000), - // A relabeled key no longer holds the declared label's values. - ( - r#"sum by (job) (label_replace(rate(x[5m]), "job", "$1", "instance", "(.*)"))"#, - 10_000, - ), - ("sort_desc(sum by (job) (rate(x[5m])))", 10), - // `count_values("job", ...)` overwrites `job` with sample values. - ( - r#"sum by (job) (count_values by (job) ("job", rate(x[5m])))"#, - 10_000, - ), - ] { - assert_eq!(values_of(&root(query), &data), Some(values), "{query}"); - } - } - - /// `G` counts the values the root can return: an `or` returns the - /// values of both branches (here up to `series(x)` series plus the - /// groups of `x`), a constant vector one more, while a join or a fold - /// of one source counts it once. - #[test] - fn values_per_result_adds_the_branches_of_an_or() { - let root = - |query: &str| QueryRoot::Operator(lower_promql(query, AccuracyTarget::max_error(0.01))); - let data = with_series(Some(10)); - for (query, values) in [ - ("rate(x[5m]) or sum without (instance) (rate(x[5m]))", 20), - ("rate(x[5m]) or vector(0)", 11), - ("rate(x[5m]) or rate(m[5m])", 20), - ("(rate(x[5m]) or vector(0)) or vector(1)", 12), - ("sum by (job) (rate(x[5m]) or rate(x[5m]))", 20), - ("rate(x[5m]) / rate(x[5m])", 10), - ("sum without (instance) (rate(x[5m])) * rate(x[5m])", 10), - ("rate(x[5m]) + rate(m[5m])", 20), - ] { - assert_eq!(values_of(&root(query), &data), Some(values), "{query}"); - } - } - - /// `vector(s)` returns one row, but the operators below it still read - /// their sources' rows: the input row count of - /// `vector(scalar(max(rate(x[5m]))))` counts the 10 series the `max` - /// folds (plus the one constant row), never shrinking going up the tree, - /// while its `G` is 1. - #[test] - fn a_vector_of_a_scalar_keeps_the_rows_read_below_it() { - let root = - |query: &str| QueryRoot::Operator(lower_promql(query, AccuracyTarget::max_error(0.01))); - let data = with_series(Some(10)); - let vector = root("vector(scalar(max(rate(x[5m]))))"); - assert_eq!(series_read(&vector, &data), Some(11)); - assert_eq!(values_of(&vector, &data), Some(1)); - assert_eq!(values_of(&root("vector(1)"), &data), Some(1)); - let or = root("rate(x[5m]) or vector(scalar(max(rate(x[5m]))))"); - assert_eq!(series_read(&or, &data), Some(21)); - } - - /// Deployment evidence that every output was measured exact. - struct EveryOutputMeasuredExact; - - impl AccuracyEvidenceProvider for EveryOutputMeasuredExact { - fn output_guarantee(&self, _output: &QueryRoot) -> Option { - Some(AccuracyGuarantee::exact("measured")) - } - } - - /// Deployment evidence of a small error in a fixed metric for every - /// output that is not exact. - struct EveryOutputMeasuredIn(AccuracyMetric); - - impl AccuracyEvidenceProvider for EveryOutputMeasuredIn { - fn output_guarantee(&self, _output: &QueryRoot) -> Option { - Some(AccuracyGuarantee::new( - self.0, - BoundExpr::Constant { value: 1e-6 }, - ProbabilityExpr::Constant { value: 1e-6 }, - vec![], - )) - } - } - - /// A bound stated in another metric misses the target however small it - /// is: a relative bound does not bound a top-k frequency's L1-scaled - /// error, so only the exact candidate stays valid; the same evidence in - /// the target's metric admits the sketches. - #[test] - fn a_bound_in_another_metric_misses_the_target() { - let counter = with_metric_types(data(), &[("m", PrometheusMetricType::Counter)]); - let target = AccuracyTarget::max_error(0.5) - .failure_probability(0.5) - .accuracy_metric(AccuracyMetric::ABSOLUTE_STREAM_L1); - let select = |metric| { - select_among_plans( - &candidates(), - &[entry_with(target.clone(), repeating(10_000))], - &counter, - SelectionModels::builtin().with_evidence(&EveryOutputMeasuredIn(metric)), - ) - .unwrap() - }; - let relative = select(AccuracyMetric::Relative); - assert_eq!(relative.selected, "P1"); - let mut invalid: Vec<_> = relative - .rejected - .iter() - .filter(|r| r.cause.failed_a_check()) - .collect(); - invalid.sort_by(|a, b| a.id.cmp(&b.id)); - assert_eq!( - invalid.iter().map(|r| r.id.as_str()).collect::>(), - ["P2", "P3"] - ); - for rejected in invalid { - assert!( - rejected.reason.contains("no evidence that it meets"), - "{}", - rejected.reason - ); - } - let l1 = select(AccuracyMetric::ABSOLUTE_STREAM_L1); - assert!( - l1.rejected.iter().all(|r| !r.cause.failed_a_check()), - "{:?}", - l1.rejected - ); - } - - /// A difference of two quantile estimates has no output guarantee, though - /// each estimate is certified: its summary candidates stay listed as - /// invalid with a reason. The deployment's empirical accuracy for the - /// output admits them. - #[test] - fn combined_estimates_need_evidence_about_the_output() { - let target = AccuracyTarget::max_error(0.01).failure_probability(0.001); - let candidates = all_candidates( - "quantile_over_time(0.99, m[1m]) - quantile_over_time(0.5, m[1m])", - &target, - ); - let summaries: Vec<_> = candidates - .iter() - .filter(|c| has_summary_estimate(c)) - .map(|c| c.id.as_str()) - .collect(); - assert!(!summaries.is_empty()); - - let builtin = select_among_plans( - &candidates, - &[entry_with( - target.clone().accuracy_metric(AccuracyMetric::ABSOLUTE), - repeating(10_000), - )], - &data(), - SelectionModels::builtin(), - ) - .unwrap(); - for id in &summaries { - let rejection = builtin.rejected.iter().find(|r| r.id == *id).unwrap(); - assert_eq!(rejection.cause, RejectionCause::NoAccuracyGuarantee); - assert!( - rejection.reason.contains("no guarantee"), - "{}", - rejection.reason - ); - } - assert!(!summaries.contains(&builtin.selected.as_str())); - - let measured = select_among_plans( - &candidates, - &[entry_with( - target.clone().accuracy_metric(AccuracyMetric::ABSOLUTE), - repeating(10_000), - )], - &data(), - SelectionModels::builtin().with_evidence(&EveryOutputMeasuredExact), - ) - .unwrap(); - for id in &summaries { - assert!(measured.costs.contains_key(*id), "{id} is valid"); - } - } - - /// Deployment evidence of a large absolute error (1.0) for every output. - struct EveryOutputMeasuredLoose; - - impl AccuracyEvidenceProvider for EveryOutputMeasuredLoose { - fn output_guarantee(&self, _output: &QueryRoot) -> Option { - Some(AccuracyGuarantee::new( - AccuracyMetric::ABSOLUTE, - BoundExpr::Constant { value: 1.0 }, - ProbabilityExpr::Constant { value: 1e-6 }, - vec![], - )) - } - } - - /// A model that accepts every output directly, whatever its guarantee. - struct AcceptsEveryOutput; - - impl AccuracyModel for AcceptsEveryOutput { - fn satisfies(&self, guarantee: &AccuracyGuarantee, target: &AccuracyTarget) -> bool { - BuiltinAccuracyModel.satisfies(guarantee, target) - } - - fn accepts_output_regardless_of_guarantee( - &self, - _output: &QueryRoot, - _target: &AccuracyTarget, - ) -> bool { - true - } - } - - /// The hook overrides guarantees, not the executor's requirement on - /// Count-Min update weights: over samples of a metric declared a gauge, - /// the CmsWithHeap candidate (P2) is rejected as - /// `CountMinWeightsNotProvenNonNegative` under a model that accepts - /// every output, as under the built-in one. - #[test] - fn unproven_count_min_weights_reject_even_an_accepting_model() { - let target = AccuracyTarget::max_error(0.01) - .failure_probability(0.001) - .accuracy_metric(AccuracyMetric::ABSOLUTE_STREAM_L1); - let gauge = with_metric_types(data(), &[("m", PrometheusMetricType::Gauge)]); - for accuracy in [ - &AcceptsEveryOutput as &dyn AccuracyModel, - &BuiltinAccuracyModel, - ] { - let selection = select_among_plans( - &candidates(), - &[entry_with(target.clone(), repeating(10_000))], - &gauge, - SelectionModels::builtin().with_accuracy(accuracy), - ) - .unwrap(); - let p2 = selection.rejected.iter().find(|r| r.id == "P2").unwrap(); - assert_eq!( - p2.cause, - RejectionCause::CountMinWeightsNotProvenNonNegative, - "{}", - p2.reason - ); - assert!( - p2.reason.contains("not declared a counter"), - "{}", - p2.reason - ); - assert!(p2.cause.failed_a_check()); - } - } - - /// `IgnoreAccuracyGuarantees` (issue #774) does not override the - /// executor's requirement on Count-Min update weights either: over - /// samples of a gauge, P2's CmsWithHeap is still rejected as - /// `CountMinWeightsNotProvenNonNegative`. - #[test] - fn unproven_count_min_weights_reject_even_when_ignoring_guarantees() { - let target = AccuracyTarget::max_error(0.01) - .failure_probability(0.001) - .accuracy_metric(AccuracyMetric::ABSOLUTE_STREAM_L1); - let gauge = with_metric_types(data(), &[("m", PrometheusMetricType::Gauge)]); - let settings = CostInputs { - ..CostInputs::default() - }; - let selection = select_among_plans( - &candidates(), - &[entry_with(target, repeating(10_000))], - &gauge, - SelectionModels::builtin() - .with_cost_inputs(&settings) - .with_accuracy(&asap_accuracy::IgnoreAccuracyGuarantees), - ) - .unwrap(); - let p2 = selection.rejected.iter().find(|r| r.id == "P2").unwrap(); - assert_eq!( - p2.cause, - RejectionCause::CountMinWeightsNotProvenNonNegative, - "{}", - p2.reason - ); - } - - /// With `IgnoreAccuracyGuarantees` (issue #774), each summary - /// candidate the built-in model rejects (no guarantee, or one that - /// misses the target) is valid, and its accepted output records the - /// cause the guarantee check would have rejected it with; an output - /// whose guarantee meets the target records none. - #[test] - fn ignoring_guarantees_accepts_and_marks_outputs_the_check_rejects() { - let target = AccuracyTarget::max_error(0.01) - .failure_probability(0.001) - .accuracy_metric(AccuracyMetric::ABSOLUTE); - let entries = [entry_with(target, repeating(10_000))]; - let settings = CostInputs { - ..CostInputs::default() - }; - let builtin = - select_among_plans(&candidates(), &entries, &data(), SelectionModels::builtin()) - .unwrap(); - let ignoring = select_among_plans( - &candidates(), - &entries, - &data(), - SelectionModels::builtin() - .with_cost_inputs(&settings) - .with_accuracy(&asap_accuracy::IgnoreAccuracyGuarantees), - ) - .unwrap(); - let mut marked = 0; - for rejection in builtin.rejected.iter().filter(|r| { - matches!( - r.cause, - RejectionCause::NoAccuracyGuarantee | RejectionCause::AccuracyTargetMissed - ) - }) { - let [accepted] = ignoring.accepted[&rejection.id].as_slice() else { - panic!("{} has one query", rejection.id) - }; - assert!(accepted.accepted_regardless_of_guarantee); - assert_eq!(accepted.guarantee_check_rejection, Some(rejection.cause)); - assert!(accepted.to_string().contains("ignoring"), "{accepted}"); - marked += 1; - } - assert!(marked > 0, "{:?}", builtin.rejected); - for (id, queries) in &builtin.accepted { - assert_eq!( - ignoring.accepted[id][0].guarantee_check_rejection, None, - "{id}" - ); - assert_eq!(queries[0].guarantee_check_rejection, None, "{id}"); - } - } - - /// `node` with every CmsWithHeap summary build replaced by a Hydra - /// Count-Min over the same update. - fn with_hydra_count_min(node: &Rc) -> Rc { - use asap_types::ir::schema::{HydraKind, HydraParams, SketchParams}; - let mut rebuilt = node.with_new_children(with_hydra_count_min).unwrap(); - if let Operator::ASAP(ASAPOp::SummaryAgg { family, .. }) = &mut rebuilt.operator { - if let FieldDataType::Sketch(kind, _) = family { - if let SketchParams::CmsWithHeap { width, depth, .. } = kind.params() { - let (width, depth) = (*width, *depth); - *family = FieldDataType::Sketch( - SketchKind::new(SketchAlgorithm::Cms, SketchParams::Cms { width, depth }), - GroupingStrategy::SharedMultiSubpopulation { - kind: HydraKind::HydraCms, - params: HydraParams::HydraCms { - width, - depth, - shared_rows: depth, - shared_columns: width, - }, - }, - ); - } - } - } - Rc::new(rebuilt) - } - - /// A Hydra Count-Min over unproven weights is rejected by its own name: - /// the reason says `HydraCms`, not `Cms`. - #[test] - fn unproven_hydra_count_min_weights_name_hydra_cms() { - let target = AccuracyTarget::max_error(0.01) - .failure_probability(0.001) - .accuracy_metric(AccuracyMetric::ABSOLUTE_STREAM_L1); - let gauge = with_metric_types(data(), &[("m", PrometheusMetricType::Gauge)]); - let mut p2 = candidates().into_iter().find(|c| c.id == "P2").unwrap(); - p2.roots = p2 - .roots - .iter() - .map(|root| match root { - QueryRoot::Operator(node) => QueryRoot::Operator(with_hydra_count_min(node)), - other => other.clone(), - }) - .collect(); - let CheckFailure { cause, reason, .. } = - accuracy_check(&p2, &[target], &gauge, &SelectionModels::builtin()).unwrap_err(); - assert_eq!(cause, RejectionCause::CountMinWeightsNotProvenNonNegative); - assert!( - reason.contains("HydraCms needs non-negative update weights"), - "{reason}" - ); - assert!(reason.contains("not declared a counter"), "{reason}"); - } - - /// The model's hook has priority over every guarantee. The summary - /// candidates of a difference of quantile estimates have no theoretical - /// guarantee: with evidence that misses the target the built-in model - /// rejects them as `AccuracyTargetMissed`, and with no evidence as - /// `NoAccuracyGuarantee`; a model that accepts every output accepts them - /// in both cases, regardless of guarantee. - #[test] - fn the_accuracy_model_accepts_an_output_whatever_its_guarantee() { - let target = AccuracyTarget::max_error(0.01).failure_probability(0.001); - let candidates = all_candidates( - "quantile_over_time(0.99, m[1m]) - quantile_over_time(0.5, m[1m])", - &target, - ); - let summaries: Vec<_> = candidates - .iter() - .filter(|c| has_summary_estimate(c)) - .map(|c| c.id.as_str()) - .collect(); - assert!(!summaries.is_empty()); - let select = |accuracy: &dyn AccuracyModel, evidence: &dyn AccuracyEvidenceProvider| { - select_among_plans( - &candidates, - &[entry_with( - target.clone().accuracy_metric(AccuracyMetric::ABSOLUTE), - repeating(10_000), - )], - &data(), - SelectionModels::builtin() - .with_accuracy(accuracy) - .with_evidence(evidence), - ) - .unwrap() - }; - for (evidence, builtin_cause) in [ - ( - &EveryOutputMeasuredLoose as &dyn AccuracyEvidenceProvider, - RejectionCause::AccuracyTargetMissed, - ), - (&NoAccuracyEvidence, RejectionCause::NoAccuracyGuarantee), - ] { - let builtin = select(&BuiltinAccuracyModel, evidence); - let accepting = select(&AcceptsEveryOutput, evidence); - for id in &summaries { - let rejection = builtin.rejected.iter().find(|r| r.id == *id).unwrap(); - assert_eq!(rejection.cause, builtin_cause, "{}", rejection.reason); - assert!(accepting.costs.contains_key(*id), "{id} is valid"); - assert!( - accepting.accepted[*id][0].accepted_regardless_of_guarantee, - "{id}" - ); - } - } - } - - /// `candidates` with every state node (a summary build or an exact - /// aggregate) kept at ingestion time, so selection charges its kept - /// bytes every second (a `MatAtQuery` state is held only for one - /// evaluation). - fn keeping_state(mut candidates: Vec) -> Vec { - for candidate in &mut candidates { - for node in &mut candidate.dag.nodes { - if matches!( - node.payload, - Payload::ASAP(ASAPOp::SummaryAgg { .. }) - | Payload::NonASAP(NonASAPOp::Aggregate { .. }) - ) { - node.materialization = - Materialization::MatAtIngestion(asap_types::physical::Medium::Memory); - } - } - } - candidates - } - - /// Kept exact quantile state keeps every value of its group: over one - /// group of 10 000 values it costs more than a quantile sketch sized for - /// ε = 0.01, δ = 0.01, which is selected. Over 100 groups of 100 values - /// (100 jobs declared), fewer than the sketch's `k`, the sketch keeps - /// every value too, in an item array grown past them, so its state is - /// no smaller than the exact state, which is selected. - #[test] - fn exact_quantile_state_costs_more_than_a_sketch_over_large_groups() { - let target = AccuracyTarget::max_error(0.01).failure_probability(0.01); - let mut data = data(); - data.sources.get_mut("x").unwrap().distinct_values_per_label = - [("job".to_string(), 100)].into(); - for (query, large_groups) in [ - ("quantile(0.99, rate(x[5m]))", true), - ("quantile by (job) (0.99, rate(x[5m]))", false), - ] { - let checked = target.clone().accuracy_metric(AccuracyMetric::Rank); - let candidates = keeping_state(all_candidates_sized(query, &target, &|roots| { - sized_as_pass1(&roots, std::slice::from_ref(&checked), &data) - })); - let selection = select_among_plans( - &candidates, - &[entry_with( - target.clone().accuracy_metric(AccuracyMetric::Rank), - repeating(10_000), - )], - &data, - SelectionModels::builtin(), - ) - .unwrap(); - let kept_bytes = |sketch: bool| { - let candidate = candidates - .iter() - .find(|c| { - has_summary_estimate(c) == sketch && selection.costs.contains_key(&c.id) - }) - .unwrap(); - selection.costs[&candidate.id].usage.average_memory_bytes - }; - let selected = candidates - .iter() - .find(|c| c.id == selection.selected) - .unwrap(); - match large_groups { - true => { - assert!(has_summary_estimate(selected), "{query}: {selection:?}"); - assert!(kept_bytes(true) < kept_bytes(false), "{query}"); - } - false => { - assert!(!has_summary_estimate(selected), "{query}: {selection:?}"); - assert!(kept_bytes(true) >= kept_bytes(false), "{query}"); - } - } - } - } - - /// Exact L2 state keeps one counter per distinct value of its group, so - /// it is priced like exact quantile state, which keeps every value. - #[test] - fn exact_l2_state_is_priced_like_exact_quantile_state() { - let aggregate_cost = |query: &str| { - let candidates = all_candidates(query, &AccuracyTarget::exact()); - let selection = select_among_plans( - &candidates, - &[entry(repeating(10_000))], - &data(), - SelectionModels::builtin(), - ) - .unwrap(); - let exact = candidates - .iter() - .find(|c| !has_summary_estimate(c)) - .unwrap(); - selection.costs[&exact.id] - .per_node - .values() - .find(|n| n.detail.starts_with("hash aggregate")) - .unwrap() - .cost_per_second - }; - assert_eq!( - aggregate_cost("l2_over_time(x[5m])"), - aggregate_cost("quantile_over_time(0.99, x[5m])") - ); - } - - /// Kept exact distinct-count state keeps every item of its group: over - /// 10 000 groups of 300 rows it is smaller than a distinct-count sketch - /// sized for ε = 0.01 and is selected; over one group of 3 000 000 rows a - /// sketch is. - #[test] - fn exact_distinct_count_beats_a_sketch_over_small_groups() { - let target = AccuracyTarget::max_error(0.01).failure_probability(0.01); - let one_series = with_series(Some(1)); - for (data, sketch_wins) in [(data(), false), (one_series, true)] { - let candidates = keeping_state(all_candidates("distinct_over_time(x[5m])", &target)); - let selection = select_among_plans( - &candidates, - &[entry_with( - target.clone().accuracy_metric(AccuracyMetric::Relative), - repeating(10_000), - )], - &data, - SelectionModels::builtin(), - ) - .unwrap(); - let selected = candidates - .iter() - .find(|c| c.id == selection.selected) - .unwrap(); - assert_eq!(has_summary_estimate(selected), sketch_wins, "{selection:?}"); - } - } - - /// The candidates of `query` that build a DDSketch. - fn ddsketch_candidates(candidates: &[PhysicalWorkloadPlan]) -> Vec { - candidates - .iter() - .filter(|c| { - c.dag.nodes.iter().any(|node| { - matches!(&node.payload, - Payload::ASAP(ASAPOp::SummaryAgg { family: FieldDataType::Sketch(kind, _), .. }) - if kind.algorithm() == &SketchAlgorithm::DDSketch) - }) - }) - .map(|c| c.id.clone()) - .collect() - } - - /// A per-series DDSketch quantile is within a relative error, which - /// carries through `avg` and `sum` only over non-negative values, and - /// through `max` whatever their sign. With `x` declared a gauge whose - /// samples are non-negative, the DDSketch candidate of - /// `avg by (region) (quantile_over_time(0.99, x[100s]))` (and of its `sum`) - /// is valid and accepted with a relative-error guarantee; without the - /// declaration it is rejected. Even `max` needs the primitive interpolated - /// quantile contract, which can fail across zero. - #[test] - fn a_non_negative_gauge_carries_a_ddsketch_relative_bound_through_avg() { - let target = AccuracyTarget::max_error(0.01) - .failure_probability(0.01) - .accuracy_metric(AccuracyMetric::Relative); - // An `offset` moves the window, not the samples' sign. - for (outer, window, needs_sign) in [ - ("avg", "x[100s]", true), - ("sum", "x[100s]", true), - ("max", "x[100s]", true), - ("avg", "x[100s] offset 1h", true), - ] { - let query = format!("{outer} by (region) (quantile_over_time(0.99, {window}))"); - let candidates = all_candidates(&query, &target); - let ddsketch = ddsketch_candidates(&candidates); - assert!(!ddsketch.is_empty(), "{query}"); - for values_non_negative in [true, false] { - let mut data = with_metric_types(data(), &[("x", PrometheusMetricType::Gauge)]); - data.sources.get_mut("x").unwrap().values_non_negative = values_non_negative; - let selection = select_among_plans( - &candidates, - &[entry_with(target.clone(), repeating(10_000))], - &data, - SelectionModels::builtin(), - ) - .unwrap(); - for id in &ddsketch { - let accepted = selection.accepted.get(id); - if values_non_negative || !needs_sign { - let [accepted] = accepted.map(Vec::as_slice).unwrap_or_default() else { - panic!("{query}, {id} not valid: {:?}", selection.rejected); - }; - assert_eq!(accepted.accepted_metric, Some(AccuracyMetric::Relative)); - assert!(!accepted.accepted_regardless_of_guarantee); - } else { - assert!(accepted.is_none(), "{query}, {id}: {accepted:?}"); - let rejection = selection.rejected.iter().find(|r| &r.id == id).unwrap(); - assert!(rejection.cause.failed_a_check(), "{rejection:?}"); - } - } - } - } - } - - /// PromQL answers a quantile with φ outside [0, 1] as -Inf or +Inf, - /// which no sketch estimates and the executor's sketches refuse: such a - /// quantile gets no sketch candidate, so the exact plan is selected, and - /// a sketch never inherits its samples' sign for it, even over a gauge - /// declared non-negative. - #[test] - fn a_quantile_outside_the_unit_interval_is_answered_exactly() { - let target = AccuracyTarget::max_error(0.01) - .failure_probability(0.01) - .accuracy_metric(AccuracyMetric::Relative); - for query in [ - "avg by (region) (quantile_over_time(-1, x[100s]))", - "avg by (region) (quantile_over_time(2, x[100s]))", - "quantile_over_time(2, x[100s])", - "max by (region) (quantile_over_time(-1, x[100s]))", - ] { - let candidates = all_candidates(query, &target); - assert!( - candidates.iter().all(|c| !has_summary_estimate(c)), - "{query}: a sketch candidate is offered" - ); - let mut data = with_metric_types(data(), &[("x", PrometheusMetricType::Gauge)]); - data.sources.get_mut("x").unwrap().values_non_negative = true; - let selection = select_among_plans( - &candidates, - &[entry_with(target.clone(), repeating(10_000))], - &data, - SelectionModels::builtin(), - ) - .unwrap(); - let selected = candidates - .iter() - .find(|c| c.id == selection.selected) - .unwrap(); - assert!( - !has_summary_estimate(selected), - "{query}: {}", - selection.selected - ); - } - } - - /// `topk(k, irate(m[w]))`, now a heavy-hitter ranking as `rate`'s is, - /// still plans: Stages 1-3 compose its candidates and select one. - #[test] - fn a_direct_irate_ranking_plans() { - let target = AccuracyTarget::max_error(0.01).failure_probability(0.001); - for function in ["rate", "irate"] { - let query = format!("topk(10, {function}(data_total[5m]))"); - let candidates = all_candidates(&query, &target); - let data = with_metric_types(data(), &[("data_total", PrometheusMetricType::Counter)]); - let selection = select_among_plans( - &candidates, - &[entry_with(target.clone(), repeating(10_000))], - &data, - SelectionModels::builtin(), - ); - assert!(selection.is_ok(), "{query}: {selection:?}"); - } - } - - /// A candidate that ties the selected one is rejected for losing on - /// enumeration order, not as costlier. - #[test] - fn a_tie_is_reported_as_lost_on_enumeration_order() { - let mut candidates = candidates(); - candidates.truncate(1); - let mut twin = candidates[0].clone(); - twin.id = "P1-twin".into(); - candidates.push(twin); - let selection = select_among_plans( - &candidates, - &[entry(repeating(10_000))], - &data(), - SelectionModels::builtin(), - ) - .unwrap(); - assert_eq!(selection.selected, "P1"); - let reason = &selection.rejected[0].reason; - assert!( - reason.contains("tied with the selected candidate P1") - && reason.contains("enumeration order"), - "{reason}" - ); - } - - /// At δ = 0.001 a KLL candidate meets the target and is valid. - #[test] - fn kll_candidate_meets_a_failure_probability_below_one_percent() { - let target = AccuracyTarget::max_error(0.01).failure_probability(0.001); - let candidates = all_candidates("quantile(0.99, rate(x[5m]))", &target); - let selection = select_among_plans( - &candidates, - &[entry_with( - target.clone().accuracy_metric(AccuracyMetric::Rank), - repeating(10_000), - )], - &data(), - SelectionModels::builtin(), - ) - .unwrap(); - let kll = candidates - .iter() - .find(|c| { - c.dag.nodes.iter().any(|n| { - matches!(&n.payload, Payload::ASAP(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), .. - }) if *kind.algorithm() == SketchAlgorithm::Kll) - }) - }) - .unwrap(); - assert!(selection.costs.contains_key(&kll.id), "{selection:?}"); - } - - /// `scalar(v)` inherits `v`'s guarantee; arithmetic over it composes - /// (`c · x` scales an absolute bound, `x − y` adds them); a rank error - /// does not compose (a rank interval counts as none); one over exact - /// operands only is exact. - #[test] - fn scalar_root_guarantees_compose_through_arithmetic() { - let node = lower_promql("sum(m)", AccuracyTarget::exact()); - let approximate = AccuracyGuarantee::new( - AccuracyMetric::ABSOLUTE, - BoundExpr::Constant { value: 0.01 }, - ProbabilityExpr::Constant { value: 0.001 }, - vec![], - ); - let with = |guarantee: Option| { - ScalarExpr::PromqlScalarFromVector(Rc::new((*node).clone().with_guarantee(guarantee))) - }; - let doubled = |operand: ScalarExpr| { - QueryRoot::Scalar(ScalarExpr::Arithmetic { - op: ArithmeticOpKind::Mul, - left: Box::new(operand), - right: Box::new(ScalarExpr::literal_f64(2.0)), - semantics: ExprSemantics::Promql, - }) - }; - assert_eq!( - root_guarantee(&QueryRoot::Scalar(with(Some(approximate.clone())))), - Some(approximate.clone()) - ); - let scaled = root_guarantee(&doubled(with(Some(approximate.clone())))).unwrap(); - assert_eq!(scaled.spec.accuracy_metric, AccuracyMetric::ABSOLUTE); - assert!((scaled.spec.error.evaluate().unwrap() - 0.02).abs() < 1e-12); - assert_eq!(scaled.spec.failure_probability.evaluate(), Some(0.001)); - let difference = QueryRoot::Scalar(ScalarExpr::Arithmetic { - op: ArithmeticOpKind::Sub, - left: Box::new(with(Some(approximate.clone()))), - right: Box::new(with(Some(approximate.clone()))), - semantics: ExprSemantics::Promql, - }); - let sum = root_guarantee(&difference).unwrap(); - assert!((sum.spec.error.evaluate().unwrap() - 0.02).abs() < 1e-12); - let mut rank = approximate; - rank.spec.accuracy_metric = AccuracyMetric::Rank; - assert_eq!(root_guarantee(&doubled(with(Some(rank)))), None); - assert!( - root_guarantee(&doubled(with(Some(AccuracyGuarantee::exact("exact"))))) - .is_some_and(|g| g.is_exact()) - ); - } - - /// Every candidate of the Stage 1 to Stage 3 run of one query. - fn run( - query: &str, - target: &AccuracyTarget, - data: &DataWorkload, - models: SelectionModels<'_>, - ) -> (Vec, WholePlanSelection) { - run_selecting(query, target, target, data, models) - } - - /// As [`run`], with summaries sized for `planned` and outputs checked - /// against `selected`. - fn run_selecting( - query: &str, - planned: &AccuracyTarget, - selected: &AccuracyTarget, - data: &DataWorkload, - models: SelectionModels<'_>, - ) -> (Vec, WholePlanSelection) { - let root = lower_promql(query, planned.clone()); - let entries = [entry_with(selected.clone(), QueryRecurrence::Unknown)]; - let roots = sized_as_pass1( - &[QueryRoot::Operator(root)], - std::slice::from_ref(planned), - data, - ); - let candidates = exhaustive_candidates(&roots, &entries, data); - let selection = select_among_plans(&candidates, &entries, data, models).unwrap(); - (candidates, selection) - } - - /// Every whole plan of `roots`: each combination of Pass 1's choices - /// composed, then every materialization assignment Stage 2 offers for - /// it, with unique ids. - fn exhaustive_candidates( - roots: &[QueryRoot], - entries: &[QueryWorkloadEntry], - data: &DataWorkload, - ) -> Vec { - use asap_logical_asap_optimization::test_support::{ - compose_logical_candidate, generate_per_computation_candidates, - }; - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(roots), - ) - .unwrap(); - let mut choices = vec![vec![]]; - for target in &inventory.targets { - choices = choices - .into_iter() - .flat_map(|choice: Vec| { - (0..target.choice_count()).map(move |index| { - let mut choice = choice.clone(); - choice.push(index); - choice - }) - }) - .collect(); - } - let mut candidates = Vec::new(); - for (index, choice) in choices.iter().enumerate() { - let Ok(composed) = compose_logical_candidate(&inventory, choice) else { - continue; - }; - let id = format!("L{}", index + 1); - let Ok(physical) = optimize_physical_asap(&id, &composed, entries, data, false) else { - continue; - }; - for (k, mut plan) in physical.into_iter().enumerate() { - plan.id = format!("P{}-{k}", index + 1); - plan.from_logical = id.clone(); - candidates.push(plan); - } - } - candidates - } - - fn sketches(candidate: &PhysicalWorkloadPlan) -> Vec { - candidate - .dag - .nodes - .iter() - .filter_map(|n| match &n.payload { - Payload::ASAP(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), - .. - }) => Some(*kind.algorithm()), - _ => None, - }) - .collect() - } - - fn reads_only(candidate: &PhysicalWorkloadPlan, algorithm: SketchAlgorithm) -> bool { - let sketches = sketches(candidate); - !sketches.is_empty() && sketches.iter().all(|a| *a == algorithm) - } - - fn output_guarantee(candidate: &PhysicalWorkloadPlan) -> Option { - root_guarantee(&candidate.roots[0]) - } - - /// `max by (job)` over per-series DDSketch quantiles keeps their relative - /// bound, and `2 · x` keeps it too (scaling preserves a relative error); - /// so those candidates meet the target. Over KLL (a rank error) the - /// result is a rank interval: no guarantee in the MVP. - #[test] - fn extremum_and_scaling_over_per_series_estimates_compose() { - let target = AccuracyTarget::max_error(0.01).accuracy_metric(AccuracyMetric::Relative); - for query in [ - "max by (job) (quantile_over_time(0.99, m[1m]))", - "2 * max by (job) (quantile_over_time(0.99, m[1m]))", - ] { - let (candidates, selection) = run( - query, - &target, - &non_negative_data(), - SelectionModels::builtin(), - ); - let ddsketch: Vec<_> = candidates - .iter() - .filter(|c| reads_only(c, SketchAlgorithm::DDSketch)) - .collect(); - assert!(!ddsketch.is_empty(), "{query}"); - for candidate in ddsketch { - let g = bound_of( - &candidate.roots[0], - &non_negative_data(), - &SelectionModels::builtin(), - ) - .expect(query); - assert_eq!(g.spec.accuracy_metric, AccuracyMetric::Relative, "{query}"); - assert!(g.spec.error.evaluate().unwrap() <= 0.01, "{query}"); - assert!(selection.costs.contains_key(&candidate.id), "{query}"); - } - for candidate in candidates - .iter() - .filter(|c| reads_only(c, SketchAlgorithm::Kll)) - { - assert_eq!( - bound_of( - &candidate.roots[0], - &non_negative_data(), - &SelectionModels::builtin() - ), - None, - "{query}" - ); - } - } - } - - /// Negation, `abs` and `clamp` keep a DDSketch quantile's relative bound, - /// so those candidates are selectable; a shift by a constant, a - /// comparison (filter or `bool`) over the estimate compose to no - /// guarantee in the MVP, so those are not. - #[test] - fn lipschitz_functions_keep_and_shifts_and_comparisons_drop_a_relative_bound() { - let target = AccuracyTarget::max_error(0.01).accuracy_metric(AccuracyMetric::Relative); - let q = "quantile_over_time(0.99, m[1m])"; - for (query, kept) in [ - (format!("-{q}"), true), - (format!("abs({q})"), true), - (format!("clamp({q}, 1, 100)"), true), - (format!("clamp_max({q}, 100)"), true), - (format!("{q} + 1"), false), - (format!("{q} > 5"), false), - (format!("{q} > bool 5"), false), - ] { - let (candidates, selection) = run( - &query, - &target, - &non_negative_data(), - SelectionModels::builtin(), - ); - let ddsketch: Vec<_> = candidates - .iter() - .filter(|c| reads_only(c, SketchAlgorithm::DDSketch)) - .collect(); - assert!(!ddsketch.is_empty(), "{query}"); - for candidate in ddsketch { - let g = bound_of( - &candidate.roots[0], - &non_negative_data(), - &SelectionModels::builtin(), - ); - if kept { - let g = g.expect(&query); - assert_eq!(g.spec.accuracy_metric, AccuracyMetric::Relative, "{query}"); - assert!(g.spec.error.evaluate().unwrap() <= 0.01, "{query}"); - assert!(selection.costs.contains_key(&candidate.id), "{query}"); - } else { - assert_eq!(g, None, "{query}"); - assert!(!selection.costs.contains_key(&candidate.id), "{query}"); - } - } - } - } - - /// A time-window fold over a subquery of estimates folds one estimate - /// per subquery step and series, so its failure probability is scaled - /// by the step count `⌈range / resolution⌉` (here 10), not by the input - /// series count. Without an explicit resolution the step count is not in - /// the query, and the fold has no guarantee. - #[test] - fn a_fold_over_subquery_estimates_counts_its_steps() { - let target = AccuracyTarget::max_error(0.01).accuracy_metric(AccuracyMetric::Relative); - let q = "quantile_over_time(0.99, m[1m])"; - let (candidates, selection) = run( - &format!("max_over_time({q}[10m:1m])"), - &target, - &non_negative_data(), - SelectionModels::builtin(), - ); - let ddsketch: Vec<_> = candidates - .iter() - .filter(|c| reads_only(c, SketchAlgorithm::DDSketch)) - .collect(); - assert!(!ddsketch.is_empty()); - for candidate in ddsketch { - let g = bound_of( - &candidate.roots[0], - &non_negative_data(), - &SelectionModels::builtin(), - ) - .expect("a guarantee"); - assert_eq!(g.spec.accuracy_metric, AccuracyMetric::Relative); - assert!(g.spec.error.evaluate().unwrap() <= 0.01); - let ProbabilityExpr::Scaled { count, .. } = &g.spec.failure_probability else { - panic!("{g:?}"); - }; - assert_eq!(count, &BoundExpr::Constant { value: 10.0 }); - assert!(selection.costs.contains_key(&candidate.id)); - } - let (candidates, _) = run( - &format!("max_over_time({q}[10m:])"), - &target, - &non_negative_data(), - SelectionModels::builtin(), - ); - for candidate in candidates - .iter() - .filter(|c| reads_only(c, SketchAlgorithm::DDSketch)) - { - assert_eq!( - bound_of( - &candidate.roots[0], - &non_negative_data(), - &SelectionModels::builtin() - ), - None - ); - } - } - - /// Counting estimates by a label keeps the count exact: the group keys - /// are labels, not the estimates (#669). - #[test] - fn counting_estimates_by_a_label_stays_exact() { - let target = AccuracyTarget::max_error(0.01); - let (candidates, _) = run( - "count by (le) (quantile_over_time(0.99, m[1m]))", - &target, - &data(), - SelectionModels::builtin(), - ); - // The inner quantile is a sketch; the count itself is exact. - let sketched: Vec<_> = candidates - .iter() - .filter(|c| { - let sketches = sketches(c); - !sketches.is_empty() - && sketches - .iter() - .all(|a| matches!(a, SketchAlgorithm::Kll | SketchAlgorithm::DDSketch)) - }) - .collect(); - assert!(!sketched.is_empty()); - for candidate in sketched { - assert!( - output_guarantee(candidate).is_some_and(|g| g.is_exact()), - "{:?}", - sketches(candidate) - ); - } - } - - /// `avg by (job)` over per-series DDSketch quantiles has a relative bound - /// that needs the values' sign, a statistic no input supplies: it stays - /// symbolic and plan selection treats it as unmet. - #[test] - fn average_over_relative_estimates_needs_the_sign() { - use asap_logical_asap_optimization::test_support as pcc; - use asap_types::stages::HowToComputeAggregateMeasure; - let target = AccuracyTarget::max_error(0.01); - let root = lower_promql( - "avg by (job) (quantile_over_time(0.99, m[1m]))", - target.clone(), - ); - let inventory = pcc::generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(root), - ]), - ) - .unwrap(); - // Exercise the intended approximation directly; a joint optimizer may - // legitimately prefer an exact implementation or a different sharing - // topology, neither of which tests this compositional error contract. - let choices: Vec<_> = inventory - .targets - .iter() - .map(|target| { - target - .ways_to_compute - .iter() - .position(|candidate| { - *candidate - == HowToComputeAggregateMeasure::Summary(SketchAlgorithm::DDSketch) - }) - .unwrap_or(0) - }) - .collect(); - assert!(inventory - .targets - .iter() - .zip(&choices) - .any(|(target, &choice)| target.ways_to_compute[choice] - == HowToComputeAggregateMeasure::Summary(SketchAlgorithm::DDSketch))); - let outputs = pcc::compose_logical_candidate(&inventory, &choices).unwrap(); - let output = &outputs[0]; - let guarantee = root_guarantee(output).expect("a symbolic bound"); - assert!(guarantee.spec.error.has_unknown(), "{guarantee:?}"); - assert!(guarantee_check(0, output, &target, &data(), &SelectionModels::builtin()).is_err()); - } - - /// `max` folds one value per series, so its failure probability is - /// scaled by the symbolic input row count. Stage 3 binds it from the - /// series count of the source the query reads; without one, the target - /// is unmet. - #[test] - fn stage3_binds_the_input_row_count_from_the_data_workload() { - let target = AccuracyTarget::max_error(0.01) - .failure_probability(0.001) - .accuracy_metric(AccuracyMetric::Relative); - let query = "max by (job) (quantile_over_time(0.99, m[1m]))"; - let ddsketch = |candidates: &[PhysicalWorkloadPlan]| -> Vec { - candidates - .iter() - .filter(|c| reads_only(c, SketchAlgorithm::DDSketch)) - .map(|c| c.id.clone()) - .collect() - }; - let (candidates, with) = run( - query, - &target, - &non_negative_data(), - SelectionModels::builtin(), - ); - let ids = ddsketch(&candidates); - assert!(!ids.is_empty()); - assert!(ids.iter().all(|id| with.costs.contains_key(id))); - let mut unknown = with_series(None); - for facts in unknown.sources.values_mut() { - facts.values_non_negative = true; - } - // Evaluate the same physical candidates: IDs from independent timed - // searches do not identify the same implementation or connections. - let without = select_among_plans( - &candidates, - &[entry_with(target, QueryRecurrence::Unknown)], - &unknown, - SelectionModels::builtin(), - ) - .unwrap(); - assert!(ids.iter().all(|id| !without.costs.contains_key(id))); - } - - /// Deployment contract: every operand lies in `[1, 1000]`. - struct BoundedOperands; - - impl AccuracyEvidenceProvider for BoundedOperands { - fn operand_domain(&self, _operand: &OperatorNode) -> Option { - Some(asap_accuracy::OperandDomain { - lower: 1.0, - upper: 1_000.0, - }) - } - } - - /// A ratio of two DDSketch quantiles has a closed-form relative bound, - /// `(a + b)/(1 − b)`, with no operand domain: each operand sized for - /// 0.01 gives about 0.0202, which meets a 0.05 target. - #[test] - fn ddsketch_ratio_meets_its_target_without_operand_domains() { - let planned = AccuracyTarget::max_error(0.01).accuracy_metric(AccuracyMetric::Relative); - let target = AccuracyTarget::max_error(0.05).accuracy_metric(AccuracyMetric::Relative); - let query = "quantile_over_time(0.99, m[1m]) / quantile_over_time(0.5, m[1m])"; - let (candidates, selection) = run_selecting( - query, - &planned, - &target, - &non_negative_data(), - SelectionModels::builtin(), - ); - let ids: Vec<_> = candidates - .iter() - .filter(|c| sketches(c) == [SketchAlgorithm::DDSketch, SketchAlgorithm::DDSketch]) - .map(|c| c.id.clone()) - .collect(); - // The Pass 1 candidate and its Pass 2 candidates sharing the input. - assert!(!ids.is_empty()); - assert!( - ids.iter().all(|id| selection.costs.contains_key(id)), - "{selection:?}" - ); - } - - /// `query` carrying `guarantee`, as a `scalar(…)` operand. - fn scalar_of(query: &str, guarantee: AccuracyGuarantee) -> ScalarExpr { - let node = lower_promql(query, AccuracyTarget::exact()); - ScalarExpr::PromqlScalarFromVector(Rc::new((*node).clone().with_guarantee(Some(guarantee)))) - } - - fn guarantee(metric: AccuracyMetric, bound: f64, delta: f64) -> AccuracyGuarantee { - AccuracyGuarantee::new( - metric, - BoundExpr::Constant { value: bound }, - ProbabilityExpr::Constant { value: delta }, - vec![], - ) - } - - fn scalar_root(op: ArithmeticOpKind, left: ScalarExpr, right: ScalarExpr) -> QueryRoot { - QueryRoot::Scalar(ScalarExpr::Arithmetic { - op, - left: Box::new(left), - right: Box::new(right), - semantics: ExprSemantics::Promql, - }) - } - - fn norms(sources: &[(&str, f64)]) -> DataWorkload { - let mut data = data(); - for (source, l1) in sources { - data.sources - .entry(source.to_string()) - .or_default() - .stream_norms = Some(asap_types::workload::StreamNorms { - l1: Some(*l1), - l2: None, - }); - } - data - } - - /// A difference of two Count-Min estimates of the raw values of - /// different metrics is bound by each metric's own stream norm from the - /// data workload - /// (`0.01 · 100 + 0.02 · 1000`); a metric without a norm leaves the - /// bound unmet. - #[test] - fn stage3_binds_each_metric_stream_norm() { - // Count-Min summaries of each metric's raw value column. - let raw = |query: &str, g: AccuracyGuarantee| { - ScalarExpr::PromqlScalarFromVector( - asap_logical_asap_optimization::test_support::estimate_over( - asap_logical_asap_optimization::test_support::weighted( - asap_logical_asap_optimization::test_support::cms_summary(query), - asap_types::ir::schema::SummaryInputExpr::Column( - asap_types::ir::scalar::ColumnRef::SampleValue, - ), - ), - g, - ), - ) - }; - let root = scalar_root( - ArithmeticOpKind::Sub, - raw( - "count by (job) (a)", - guarantee(AccuracyMetric::ABSOLUTE_STREAM_L1, 0.01, 0.001), - ), - raw( - "count by (job) (b)", - guarantee(AccuracyMetric::ABSOLUTE_STREAM_L1, 0.02, 0.001), - ), - ); - let models = SelectionModels::builtin(); - let both = bound_of(&root, &norms(&[("a", 100.0), ("b", 1_000.0)]), &models).unwrap(); - assert!( - (both.spec.error.evaluate().unwrap() - 21.0).abs() < 1e-9, - "{both:?}" - ); - let one = bound_of(&root, &norms(&[("a", 100.0)]), &models).unwrap(); - assert!(one.spec.error.has_unknown()); - } - - /// A stream norm is bound only for a summary of the source's raw value - /// column: a count over the same source (constant weight) needs the - /// source's row count, which nothing binds, and a summary of a computed - /// value (`m * 1000`) needs a norm nothing binds; both stay unmet. - #[test] - fn stage3_binds_a_stream_norm_only_for_the_raw_value_column() { - use asap_types::ir::scalar::ColumnRef; - use asap_types::ir::schema::SummaryInputExpr; - let frequency = guarantee(AccuracyMetric::ABSOLUTE_STREAM_L1, 0.01, 0.001); - let doubled = |summary: Rc| { - scalar_root( - ArithmeticOpKind::Mul, - ScalarExpr::literal_f64(2.0), - ScalarExpr::PromqlScalarFromVector( - asap_logical_asap_optimization::test_support::estimate_over( - summary, - frequency.clone(), - ), - ), - ) - }; - let models = SelectionModels::builtin(); - let data = norms(&[("m", 100.0)]); - let column = || SummaryInputExpr::Column(ColumnRef::SampleValue); - let raw = doubled(asap_logical_asap_optimization::test_support::weighted( - asap_logical_asap_optimization::test_support::cms_summary("count by (job) (m)"), - column(), - )); - let g = bound_of(&raw, &data, &models).unwrap(); - assert!( - (g.spec.error.evaluate().unwrap() - 2.0).abs() < 1e-12, - "{g:?}" - ); - for unbound in [ - doubled(Rc::new( - asap_logical_asap_optimization::test_support::cms_summary("count by (job) (m)"), - )), - doubled(asap_logical_asap_optimization::test_support::weighted( - asap_logical_asap_optimization::test_support::cms_summary( - "count by (job) (m * 1000)", - ), - column(), - )), - ] { - let g = bound_of(&unbound, &data, &models).unwrap(); - assert!(g.spec.error.has_unknown(), "{g:?}"); - } - } - - /// A product and a quotient of two estimates are bound from the - /// deployment's operand domains (`[1, 1000]`): - /// `|x|·e_y + |y|·e_x + e_x·e_y` and `(|x|·e_y + |y|·e_x) / (|y|·(|y| − e_y))` - /// at the domain's extremes; without domains they stay unmet. A ratio of - /// relative errors needs no domain: `(a + b)/(1 − b)`. - #[test] - fn stage3_binds_products_and_quotients_from_operand_domains() { - let (x, y) = ( - scalar_of("sum(a)", guarantee(AccuracyMetric::ABSOLUTE, 0.01, 0.001)), - scalar_of("sum(b)", guarantee(AccuracyMetric::ABSOLUTE, 0.02, 0.001)), - ); - let product = scalar_root(ArithmeticOpKind::Mul, x.clone(), y.clone()); - let quotient = scalar_root(ArithmeticOpKind::Div, x, y); - let bounded = SelectionModels::builtin().with_evidence(&BoundedOperands); - for (root, expected) in [ - (&product, 1_000.0 * 0.02 + 1_000.0 * 0.01 + 0.01 * 0.02), - ("ient, (1_000.0 * 0.02 + 1_000.0 * 0.01) / (1.0 - 0.02)), - ] { - let g = bound_of(root, &data(), &bounded).unwrap(); - assert!( - (g.spec.error.evaluate().unwrap() - expected).abs() < 1e-9, - "{g:?}" - ); - assert!((g.spec.failure_probability.evaluate().unwrap() - 0.002).abs() < 1e-12); - let unbound = bound_of(root, &data(), &SelectionModels::builtin()).unwrap(); - assert!(unbound.spec.error.has_unknown(), "{unbound:?}"); - } - let ratio = scalar_root( - ArithmeticOpKind::Div, - scalar_of("sum(a)", guarantee(AccuracyMetric::Relative, 0.01, 0.0)), - scalar_of("sum(b)", guarantee(AccuracyMetric::Relative, 0.01, 0.0)), - ); - let g = bound_of(&ratio, &data(), &SelectionModels::builtin()).unwrap(); - assert!( - (g.spec.error.evaluate().unwrap() - 0.02 / 0.99).abs() < 1e-12, - "{g:?}" - ); - } - - /// A ratio divided by an exact vector, `(q99 / q50) / max_over_time(m)`, - /// keeps the inner ratio's relative bound, so the candidate is - /// selectable with no operand domain. - #[test] - fn a_scaled_ratio_keeps_its_relative_bound() { - let planned = AccuracyTarget::max_error(0.01).accuracy_metric(AccuracyMetric::Relative); - let target = AccuracyTarget::max_error(0.05).accuracy_metric(AccuracyMetric::Relative); - let query = "(quantile_over_time(0.99, m[1m]) / quantile_over_time(0.5, m[1m])) \ - / max_over_time(m[1m])"; - let (candidates, selection) = run_selecting( - query, - &planned, - &target, - &non_negative_data(), - SelectionModels::builtin(), - ); - let ids: Vec<_> = candidates - .iter() - .filter(|c| reads_only(c, SketchAlgorithm::DDSketch)) - .map(|c| c.id.clone()) - .collect(); - assert!(!ids.is_empty()); - assert!( - ids.iter().all(|id| selection.costs.contains_key(id)), - "{selection:?}" - ); - } - - /// The input row count is bound only from the series count of the - /// sources the query reads: another source's declared series count - /// leaves it unbound. - #[test] - fn stage3_binds_only_the_series_count_of_the_sources_read() { - let target = AccuracyTarget::max_error(0.01).failure_probability(0.001); - let query = "max by (job) (quantile_over_time(0.99, m[1m]))"; - let mut other_source = data(); - other_source.sources.remove("m"); - let (candidates, selection) = - run(query, &target, &other_source, SelectionModels::builtin()); - let ddsketch: Vec<_> = candidates - .iter() - .filter(|c| reads_only(c, SketchAlgorithm::DDSketch)) - .collect(); - assert!(!ddsketch.is_empty()); - assert!(ddsketch - .iter() - .all(|c| !selection.costs.contains_key(&c.id))); - } - - /// `quantile_over_time(0.99, m[1m])` with its KLL candidate chosen, its - /// state materialized as `materialization`. - fn kll_dag(materialization: Materialization) -> PhysicalASAPDAG { - use asap_types::ir::{ - apply_materialization_timings, MaterializationAssignment, NodeToTimedNodeCache, - }; - let root = lower_promql( - "quantile_over_time(0.99, m[1m])", - AccuracyTarget::max_error(0.01).failure_probability(0.01), - ); - let inventory = - asap_logical_asap_optimization::test_support::generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(root), - ]), - ) - .unwrap(); - let choice: Vec<_> = inventory - .targets - .iter() - .map(|t| { - t.ways_to_compute - .iter() - .position(|c| { - *c == asap_types::stages::HowToComputeAggregateMeasure::Summary( - SketchAlgorithm::Kll, - ) - }) - .unwrap_or(0) - }) - .collect(); - let roots = asap_logical_asap_optimization::test_support::compose_logical_candidate( - &inventory, &choice, - ) - .unwrap(); - let QueryRoot::Operator(root) = &roots[0] else { - panic!("operator root") - }; - let state = OperatorNode::reachable(root) - .into_iter() - .find(|n| matches!(n.asap(), Some(ASAPOp::SummaryAgg { .. }))) - .expect("a KLL state"); - let mut assignment = - MaterializationAssignment::with_default(Materialization::NotMaterialized); - assignment.set(&state, materialization); - let timed = - apply_materialization_timings(root, &assignment, &mut NodeToTimedNodeCache::new()) - .unwrap(); - asap_types::ir::physical_export::compile_physical_asap_workload_with_node_ids(&[ - QueryRoot::Operator(timed), - ]) - .unwrap() - .dag - } - - /// The physical DAG of `query`'s first per-computation candidates, - /// with nothing materialized. - fn first_candidate_dag(query: &str) -> PhysicalASAPDAG { - let root = lower_promql(query, AccuracyTarget::exact()); - let inventory = - asap_logical_asap_optimization::test_support::generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(root), - ]), - ) - .unwrap(); - let choice = vec![0; inventory.targets.len()]; - let roots: Vec<_> = - asap_logical_asap_optimization::test_support::compose_logical_candidate( - &inventory, &choice, - ) - .unwrap(); - optimize_physical_asap("L1", &roots, &[], &data(), false) - .unwrap() - .remove(0) - .dag - } - - /// A time shift is free and does not change what the scan below it - /// reads: the scan reads the window above the shift, as without it. A - /// time range is priced by the rows it keeps. - #[test] - fn time_shift_is_free_and_the_scan_reads_the_shifted_window() { - let every_minute = [repeating(60_000)]; - let plain = first_candidate_dag("sum_over_time(m[5m])"); - let shifted = first_candidate_dag("sum_over_time(m[5m] offset 1h)"); - let (plain_cost, shifted_cost) = ( - priced(&plain, &every_minute), - priced(&shifted, &every_minute), - ); - let cost_of = |dag: &PhysicalASAPDAG, cost: &PlanCost, pick: fn(&Payload) -> bool| { - cost.per_node[&node_of(dag, pick)].cost_per_second.unwrap() - }; - let is_shift = |p: &Payload| matches!(p, Payload::NonASAP(NonASAPOp::TimeShift { .. })); - let is_range = |p: &Payload| matches!(p, Payload::NonASAP(NonASAPOp::TimeRange { .. })); - assert_eq!(cost_of(&shifted, &shifted_cost, is_shift), 0.0); - assert!(close( - cost_of(&shifted, &shifted_cost, is_scan), - cost_of(&plain, &plain_cost, is_scan) - )); - assert!(close( - shifted_cost.cost_per_second.unwrap(), - plain_cost.cost_per_second.unwrap() - )); - let range = &plain_cost.per_node[&node_of(&plain, is_range)]; - assert!(range.detail.contains("keep"), "{}", range.detail); - } - - const AT_INGESTION: Materialization = - Materialization::MatAtIngestion(asap_types::physical::Medium::Memory); - const AT_QUERY: Materialization = - Materialization::MatAtQuery(asap_types::physical::Medium::Memory); - - fn node_of(dag: &PhysicalASAPDAG, pick: impl Fn(&Payload) -> bool) -> PhysicalASAPNodeId { - dag.nodes.iter().find(|n| pick(&n.payload)).unwrap().id - } - - fn is_scan(payload: &Payload) -> bool { - matches!(payload, Payload::NonASAP(NonASAPOp::Scan { .. })) - } - - fn is_build(payload: &Payload) -> bool { - matches!(payload, Payload::ASAP(ASAPOp::SummaryAgg { .. })) - } - - fn priced(dag: &PhysicalASAPDAG, recurrence: &[QueryRecurrence]) -> PlanCost { - price(dag, recurrence, &data(), &CostInputs::default()).unwrap() - } - - /// The default settings with cost model 1's unit costs edited. - fn unit_costs(edit: impl FnOnce(&mut ResourceUnitCost)) -> CostInputs { - let mut prices = ResourceUnitCost::illustrative(); - edit(&mut prices); - CostInputs { - cost_model: CostModel::UsageOverTime(prices), - ..CostInputs::default() - } - } - - fn close(a: f64, b: f64) -> bool { - (a - b).abs() <= 1e-12 * a.abs().max(b.abs()).max(1e-300) - } - - /// An ingestion-time node is priced over one second of arriving rows, - /// once per second, whatever its readers' evaluation rate; a query-time - /// node scales with that rate. - #[test] - fn ingestion_time_node_is_priced_per_second_of_arriving_rows() { - let dag = kll_dag(AT_INGESTION); - let (fast, slow) = ( - priced(&dag, &[repeating(1_000)]), - priced(&dag, &[repeating(10_000)]), - ); - for id in [node_of(&dag, is_scan), node_of(&dag, is_build)] { - assert_eq!( - fast.per_node[&id].cost_per_second, - slow.per_node[&id].cost_per_second - ); - assert!(fast.per_node[&id].detail.contains("; x 1.0000/s")); - } - let estimate = dag.roots[0]; - assert!(close( - fast.per_node[&estimate].cost_per_second.unwrap(), - 10.0 * slow.per_node[&estimate].cost_per_second.unwrap() - )); - } - - /// Kept state is charged per byte per second: at ingestion time, twice - /// the state's bytes (the window being built and the completed one) for - /// every second. At query time the state is reused only within one - /// evaluation, so it is held only while that evaluation runs: its bytes - /// times the evaluation's latency, once per evaluation. State computed - /// and dropped within a query (`NotMaterialized`) is held while that - /// query runs too, and charged the same way. - #[test] - fn kept_state_is_charged_per_byte_per_second() { - let cost = |dag: &PhysicalASAPDAG, w| { - let settings = unit_costs(|c| c.cost_per_memory_byte_second = w); - price(dag, &recurrences(1), &data(), &settings).unwrap() - }; - for materialization in [AT_INGESTION, AT_QUERY, Materialization::NotMaterialized] { - let dag = kll_dag(materialization); - let build = node_of(&dag, is_build); - let (with, without) = (cost(&dag, 1.25e-7), cost(&dag, 0.0)); - let Payload::ASAP(ASAPOp::SummaryAgg { family, .. }) = - &dag.nodes.iter().find(|n| n.id == build).unwrap().payload - else { - unreachable!() - }; - // One state per series: data() declares 10 000. At query time - // each sees its series' 12 samples: the 120 000 rows of its - // source's 1-min window (data()'s 10 000 rows per second over its - // five sources); at ingestion time its samples are unknown. - let samples = match materialization { - AT_INGESTION => StateInputRows::UNKNOWN, - _ => StateInputRows { - rows_per_state: Some(12), - non_negative: false, - }, - }; - let state = 10_000.0 * summary_shape(family, samples).1 as f64; - let byte_seconds = if materialization == AT_INGESTION { - 2.0 * state - } else { - let evaluation_s = with.usage.latency_per_query[0].total_ms() / 1_000.0; - assert!(evaluation_s > 0.0 && evaluation_s < 1.0, "{evaluation_s}"); - // Once per evaluation of the query repeating every 10 s. - assert!(with.per_node[&build].detail.contains("; x 0.1000/s")); - state * evaluation_s * 0.1 - }; - let term = with.per_node[&build].cost_per_second.unwrap() - - without.per_node[&build].cost_per_second.unwrap(); - assert!( - close(term, 1.25e-7 * byte_seconds), - "{materialization:?}: {term} vs {}", - 1.25e-7 * byte_seconds - ); - // The candidate's memory term covers every node's held memory, - // the build's included. - let candidate_term = with.cost_per_second.unwrap() - without.cost_per_second.unwrap(); - assert!(close( - candidate_term, - 1.25e-7 * with.usage.average_memory_bytes - )); - assert!(candidate_term >= term); - } - } - - /// Under cost model 1, memory is charged as bytes held per second - /// (average memory): the memory a computation holds while it runs (a - /// sketch being built, an exact quantile's values) counts for the time - /// it is held, once per evaluation. The memory price's term is exactly - /// the average memory, which is below the peak memory. - #[test] - fn memory_held_only_while_a_query_runs_is_charged_for_the_time_it_is_held() { - let memory_term = |dag: &PhysicalASAPDAG| { - let cost = |price_per_byte_second| { - let settings = - unit_costs(|c| c.cost_per_memory_byte_second = price_per_byte_second); - price(dag, &recurrences(1), &data(), &settings).unwrap() - }; - let (with, without) = (cost(1.0), cost(0.0)); - let term = with.cost_per_second.unwrap() - without.cost_per_second.unwrap(); - (term, with.usage) - }; - let mut dags: Vec<(String, PhysicalASAPDAG)> = [ - "quantile_over_time(0.99, m[1m])", - "quantile by (job) (0.99, m)", - ] - .into_iter() - .map(|query| { - let root = lower_promql(query, AccuracyTarget::exact()); - let dag = - optimize_physical_asap("L", &[QueryRoot::Operator(root)], &[], &data(), false) - .unwrap() - .remove(0) - .dag; - (query.to_string(), dag) - }) - .collect(); - for materialization in [AT_INGESTION, AT_QUERY, Materialization::NotMaterialized] { - dags.push((format!("{materialization:?}"), kll_dag(materialization))); - } - for (name, dag) in &dags { - let (term, usage) = memory_term(dag); - assert!( - close(term, usage.average_memory_bytes), - "{name}: {term} vs {}", - usage.average_memory_bytes - ); - assert!( - usage.peak_memory_bytes as f64 > usage.average_memory_bytes, - "{name}: {usage:?}" - ); - } - // Not materialized, nothing is kept, but the sketch being built is - // held while the query runs: the memory price adds to the cost. - let (term, _) = memory_term(&kll_dag(Materialization::NotMaterialized)); - assert!(term > 0.0, "{term}"); - } - - /// Kept exact state that grows with the data is charged by the values it - /// keeps: a materialized exact quantile keeps every value of its group, - /// a sum one value per group. - #[test] - fn kept_exact_state_is_charged_by_the_values_it_keeps() { - let kept = |query: &str| { - let root = lower_promql(query, AccuracyTarget::exact()); - let mut dag = - optimize_physical_asap("L", &[QueryRoot::Operator(root)], &[], &data(), false) - .unwrap() - .remove(0) - .dag; - let aggregate = dag - .nodes - .iter_mut() - .find(|n| matches!(n.payload, Payload::NonASAP(NonASAPOp::Aggregate { .. }))) - .unwrap(); - aggregate.materialization = AT_QUERY; - let id = aggregate.id; - let cost = |w| { - let settings = unit_costs(|c| c.cost_per_memory_byte_second = w); - // 10 jobs, so each group keeps many values. - let mut data = arriving(10_000.0); - data.sources.get_mut("m").unwrap().distinct_values_per_label = - [("job".to_string(), 10)].into(); - price(&dag, &recurrences(1), &data, &settings) - .unwrap() - .per_node[&id] - .cost_per_second - .unwrap() - }; - (cost(1.0) - cost(0.0)).round() as u64 - }; - let (quantile, sum) = ( - kept("quantile by (job) (0.99, m)"), - kept("sum by (job) (m)"), - ); - assert!(quantile > 10 * sum, "{quantile} vs {sum}"); - } - - /// A `NotMaterialized` node read by two queries is computed for each, at - /// each one's rate; `MatAtQuery`, it is computed once at their combined - /// rate, where queries at the same interval run together. - #[test] - fn rates_follow_per_consumer_charging() { - let root = lower_promql("sum by (job) (rate(m[1m]))", AccuracyTarget::exact()); - let one = optimize_physical_asap( - "L", - &[QueryRoot::Operator(root.clone())], - &[], - &data(), - false, - ) - .unwrap() - .remove(0) - .dag; - let mut two = optimize_physical_asap( - "L", - &[QueryRoot::Operator(root.clone()), QueryRoot::Operator(root)], - &[], - &data(), - false, - ) - .unwrap() - .remove(0) - .dag; - let without_memory = unit_costs(|c| c.cost_per_memory_byte_second = 0.0); - let total = |dag: &PhysicalASAPDAG, recurrence: &[QueryRecurrence]| { - price(dag, recurrence, &data(), &without_memory) - .unwrap() - .cost_per_second - .unwrap() - }; - let single = total(&one, &recurrences(1)); - let mixed = [repeating(10_000), repeating(20_000)]; - assert!(close(total(&two, &recurrences(2)), 2.0 * single)); - assert!(close(total(&two, &mixed), 1.5 * single)); - assert_eq!(two.roots[0], two.roots[1], "one root node for both queries"); - for node in &mut two.nodes { - node.materialization = AT_QUERY; - } - assert!(close(total(&two, &recurrences(2)), single)); - assert!(close(total(&two, &mixed), 1.5 * single)); - } - - /// Queries that refresh together at one interval and step evaluate at - /// the same instants: a node they share runs at the most steps among - /// them, not their sum. One over the last hour (61 steps) and one over - /// the last 10 min (11 steps), both every minute at 1-min steps, and a - /// third with no range (its one evaluation is the others' last step): - /// 61 evaluations a minute. A different step adds only the instants - /// the others do not evaluate at. - #[test] - fn readers_at_one_interval_and_step_share_their_steps() { - let every_minute = repeating(60_000); - let range = |span_ms: u64, step_ms: u64| Some(RangeEvaluation { step_ms, span_ms }); - let rate = |queries: &[Option]| { - evaluation_rate(queries.iter().map(|r| (&every_minute, *r)), 3_600.0).unwrap() - }; - let close = |a: f64, b: f64| (a - b).abs() < 1e-12; - assert!(close(rate(&[range(3_600_000, 60_000)]), 61.0 / 60.0)); - assert!(close( - rate(&[range(3_600_000, 60_000), range(600_000, 60_000), None]), - 61.0 / 60.0 - )); - assert!(close(rate(&[None, None]), 1.0 / 60.0)); - // A one-step range evaluates only at the request's end. - assert!(close( - rate(&[range(3_600_000, 60_000), range(0, 30_000)]), - 61.0 / 60.0 - )); - // A 30 s step over the last 10 min shares 11 of its 21 instants - // with the 1-min steps over the last hour: 71 distinct instants. - assert!(close( - rate(&[range(3_600_000, 60_000), range(600_000, 30_000)]), - 71.0 / 60.0 - )); - } - - /// Queries every 20 s at phases 0 and 5 s evaluate at different - /// instants: a node they share runs at both, twice as often as at one - /// phase. Phases 0 and 40 s are one phase on 20 s, and a query of - /// unknown phase runs with the known one. - #[test] - fn readers_at_different_phases_evaluate_apart() { - let at = |phase_ms: u64| { - QueryRecurrence::Repeated(RepeatedDemand::FixedIntervalAt { - interval: asap_types::workload::RepetitionInterval(20_000), - evaluation_phase: asap_types::workload::TimestampMs(phase_ms), - }) - }; - let unknown = repeating(20_000); - let rate = |queries: &[&QueryRecurrence]| { - evaluation_rate(queries.iter().map(|r| (*r, None)), 3_600.0).unwrap() - }; - let close = |a: f64, b: f64| (a - b).abs() < 1e-12; - assert!(close(rate(&[&at(0), &at(5_000)]), 2.0 / 20.0)); - assert!(close(rate(&[&at(0), &at(40_000)]), 1.0 / 20.0)); - assert!(close(rate(&[&at(5_000), &unknown]), 1.0 / 20.0)); - assert!(close(rate(&[&at(0), &at(5_000), &unknown]), 2.0 / 20.0)); - } - - /// One-off and unknown recurrence are spread over the horizon: - /// invocations × per-evaluation cost / non_repeating_query_horizon_seconds. - #[test] - fn one_off_recurrence_is_spread_over_the_horizon() { - let root = lower_promql("sum by (job) (rate(m[1m]))", AccuracyTarget::exact()); - let dag = optimize_physical_asap("L", &[QueryRoot::Operator(root)], &[], &data(), false) - .unwrap() - .remove(0) - .dag; - let horizon = CostInputs::default().non_repeating_query_horizon_seconds; - assert_eq!(horizon, 3_600.0); - let per_evaluation = priced(&dag, &[repeating(1_000)]).cost_per_second.unwrap(); - let once = priced( - &dag, - &[QueryRecurrence::OneTime { - invocations: 3, - execute_at: None, - }], - ) - .cost_per_second - .unwrap(); - assert!(close(once, 3.0 * per_evaluation / horizon), "{once}"); - assert!(close( - priced(&dag, &[QueryRecurrence::Unknown]) - .cost_per_second - .unwrap(), - per_evaluation / horizon - )); - } - - /// Settings outside their domain are an error, not a price, checked once - /// before any candidate: a price the cost model reads that is negative, - /// the model's prices all 0 (cost model 1 with only provisioned prices - /// set), or a horizon that is not positive. A price the model does not - /// read is not checked. - #[test] - fn invalid_settings_are_rejected() { - let candidates = tumbling_window_candidates(10_000.0); - let select = |settings: &CostInputs| { - select_among_plans( - &candidates, - &[every_minute()], - &arriving(10_000.0), - SelectionModels::builtin().with_cost_inputs(settings), - ) - }; - let only_provisioned = ResourceUnitCost { - cost_per_cpu_op: 0.0, - cost_per_scanned_byte: 0.0, - cost_per_memory_byte_second: 0.0, - cost_per_disk_byte_second: 0.0, - ..ResourceUnitCost::illustrative() - }; - for settings in [ - unit_costs(|c| c.cost_per_memory_byte_second = -1.0), - under(CostModel::UsageOverTime(only_provisioned.clone())), - CostInputs { - non_repeating_query_horizon_seconds: 0.0, - ..CostInputs::default() - }, - ] { - assert!( - matches!( - select(&settings), - Err(ErrorSelectingPlan::InvalidSettings(_)) - ), - "{settings:?}" - ); - } - assert!(select(&under(CostModel::ProvisionedCapacity(only_provisioned))).is_ok()); - assert!(select(&unit_costs( - |c| c.cost_per_provisioned_cpu_core_second = -1.0 - )) - .is_ok()); - } - - // ── Stage 2 materialization and the latency check ──────────────────── - - const MINUTE: u32 = 60_000; - - /// The p99 target [`tumbling_window_candidates`] are sized for. - fn p99_target() -> AccuracyTarget { - AccuracyTarget::max_error(0.01) - .failure_probability(0.001) - .accuracy_metric(AccuracyMetric::Rank) - } - - /// A predictable p99 query evaluated every minute, with no latency bound. - fn every_minute() -> QueryWorkloadEntry { - QueryWorkloadEntry { - predictability: asap_types::workload::Predictability::Predictable { known_at: None }, - ..entry_with(p99_target(), repeating(MINUTE)) - } - } - - /// [`data`] with each test source arriving at `rows_per_s`: pricing - /// gives each source its series share of the workload's rate. - fn arriving(rows_per_s: f64) -> DataWorkload { - DataWorkload { - arrival: DataArrival::Ingesting { - rows_per_second: rows_per_s * TEST_SOURCES.len() as f64, - }, - ..data() - } - } - - /// The physical candidates of a per-series p99 over 5 min with its KLL - /// in five 1-min tumbling windows, every minute over data arriving at - /// `rows_per_s`: rebuilt at query time (B2), then maintained at - /// ingestion time (B1). - fn tumbling_window_candidates(rows_per_s: f64) -> Vec { - let candidates = - tumbling_window_candidates_for(rows_per_s, &DeploymentCapabilities::default()); - assert_eq!(candidates.len(), 2); - candidates - } - - /// [`tumbling_window_candidates`] for a deployment with `deployment`'s - /// capabilities: B2, B1, then, when it keeps query-time results, B3 - /// (built at query time and kept across evaluations). - fn tumbling_window_candidates_for( - rows_per_s: f64, - deployment: &DeploymentCapabilities, - ) -> Vec { - let root = lower_promql("quantile_over_time(0.99, m[5m])", p99_target()); - let inventory = - asap_logical_asap_optimization::test_support::generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(root), - ]), - ) - .unwrap(); - let kll = inventory.targets[0] - .ways_to_compute - .iter() - .position(|c| { - matches!( - c, - asap_types::stages::HowToComputeAggregateMeasure::Summary(SketchAlgorithm::Kll) - ) - }) - .unwrap(); - let roots: Vec<_> = - asap_logical_asap_optimization::test_support::compose_logical_candidate( - &inventory, - &[kll], - ) - .unwrap(); - let roots = sized_as_pass1(&roots, &[p99_target()], &arriving(rows_per_s)); - let rule_candidates = - asap_logical_asap_optimization::test_support::tumbling_window_candidates( - &roots, - &[Some(u64::from(MINUTE))], - ); - let roots = - asap_logical_asap_optimization::test_support::window_composition_pass2_candidates( - &roots, - &rule_candidates, - ) - .unwrap() - .pop() - .unwrap() - .roots - .unwrap(); - let mut candidates = optimize_physical_asap( - "L1", - &roots, - &[every_minute()], - &arriving(rows_per_s), - deployment.query_time_results_retained, - ) - .unwrap(); - for (k, c) in candidates.iter_mut().enumerate() { - c.id = format!("P1-m{k}"); - } - candidates - } - - /// The raw data samples the scan `scan` reads per computation, from its - /// priced detail (`scan samples; ...`). - fn scanned_samples(cost: &PlanCost, scan: PhysicalASAPNodeId) -> u64 { - let detail = &cost.per_node[&scan].detail; - detail - .strip_prefix("scan ") - .and_then(|rest| rest.split(' ').next()) - .and_then(|rows| rows.parse().ok()) - .unwrap_or_else(|| panic!("not a scan: {detail}")) - } - - /// A deployment that keeps query-time results across evaluations. - fn keeping_query_time_results() -> DeploymentCapabilities { - DeploymentCapabilities { - query_time_results_retained: true, - ..DeploymentCapabilities::default() - } - } - - /// `query_time_results_retained` gates keeping a query-time result across - /// evaluations, and only that. Without it, Stage 2 offers the 1-min - /// tumbling windows of a query repeating every minute rebuilt (B2) or maintained - /// (B1); with it, also built at query time and kept (B3, - /// `MatAtQueryKeptAcrossEvaluations`). B3 is computed once per - /// evaluation, reads one tumbling window (1 min) of raw data, keeps that much raw - /// data on disk, and holds its 4 older tumbling windows in memory between - /// evaluations (on disk, if it were placed there). The same chain - /// `MatAtQuery` is reused only within one evaluation: the scan reads the - /// 5-min window and nothing is kept between evaluations. - #[test] - fn query_time_results_retained_keeps_a_tumbling_window_chain_across_evaluations() { - assert_eq!(tumbling_window_candidates(10_000.0).len(), 2); - let candidates = tumbling_window_candidates_for(10_000.0, &keeping_query_time_results()); - assert_eq!(candidates.len(), 3); - let b2 = &candidates[0].dag; - let b3 = &candidates[2].dag; - let chain = node_of(b3, is_chain); - let with_chain = |materialization: Materialization| { - let mut dag = b3.clone(); - for node in dag.nodes.iter_mut().filter(|n| n.id == chain) { - node.materialization = materialization; - } - dag - }; - assert_eq!( - *b3, - with_chain(Materialization::MatAtQueryKeptAcrossEvaluations( - Medium::Memory - )) - ); - let recurrence = [repeating(MINUTE)]; - let settings = CostInputs::default(); - let kept = price(b3, &recurrence, &arriving(10_000.0), &settings).unwrap(); - let rebuilt = price(b2, &recurrence, &arriving(10_000.0), &settings).unwrap(); - let scan = node_of(b3, is_scan); - assert_eq!(scanned_samples(&kept, scan), 10_000 * 60); - assert_eq!( - scanned_samples(&rebuilt, node_of(b2, is_scan)), - 10_000 * 300 - ); - assert_eq!(kept.usage.disk_bytes * 5, rebuilt.usage.disk_bytes); - let Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { family, .. }) = - &b3.nodes.iter().find(|n| n.id == chain).unwrap().payload - else { - unreachable!() - }; - // Each tumbling window sees its series' 60 samples: the one minute of 10 000 - // rows per second over 10 000 series it reads. - let samples = StateInputRows { - rows_per_state: Some(60), - non_negative: false, - }; - let (_, state_bytes) = summary_shape(family, samples); - let chain_cost = &kept.per_node[&chain]; - assert!(chain_cost.detail.contains("; x 0.0167/s"), "{chain_cost:?}"); - // One state per series: arriving() declares 10 000. - let between = 4 * 10_000 * state_bytes; - assert!(chain_cost - .detail - .contains(&format!("keeps {between} bytes across evaluations"))); - // On disk, the same N − 1 older tumbling windows are kept. - let on_disk = with_chain(Materialization::MatAtQueryKeptAcrossEvaluations( - Medium::Disk, - )); - let on_disk = price(&on_disk, &recurrence, &arriving(10_000.0), &settings).unwrap(); - assert_eq!( - on_disk.usage.disk_bytes, - kept.usage.disk_bytes + between, - "{on_disk:?}" - ); - // Shared within one evaluation, nothing is kept between them. - let shared = with_chain(Materialization::MatAtQuery(Medium::Memory)); - let within_one = price(&shared, &recurrence, &arriving(10_000.0), &settings).unwrap(); - assert_eq!(scanned_samples(&within_one, scan), 10_000 * 300); - assert!(!within_one.per_node[&chain] - .detail - .contains("across evaluations")); - assert!(within_one.usage.average_memory_bytes < kept.usage.average_memory_bytes); - } - - fn is_chain(payload: &Payload) -> bool { - matches!( - payload, - Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) - ) - } - - /// #509 Example 4, B1: maintained at ingestion time, a chain of N tumbling windows - /// pays the update of the newest tumbling window only, one second of arriving rows - /// once per second, and keeps N + 1 tumbling windows (the N the query merges and - /// the one being built) per series. - #[test] - fn a_chain_at_ingestion_pays_the_newest_tumbling_window_and_keeps_n_plus_one_tumbling_windows() - { - let candidates = tumbling_window_candidates(10_000.0); - let dag = &candidates[1].dag; - let chain = node_of(dag, is_chain); - let Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { - family, - tumbling_window_count, - .. - }) = &dag.nodes.iter().find(|n| n.id == chain).unwrap().payload - else { - unreachable!() - }; - assert_eq!(*tumbling_window_count, 5); - // Each tumbling window sees its series' 60 samples: one minute of 10 000 rows - // per second over 10 000 series. - let samples = StateInputRows { - rows_per_state: Some(60), - non_negative: false, - }; - let (depth, state_bytes) = summary_shape(family, samples); - let recurrence = [repeating(MINUTE)]; - let without_memory = unit_costs(|c| c.cost_per_memory_byte_second = 0.0); - let work = &price(dag, &recurrence, &arriving(10_000.0), &without_memory) - .unwrap() - .per_node[&chain]; - assert!(work.detail.contains("; x 1.0000/s"), "{work:?}"); - assert!( - close( - work.cost_per_second.unwrap(), - 10_000.0 * (depth + 1.0) * 1e-6 - ), - "{work:?}" - ); - let with_memory = &price( - dag, - &recurrence, - &arriving(10_000.0), - &CostInputs::default(), - ) - .unwrap() - .per_node[&chain]; - let kept = 6 * 10_000 * state_bytes; - assert!( - close( - with_memory.cost_per_second.unwrap() - work.cost_per_second.unwrap(), - 1.25e-7 * kept as f64 - ), - "{with_memory:?}" - ); - assert!(with_memory.detail.contains(&format!("keeps {kept} bytes"))); - } - - /// The latency check (S6, sketch-bench cost model §5): a query's - /// latency is its query-time work on one core. Rebuilding the tumbling windows at - /// query time (B2) takes longer than merging the maintained ones (B1); - /// with a bound between the two, B2 is rejected as invalid with its - /// latency and B1 is selected. - #[test] - fn a_candidate_over_a_querys_latency_bound_is_rejected_with_its_latency() { - let candidates = tumbling_window_candidates(10_000.0); - let latency = |c: &PhysicalWorkloadPlan| { - let cost = price( - &c.dag, - &[repeating(MINUTE)], - &arriving(10_000.0), - &CostInputs::default(), - ) - .unwrap(); - cost.usage.latency_per_query[0] - }; - let (b2, b1) = (latency(&candidates[0]), latency(&candidates[1])); - assert_eq!((b2.compaction_ms, b1.compaction_ms), (0.0, 0.0)); - assert!(b1.query_ms < b2.query_ms, "{b1:?} vs {b2:?}"); - let bound = (b1.query_ms + b2.query_ms) / 2.0; - let mut bounded = every_minute(); - bounded.requirements.max_latency_ms = Some(bound); - let selection = select_among_plans( - &candidates, - &[bounded], - &arriving(10_000.0), - SelectionModels::builtin(), - ) - .unwrap(); - assert_eq!(selection.selected, candidates[1].id); - let rejected = &selection.rejected[0]; - assert_eq!(rejected.id, candidates[0].id); - assert_eq!(rejected.cause, RejectionCause::LatencyBoundExceeded); - assert!( - rejected.reason.starts_with(&format!( - "q1: latency {:.1} ms per evaluation (compaction 0.0 ms + query", - b2.total_ms() - )) && rejected - .reason - .ends_with(&format!("is over its {bound} ms bound")), - "{}", - rejected.reason - ); - assert!(!selection.costs.contains_key(&candidates[0].id)); - } - - /// A query evaluated every minute whose evaluation takes longer than a - /// minute never keeps up, so even without a latency bound the - /// candidate is invalid. Over data fast enough that rebuilding the - /// tumbling windows at query time (B2) takes over a minute while merging the - /// maintained ones (B1) does not, B2 is rejected with its latency and - /// the interval, and B1 is selected. - #[test] - fn a_candidate_slower_than_a_querys_repeat_interval_is_rejected_without_a_bound() { - let rows_per_s = 1e9; - let candidates = tumbling_window_candidates(rows_per_s); - let latency = |c: &PhysicalWorkloadPlan| { - price( - &c.dag, - &[repeating(MINUTE)], - &arriving(rows_per_s), - &CostInputs::default(), - ) - .unwrap() - .usage - .latency_per_query[0] - }; - let (b2, b1) = (latency(&candidates[0]), latency(&candidates[1])); - assert!( - b1.total_ms() < f64::from(MINUTE) && b2.total_ms() > f64::from(MINUTE), - "{b1:?} vs {b2:?}" - ); - let unbounded = every_minute(); - assert_eq!(unbounded.requirements.max_latency_ms, None); - let selection = select_among_plans( - &candidates, - &[unbounded], - &arriving(rows_per_s), - SelectionModels::builtin(), - ) - .unwrap(); - assert_eq!(selection.selected, candidates[1].id); - let rejected = &selection.rejected[0]; - assert_eq!(rejected.id, candidates[0].id); - assert_eq!(rejected.cause, RejectionCause::LatencyOverRepeatInterval); - assert!(rejected.cause.failed_a_check()); - assert_eq!( - rejected.reason, - format!( - "q1: latency {:.1} ms per evaluation (compaction {:.1} ms + query {:.1} ms) is \ - over its {MINUTE} ms repeat interval, so its evaluations cannot keep up", - b2.total_ms(), - b2.compaction_ms, - b2.query_ms - ) - ); - assert!(!selection.costs.contains_key(&candidates[0].id)); - } - - /// With no repeat interval (a one-time query) there is nothing to keep - /// up with: the slow candidate stays valid and is selected when it is - /// cheaper. - #[test] - fn a_one_time_query_has_no_repeat_interval_to_keep_up_with() { - let rows_per_s = 1e9; - let candidates = tumbling_window_candidates(rows_per_s); - let once = QueryWorkloadEntry { - recurrence: QueryRecurrence::OneTime { - invocations: 1, - execute_at: None, - }, - ..every_minute() - }; - let selection = select_among_plans( - &candidates, - &[once], - &arriving(rows_per_s), - SelectionModels::builtin(), - ) - .unwrap(); - // Run once an hour, rebuilding at query time is the cheaper plan, - // over a minute per evaluation or not. - assert_eq!(selection.selected, candidates[0].id); - assert_eq!(selection.rejected.len(), 1, "{:?}", selection.rejected); - assert_eq!(selection.rejected[0].id, candidates[1].id); - assert_eq!(selection.rejected[0].cause, RejectionCause::Costlier); - } - - /// A query over both its latency bound and its repeat interval is - /// reported over its bound: the bound is the user's own requirement. - /// The interval is also checked for a repetition at a declared phase. - #[test] - fn a_broken_latency_bound_is_reported_before_the_repeat_interval() { - let rows_per_s = 1e9; - let candidates = tumbling_window_candidates(rows_per_s); - let mut bounded = QueryWorkloadEntry { - recurrence: QueryRecurrence::Repeated(RepeatedDemand::FixedIntervalAt { - interval: asap_types::workload::RepetitionInterval(MINUTE), - evaluation_phase: asap_types::workload::TimestampMs(0), - }), - ..every_minute() - }; - let select = |entry: &QueryWorkloadEntry| { - select_among_plans( - &candidates, - std::slice::from_ref(entry), - &arriving(rows_per_s), - SelectionModels::builtin(), - ) - .unwrap() - }; - let unbounded = select(&bounded); - assert_eq!( - unbounded.rejected[0].cause, - RejectionCause::LatencyOverRepeatInterval - ); - bounded.requirements.max_latency_ms = Some(f64::from(MINUTE)); - let selection = select(&bounded); - assert_eq!(selection.selected, candidates[1].id); - assert_eq!(selection.rejected[0].id, candidates[0].id); - assert_eq!( - selection.rejected[0].cause, - RejectionCause::LatencyBoundExceeded - ); - } - - /// When a maintained state's ingestion work needs more than one core, - /// its `k` workers' partial copies of the newest window are merged - /// before the query reads it: `(k − 1)` merges of one operation for - /// each of its states per window, on the query's chain. - #[test] - fn ingestion_work_over_one_core_adds_the_newest_windows_compaction() { - let latency = |rows_per_s: f64| { - let candidates = tumbling_window_candidates(rows_per_s); - price( - &candidates[1].dag, - &[repeating(MINUTE)], - &arriving(rows_per_s), - &CostInputs::default(), - ) - .unwrap() - .usage - .latency_per_query[0] - }; - assert_eq!(latency(10_000.0).compaction_ms, 0.0); - let fast = latency(1e10); - // 10,000 series, one state each per window, 1 ns per merge. - let merges = fast.compaction_ms / (10_000.0 * 1e-6); - assert!( - merges >= 1.0 && (merges - merges.round()).abs() < 1e-6, - "{fast:?}" - ); - } - - /// The newest window's compaction merges the `k` workers' partial - /// sketches with the state's measured query-time merge when the table - /// has one: each of the `(k − 1)·I` merges takes the measured 1,200 ns - /// instead of one operation. - #[test] - fn compaction_prices_merges_from_the_measured_merge() { - let candidates = tumbling_window_candidates(1e10); - let dag = &candidates[1].dag; - let kll = dag - .nodes - .iter() - .find_map(|n| match &n.payload { - Payload::ASAP( - ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), - .. - } - | ASAPOp::SummaryTumblingWindowChain { - family: FieldDataType::Sketch(kind, _), - .. - }, - ) => Some(kind.params().clone()), - _ => None, - }) - .expect("a KLL build"); - let latency = price( - dag, - &[repeating(MINUTE)], - &arriving(1e10), - &measured(vec![sketch_entry(OperatorKind::SummaryMerge, kll, 1200.0)]), - ) - .unwrap() - .usage - .latency_per_query[0]; - assert!(latency.compaction_ms > 0.0, "{latency:?}"); - let merges = latency.compaction_ms / 1200e-6; - assert!( - merges >= 1.0 && (merges - merges.round()).abs() < 1e-6, - "{latency:?}" - ); - } - - // ── Cost models ────────────────────────────────────────────────────── - - /// [`CostInputs`] with `cost_model`. - fn under(cost_model: CostModel) -> CostInputs { - CostInputs { - cost_model, - ..CostInputs::default() - } - } - - /// The candidate's usage is what its nodes are charged for: priced at - /// one unit for a single resource under cost model 1 (CPU, scanned - /// bytes, memory byte-seconds or disk), the nodes' prices add up to the - /// candidate's usage of it. - #[test] - fn node_prices_add_up_to_the_candidates_usage() { - let only = |set: fn(&mut ResourceUnitCost)| { - let mut prices = ResourceUnitCost { - cost_per_cpu_op: 0.0, - cost_per_scanned_byte: 0.0, - cost_per_memory_byte_second: 0.0, - cost_per_provisioned_memory_byte_second: 0.0, - cost_per_provisioned_cpu_core_second: 0.0, - cost_per_disk_byte_second: 0.0, - }; - set(&mut prices); - CostInputs { - cost_model: CostModel::UsageOverTime(prices), - ..CostInputs::default() - } - }; - // Each resource's unit price and the candidate's usage of it. - type Resource = (fn(&mut ResourceUnitCost), fn(&ResourceUsage) -> f64); - let resources: [Resource; 4] = [ - (|p| p.cost_per_cpu_op = 1.0, |u| u.cpu_ops_per_second), - ( - |p| p.cost_per_scanned_byte = 1.0, - |u| u.scanned_bytes_per_second, - ), - ( - |p| p.cost_per_memory_byte_second = 1.0, - |u| u.average_memory_bytes, - ), - ( - |p| p.cost_per_disk_byte_second = 1.0, - |u| u.disk_bytes as f64, - ), - ]; - for candidate in tumbling_window_candidates(10_000.0) { - for (set, used) in resources { - let cost = price( - &candidate.dag, - &[repeating(MINUTE)], - &arriving(10_000.0), - &only(set), - ) - .unwrap(); - let charged: f64 = cost - .per_node - .values() - .filter_map(|n| n.cost_per_second) - .sum(); - assert!( - close(charged, used(&cost.usage)), - "{}: {charged} vs {}", - candidate.id, - used(&cost.usage) - ); - assert_eq!(cost.usage.latency_per_query.len(), 1); - } - } - } - - /// Cost model 2 prices the candidate's peaks per second: provisioned - /// memory, cores and disk; its nodes carry no price. B1 (maintained - /// tumbling windows) needs a core for ingestion and one for the query; B2 (rebuilt - /// at query time) one. - #[test] - fn provisioned_capacity_prices_peak_memory_cores_and_disk() { - let prices = ResourceUnitCost { - cost_per_disk_byte_second: 1e-9, - ..ResourceUnitCost::illustrative() - }; - let settings = under(CostModel::ProvisionedCapacity(prices.clone())); - let candidates = tumbling_window_candidates(10_000.0); - let cores: Vec = candidates - .iter() - .map(|candidate| { - let cost = price( - &candidate.dag, - &[repeating(MINUTE)], - &arriving(10_000.0), - &settings, - ) - .unwrap(); - let usage = &cost.usage; - let expected = prices.cost_per_provisioned_memory_byte_second - * usage.peak_memory_bytes as f64 - + prices.cost_per_provisioned_cpu_core_second * f64::from(usage.peak_cpu_cores) - + prices.cost_per_disk_byte_second * usage.disk_bytes as f64; - assert_eq!(cost.cost_per_second, Some(expected)); - assert!(cost.per_node.values().all(|n| n.cost_per_second.is_none())); - usage.peak_cpu_cores - }) - .collect(); - assert_eq!(cores, [1, 2]); - } - - /// Cost model 3 has no prices; a candidate over the cluster's cores is - /// invalid with the reason, and the rest are chosen by the objective. - #[test] - fn cluster_capacity_rejects_a_candidate_over_its_cores() { - let limits = |cpu_cores| ClusterLimits { - memory_bytes: u64::MAX, - disk_bytes: u64::MAX, - cpu_cores, - objective: ClusterObjective::LowestLatency, - }; - let candidates = tumbling_window_candidates(10_000.0); - let select = |cpu_cores| { - let settings = under(CostModel::ClusterCapacity(limits(cpu_cores))); - select_among_plans( - &candidates, - &[every_minute()], - &arriving(10_000.0), - SelectionModels::builtin().with_cost_inputs(&settings), - ) - .unwrap() - }; - let unlimited = select(None); - assert_eq!(unlimited.selected, candidates[1].id, "B1 merges faster"); - assert!(unlimited - .costs - .values() - .all(|c| c.cost_per_second.is_none())); - let one_core = select(Some(1)); - assert_eq!(one_core.selected, candidates[0].id); - let b1 = one_core - .rejected - .iter() - .find(|r| r.id == candidates[1].id) - .unwrap(); - assert_eq!(b1.cause, RejectionCause::OverClusterCapacity); - assert_eq!(b1.reason, "2 CPU cores at peak is over the cluster's 1"); - } - - /// Raw data kept on disk for query-time scans is counted once per - /// source, for the longest window read: two queries over `m` keep its - /// last 5 minutes, as the 5-minute query alone does. - #[test] - fn raw_data_on_disk_is_the_longest_window_per_source() { - let disk = |queries: &[&str]| { - let roots: Vec<_> = queries - .iter() - .map(|q| QueryRoot::Operator(lower_promql(q, AccuracyTarget::exact()))) - .collect(); - let dag = optimize_physical_asap("L", &roots, &[], &data(), false) - .unwrap() - .remove(0) - .dag; - priced(&dag, &recurrences(queries.len())).usage.disk_bytes - }; - let five_minutes = disk(&["sum_over_time(m[5m])"]); - assert!(five_minutes > disk(&["sum_over_time(m[1m])"])); - assert_eq!( - disk(&["sum_over_time(m[1m])", "sum_over_time(m[5m])"]), - five_minutes - ); - } - - /// Cost model 3's disk limit: B2 reads its 5-minute window of raw data - /// at query time and keeps it on disk; B1 keeps none. A cluster with - /// less disk than B2's raw data leaves only B1. - #[test] - fn cluster_capacity_rejects_a_candidate_over_its_disk() { - let candidates = tumbling_window_candidates(10_000.0); - let select = |disk_bytes| { - let settings = under(CostModel::ClusterCapacity(ClusterLimits { - memory_bytes: u64::MAX, - disk_bytes, - cpu_cores: None, - objective: ClusterObjective::LeastMemory, - })); - select_among_plans( - &candidates, - &[every_minute()], - &arriving(10_000.0), - SelectionModels::builtin().with_cost_inputs(&settings), - ) - .unwrap() - }; - let unlimited = select(u64::MAX); - let raw = unlimited.costs[&candidates[0].id].usage.disk_bytes; - assert!(raw > 0); - assert_eq!(unlimited.costs[&candidates[1].id].usage.disk_bytes, 0); - assert_eq!(unlimited.selected, candidates[0].id, "B2 needs less memory"); - let small = select(raw - 1); - assert_eq!(small.selected, candidates[1].id); - let b2 = small - .rejected - .iter() - .find(|r| r.id == candidates[0].id) - .unwrap(); - assert_eq!(b2.cause, RejectionCause::OverClusterCapacity); - assert_eq!( - b2.reason, - format!("disk {raw} bytes is over the cluster's {} bytes", raw - 1) - ); - } - - /// State materialized on disk is counted as disk, not memory: the KLL - /// state kept on disk at ingestion time (its window being built and the - /// completed one) or for one evaluation at query time adds its bytes to - /// disk, nothing to average memory, and costs only its disk price. - #[test] - fn state_on_disk_is_counted_as_disk_not_memory() { - use asap_types::physical::Medium; - let settings = unit_costs(|c| c.cost_per_disk_byte_second = 1e-9); - let work_only = unit_costs(|c| c.cost_per_memory_byte_second = 0.0); - let state = |dag: &PhysicalASAPDAG| node_of(dag, is_build); - for (on_disk, in_memory) in [ - (Materialization::MatAtIngestion(Medium::Disk), AT_INGESTION), - (Materialization::MatAtQuery(Medium::Disk), AT_QUERY), - ] { - let (disk_dag, memory_dag) = (kll_dag(on_disk), kll_dag(in_memory)); - let disk = price(&disk_dag, &recurrences(1), &data(), &settings).unwrap(); - let memory = price(&memory_dag, &recurrences(1), &data(), &settings).unwrap(); - // The bytes kept are the same, on disk instead of in memory. - let bytes = disk.usage.disk_bytes - memory.usage.disk_bytes; - assert!(bytes > 0); - assert!( - memory.per_node[&state(&memory_dag)] - .detail - .contains(&format!(" {bytes} bytes")), - "{in_memory:?}: {bytes}" - ); - assert!( - disk.usage.average_memory_bytes < memory.usage.average_memory_bytes, - "{on_disk:?}" - ); - assert_eq!( - disk.usage.peak_memory_bytes + bytes, - memory.usage.peak_memory_bytes, - "{on_disk:?}" - ); - // On disk, the state costs only its disk price beside the work. - let work = price(&memory_dag, &recurrences(1), &data(), &work_only) - .unwrap() - .per_node[&state(&memory_dag)] - .cost_per_second - .unwrap(); - let on_disk_node = &disk.per_node[&state(&disk_dag)]; - assert!( - close( - on_disk_node.cost_per_second.unwrap(), - work + 1e-9 * bytes as f64 - ), - "{on_disk_node:?}" - ); - } - } - - /// Under every cluster objective, two candidates with the same usage - /// tie, and the earlier is selected; the later is reported as tied. - /// Under the Pareto front neither dominates the other: both are on it. - #[test] - fn cluster_objectives_keep_the_earlier_of_tied_candidates() { - let b2 = tumbling_window_candidates(10_000.0).remove(0); - let twin = PhysicalWorkloadPlan { - id: format!("{}-twin", b2.id), - ..b2.clone() - }; - let candidates = [b2, twin]; - for objective in [ - ClusterObjective::LowestLatency, - ClusterObjective::LeastMemory, - ] { - let settings = under(CostModel::ClusterCapacity(ClusterLimits { - memory_bytes: u64::MAX, - disk_bytes: u64::MAX, - cpu_cores: None, - objective, - })); - let selection = select_among_plans( - &candidates, - &[every_minute()], - &arriving(10_000.0), - SelectionModels::builtin().with_cost_inputs(&settings), - ) - .unwrap(); - assert_eq!(selection.selected, candidates[0].id, "{objective:?}"); - let twin = &selection.rejected[0]; - assert_eq!(twin.id, candidates[1].id); - assert_eq!(twin.cause, RejectionCause::TiedLostOnEnumerationOrder); - assert!( - twin.reason.starts_with(&format!( - "tied with the selected candidate {}", - candidates[0].id - )), - "{objective:?}: {}", - twin.reason - ); - } - } - - /// Every Pass 2 window-composition candidate of a per-series p50 over - /// 2 min and a p99 over 5 min, both every minute, with both builds in - /// 1-min tumbling windows: `shared` picks one chain read by both (the 2-min query - /// merging its newest 2 tumbling windows) or a chain each. Its physical candidate - /// with every chain maintained at ingestion time. - fn nested_windows_at_ingestion(shared: bool, rows_per_s: f64) -> PhysicalWorkloadPlan { - let queries = [ - ("quantile_over_time(0.5, m[2m])", MINUTE), - ("quantile_over_time(0.99, m[5m])", MINUTE), - ]; - shared_windows_at_ingestion(queries, shared, rows_per_s) - } - - /// [`nested_windows_at_ingestion`] for two queries, each with its own - /// repeat interval in ms. - fn shared_windows_at_ingestion( - queries: [(&str, u32); 2], - shared: bool, - rows_per_s: f64, - ) -> PhysicalWorkloadPlan { - use asap_logical_asap_optimization::test_support as pcc; - use asap_types::stages::HowToComputeAggregateMeasure; - let inventory = pcc::generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups( - &queries - .iter() - .map(|(query, _)| QueryRoot::Operator(lower_promql(query, p99_target()))) - .collect::>(), - ), - ) - .unwrap(); - let kll: Vec = inventory - .targets - .iter() - .map(|target| { - target - .ways_to_compute - .iter() - .position(|c| { - matches!( - c, - HowToComputeAggregateMeasure::Summary(SketchAlgorithm::Kll) - ) - }) - .unwrap() - }) - .collect(); - let roots: Vec<_> = pcc::compose_logical_candidate(&inventory, &kll).unwrap(); - let rule_candidates = - asap_logical_asap_optimization::test_support::tumbling_window_candidates( - &roots, - &queries.map(|(_, interval_ms)| Some(u64::from(interval_ms))), - ); - let roots = - asap_logical_asap_optimization::test_support::window_composition_pass2_candidates( - &roots, - &rule_candidates, - ) - .unwrap() - .into_iter() - .find(|c| { - c.tumbling_window_candidates.len() == 2 && c.shared_chains.is_empty() != shared - }) - .unwrap() - .roots - .unwrap(); - optimize_physical_asap( - "L1", - &roots, - &queries.map(|(_, interval_ms)| QueryWorkloadEntry { - predictability: asap_types::workload::Predictability::Predictable { - known_at: None, - }, - ..entry_with(p99_target(), repeating(interval_ms)) - }), - &arriving(rows_per_s), - false, - ) - .unwrap() - .into_iter() - .find(|c| { - c.dag.nodes.iter().all(|n| match n.payload { - Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) => { - matches!(n.materialization, Materialization::MatAtIngestion(_)) - } - _ => n.materialization == Materialization::NotMaterialized, - }) - }) - .unwrap() - } - - /// One chain of 5 tumbling windows maintained at ingestion time serves a 5-min and - /// a 2-min window: the 2-min query's merge reads its newest 2 tumbling windows per - /// series, the 5-min query's all 5, and the plan costs less than a chain - /// per query, since each arriving row updates one tumbling window instead of two. - #[test] - fn a_merge_of_the_newest_tumbling_windows_is_priced_by_the_tumbling_windows_it_reads() { - let rows_per_s = 10_000.0; - let shared = nested_windows_at_ingestion(true, rows_per_s); - let chains: Vec<_> = shared - .dag - .nodes - .iter() - .filter(|n| is_chain(&n.payload)) - .collect(); - assert_eq!(chains.len(), 1); - let recurrence = [repeating(MINUTE), repeating(MINUTE)]; - let settings = CostInputs::default(); - let cost = price(&shared.dag, &recurrence, &arriving(rows_per_s), &settings).unwrap(); - let states_read = |newest: Option| -> u64 { - let merge = shared - .dag - .nodes - .iter() - .find(|n| { - matches!(&n.payload, Payload::ASAP(ASAPOp::SummaryMerge { newest_tumbling_windows, .. }) - if *newest_tumbling_windows == newest) - }) - .unwrap(); - let detail = &cost.per_node[&merge.id].detail; - detail - .strip_prefix("merge ") - .and_then(|rest| rest.split(' ').next()) - .and_then(|states| states.parse().ok()) - .unwrap_or_else(|| panic!("not a merge: {detail}")) - }; - let (newest_two, all_five) = (states_read(Some(2)), states_read(None)); - assert!(all_five > 0); - assert_eq!(newest_two * 5, all_five * 2); - let separate = nested_windows_at_ingestion(false, rows_per_s); - let separate_cost = - price(&separate.dag, &recurrence, &arriving(rows_per_s), &settings).unwrap(); - assert!( - cost.cost_per_second.unwrap() < separate_cost.cost_per_second.unwrap(), - "{:?} vs {:?}", - cost.cost_per_second, - separate_cost.cost_per_second - ); - } - - /// A 10-min quantile every minute (10 × 1-min tumbling windows) and one every 5 - /// min (2 × 5-min tumbling windows), maintained at ingestion time: one chain of - /// 1-min tumbling windows serves both, the 5-min query merging all 10 tumbling windows once - /// per 5 min, and costs less than a chain per query. - #[test] - fn queries_at_different_intervals_share_one_chain_at_ingestion_time() { - let rows_per_s = 10_000.0; - let queries = [ - ("quantile_over_time(0.5, m[10m])", MINUTE), - ("quantile_over_time(0.99, m[10m])", 5 * MINUTE), - ]; - let shared = shared_windows_at_ingestion(queries, true, rows_per_s); - let chains: Vec<_> = shared - .dag - .nodes - .iter() - .filter(|n| is_chain(&n.payload)) - .collect(); - assert_eq!(chains.len(), 1); - assert!(matches!( - chains[0].payload, - Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms: 60_000, - tumbling_window_count: 10, - .. - }) - )); - let recurrence = [repeating(MINUTE), repeating(5 * MINUTE)]; - let settings = CostInputs::default(); - let cost = price(&shared.dag, &recurrence, &arriving(rows_per_s), &settings).unwrap(); - let separate = shared_windows_at_ingestion(queries, false, rows_per_s); - let separate_cost = - price(&separate.dag, &recurrence, &arriving(rows_per_s), &settings).unwrap(); - assert!( - cost.cost_per_second.unwrap() < separate_cost.cost_per_second.unwrap(), - "{:?} vs {:?}", - cost.cost_per_second, - separate_cost.cost_per_second - ); - assert!(cost.usage.average_memory_bytes < separate_cost.usage.average_memory_bytes); - // Each query merges all 10 tumbling windows per series at its own rate: once - // a minute and once per 5 min. - let mut merges: Vec<(u64, String)> = shared - .dag - .nodes - .iter() - .filter(|n| matches!(n.payload, Payload::ASAP(ASAPOp::SummaryMerge { .. }))) - .map(|n| { - let detail = &cost.per_node[&n.id].detail; - let states = detail - .strip_prefix("merge ") - .and_then(|rest| rest.split(' ').next()) - .and_then(|states| states.parse().ok()) - .unwrap_or_else(|| panic!("not a merge: {detail}")); - let rate = detail.split("; x ").nth(1).unwrap_or_default().to_string(); - (states, rate) - }) - .collect(); - merges.sort_by(|a, b| a.1.cmp(&b.1)); - assert_eq!(merges.len(), 2); - assert_eq!(merges[0].0, merges[1].0); - assert!(merges[0].1.starts_with("0.0033/s"), "{merges:?}"); - assert!(merges[1].1.starts_with("0.0167/s"), "{merges:?}"); - } - - /// A 1-hour quantile every minute (60 × 1-min tumbling windows) and every 45 s - /// (80 × 45-s tumbling windows) can share only a chain at their gcd, 240 × 15-s - /// tumbling windows. Maintained at ingestion time it keeps more states than both - /// per-query chains together, so it is priced higher than the - /// per-query chains maintained at ingestion time. - #[test] - fn a_shared_chain_at_a_fine_gcd_is_offered_but_costs_more() { - let rows_per_s = 10_000.0; - let queries = [ - ("quantile_over_time(0.5, m[1h])", MINUTE), - ("quantile_over_time(0.99, m[1h])", 45_000), - ]; - let shared = shared_windows_at_ingestion(queries, true, rows_per_s); - let chain = node_of(&shared.dag, is_chain); - let chain = shared.dag.nodes.iter().find(|n| n.id == chain).unwrap(); - assert!(matches!( - chain.payload, - Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms: 15_000, - tumbling_window_count: 240, - .. - }) - )); - let separate = shared_windows_at_ingestion(queries, false, rows_per_s); - let recurrence = [repeating(MINUTE), repeating(45_000)]; - let settings = CostInputs::default(); - let price_of = |candidate: &PhysicalWorkloadPlan| { - price( - &candidate.dag, - &recurrence, - &arriving(rows_per_s), - &settings, - ) - .unwrap() - .cost_per_second - .unwrap() - }; - assert!( - price_of(&shared) > price_of(&separate), - "{} vs {}", - price_of(&shared), - price_of(&separate) - ); - } - - /// With more than one ingest worker, each worker beyond the first holds - /// its own partial copy of a maintained state's newest window: B1's - /// peak memory grows by (k − 1) tumbling windows of states, its average memory - /// (and so cost model 1's price) does not. - #[test] - fn partial_copies_of_the_newest_window_add_to_peak_memory() { - let rows_per_s = 1e9; - let b1 = tumbling_window_candidates(rows_per_s).remove(1); - let chain = node_of(&b1.dag, is_chain); - let cost = price( - &b1.dag, - &[repeating(MINUTE)], - &arriving(rows_per_s), - &CostInputs::default(), - ) - .unwrap(); - let detail = &cost.per_node[&chain].detail; - let workers = cost.usage.peak_cpu_cores - 1; - assert!(workers > 1, "{workers}"); - // The chain keeps 6 tumbling windows (5 read and the one being built). - let kept: u64 = detail - .split("; keeps ") - .nth(1) - .and_then(|rest| rest.split(' ').next()) - .and_then(|bytes| bytes.parse().ok()) - .unwrap_or_else(|| panic!("{detail}")); - let copies = workers - 1; - let bytes = u64::from(copies) * (kept / 6); - assert!( - detail.contains(&format!( - "; {copies} partial copies of its newest window, {bytes} bytes" - )), - "{detail}" - ); - // At the peak, not on average: the kept bytes are B1's only - // averaged memory beside negligible query-time holds. - assert!(cost.usage.peak_memory_bytes >= kept + bytes); - assert!(cost.usage.average_memory_bytes < (kept + bytes) as f64); - } - - /// The KLL build of a `quantile_over_time` candidate, with its sketch - /// parameters, and the candidate. - fn kll_build() -> (PhysicalWorkloadPlan, PhysicalASAPNodeId, SketchParams) { - let (candidates, _) = per_series_kll(AccuracyScope::PerValue); - let candidate = candidates.into_iter().next().unwrap(); - let (build, sketch) = candidate - .dag - .nodes - .iter() - .find_map(|n| match &n.payload { - Payload::ASAP(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), - .. - }) => Some((n.id, kind.params().clone())), - _ => None, - }) - .unwrap(); - (candidate, build, sketch) - } - - /// An entry pricing `operator` at both phases on `sketch`. - fn sketch_entry( - operator: OperatorKind, - sketch: SketchParams, - ns_per_unit: f64, - ) -> OperatorCost { - OperatorCost { - operator, - phase: None, - sketch: Some(sketch), - exact: None, - measure: None, - ns_per_unit, - ns_per_invocation: 0.0, - memory_bytes_per_state: None, - } - } - - /// An entry pricing `operator` at `phase` with no family. - fn plain_entry( - operator: OperatorKind, - phase: Option, - ns_per_unit: f64, - ) -> OperatorCost { - OperatorCost { - operator, - phase, - sketch: None, - exact: None, - measure: None, - ns_per_unit, - ns_per_invocation: 0.0, - memory_bytes_per_state: None, - } - } - - /// Settings pricing from `operators`. - fn measured(operators: Vec) -> CostInputs { - CostInputs { - measured_operation_costs: MeasuredOperationCosts { - operators, - ..MeasuredOperationCosts::default() - }, - ..CostInputs::default() - } - } - - /// A measured update time for the candidate's KLL prices that build - /// from it (rows × measured ns, counted in operations of - /// `ms_per_cpu_op`) and reports `Measured`; every other node keeps its - /// formulas, reports `Analytical`, and has its CPU scaled by the - /// calibration factor, measured ÷ formula time of the build: 250 ns - /// against the formula's one 1-ns operation per row. - #[test] - fn a_measured_entry_prices_its_node_and_calibrates_the_others() { - let (candidate, build, sketch) = kll_build(); - // Scanning is free here, so each node's time is its CPU, which the - // factor scales (it leaves scanned bytes unscaled). - let cpu_time_only = |mut settings: CostInputs| { - settings.resource_speed.ms_per_scanned_byte = 0.0; - settings - }; - let before = price( - &candidate.dag, - &recurrences(1), - &data(), - &cpu_time_only(CostInputs::default()), - ) - .unwrap(); - let settings = cpu_time_only(measured(vec![sketch_entry( - OperatorKind::SummaryAgg, - sketch.clone(), - 250.0, - )])); - let after = price(&candidate.dag, &recurrences(1), &data(), &settings).unwrap(); - for (id, node) in &after.per_node { - let was = before.per_node[id].cpu_core_seconds_per_second; - if *id == build { - assert_eq!(node.price_source, NodePriceSource::Measured); - assert!(node.detail.contains("x 250.000 ns"), "{}", node.detail); - } else { - assert_eq!(node.price_source, NodePriceSource::Analytical, "node {id}"); - } - assert!( - close(node.cpu_core_seconds_per_second, 250.0 * was), - "node {id}: {} vs {was}", - node.cpu_core_seconds_per_second - ); - } - assert_eq!(before.source, COST_SOURCE); - assert_eq!(after.source, MIXED_COST_SOURCE); - // The candidate's CPU grows linearly in the measured time. - let cpu_at = |ns: f64| { - let settings = measured(vec![sketch_entry( - OperatorKind::SummaryAgg, - sketch.clone(), - ns, - )]); - price(&candidate.dag, &recurrences(1), &data(), &settings) - .unwrap() - .usage - .cpu_ops_per_second - }; - assert!(close( - (cpu_at(250.0) - cpu_at(100.0)) / (cpu_at(400.0) - cpu_at(100.0)), - 0.5 - )); - } - - /// A KLL with `k` = 469, as Pass 1 sizes it from the data, is priced - /// between the measured `k` = 200 and `k` = 1024, linearly in the log - /// of the state's size; a measured `k` matches exactly whatever failure - /// probability it was sized for, and a `k` beyond the measured ones - /// takes the nearest one's price. - #[test] - fn a_kll_between_two_measured_sizes_is_interpolated() { - let kll = |k, failure_probability| SketchParams::Kll { - k, - failure_probability, - }; - let table = MeasuredOperationCosts { - operators: vec![ - sketch_entry(OperatorKind::SummaryAgg, kll(200, None), 100.0), - sketch_entry(OperatorKind::SummaryAgg, kll(1024, None), 200.0), - ], - ..MeasuredOperationCosts::default() - }; - let size = |params: &SketchParams| sketch_shape(params, StateInputRows::UNKNOWN).1 as f64; - let at = |k| { - lookup( - &table, - OperatorKind::SummaryAgg, - ExecutionTiming::IngestionTime, - StateFamily::Sketch(&kll(k, Some(0.001))), - size, - ) - .unwrap() - .ns_per_unit - }; - let expected = 100.0 - + 100.0 * (size(&kll(469, None)).ln() - size(&kll(200, None)).ln()) - / (size(&kll(1024, None)).ln() - size(&kll(200, None)).ln()); - assert!(close(at(469), expected), "{} vs {expected}", at(469)); - assert!(at(469) > 100.0 && at(469) < 200.0); - assert_eq!(at(200), 100.0); - assert_eq!(at(1024), 200.0); - assert_eq!(at(4096), 200.0); - assert_eq!(at(50), 100.0); - // Another sketch, operator or family has no entry. - assert!(lookup( - &table, - OperatorKind::SummaryMerge, - ExecutionTiming::QueryTime, - StateFamily::Sketch(&kll(469, None)), - size, - ) - .is_none()); - assert!(lookup( - &table, - OperatorKind::SummaryAgg, - ExecutionTiming::QueryTime, - StateFamily::Sketch(&SketchParams::DDSketch { alpha: 0.01 }), - size, - ) - .is_none()); - } - - /// An entry for the node's own phase wins over one for both phases. - #[test] - fn a_phase_entry_wins_over_one_for_both_phases() { - let table = MeasuredOperationCosts { - operators: vec![ - plain_entry(OperatorKind::Scan, None, 20.0), - plain_entry(OperatorKind::Scan, Some(ExecutionTiming::QueryTime), 150.0), - ], - ..MeasuredOperationCosts::default() - }; - let at = |phase| { - lookup(&table, OperatorKind::Scan, phase, StateFamily::None, |_| { - 1.0 - }) - .unwrap() - .ns_per_unit - }; - assert_eq!(at(ExecutionTiming::QueryTime), 150.0); - assert_eq!(at(ExecutionTiming::IngestionTime), 20.0); - } - - /// The physical candidates of a per-series p99 over 5 min, queried every - /// minute over data arriving at 10,000 rows per second: each Stage 1 - /// choice (the exact rebuild from raw data samples, KLL, DDSketch, ...) - /// with each Stage 2 materialization, ids `P-m`. - fn quantile_candidates() -> Vec { - let root = lower_promql("quantile_over_time(0.99, m[5m])", p99_target()); - let inventory = - asap_logical_asap_optimization::test_support::generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(root), - ]), - ) - .unwrap(); - let mut out = Vec::new(); - for choice in 0..inventory.targets[0].choice_count() { - let roots: Vec<_> = - asap_logical_asap_optimization::test_support::compose_logical_candidate( - &inventory, - &[choice], - ) - .unwrap(); - let roots = sized_as_pass1(&roots, &[p99_target()], &arriving(10_000.0)); - let candidates = - optimize_physical_asap("L", &roots, &[every_minute()], &arriving(10_000.0), false) - .unwrap(); - for (k, mut candidate) in candidates.into_iter().enumerate() { - candidate.id = format!("P{}-m{k}", choice + 1); - out.push(candidate); - } - } - out - } - - /// Whether `candidate` builds no summary: it rebuilds the quantile - /// exactly from raw data samples. - fn exact_rebuild(candidate: &PhysicalWorkloadPlan) -> bool { - !candidate.dag.nodes.iter().any(|n| { - matches!( - n.payload, - Payload::ASAP( - ASAPOp::SummaryAgg { .. } | ASAPOp::SummaryTumblingWindowChain { .. } - ) - ) - }) - } - - /// The exact rebuild of [`quantile_candidates`] and the KLL tumbling window - /// candidates ([`tumbling_window_candidates`]: B2 rebuilt at query time, B1 - /// maintained at ingestion time), with the KLL's parameters. - fn exact_and_kll() -> (Vec, SketchParams) { - let mut candidates: Vec<_> = quantile_candidates() - .into_iter() - .filter(exact_rebuild) - .collect(); - assert_eq!(candidates.len(), 1); - candidates[0].id = "EXACT".to_string(); - candidates.extend(tumbling_window_candidates(10_000.0)); - let kll = candidates[2] - .dag - .nodes - .iter() - .find_map(|n| match &n.payload { - Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { - family: FieldDataType::Sketch(kind, _), - .. - }) => Some(kind.params().clone()), - _ => None, - }) - .expect("a KLL tumbling-window chain"); - (candidates, kll) - } - - /// Cost model 1 pricing CPU only (1 per ms of one core): the tests - /// below compare plans by the CPU their tables price. - fn cpu_only(operators: Vec) -> CostInputs { - CostInputs { - cost_model: CostModel::UsageOverTime(ResourceUnitCost { - cost_per_cpu_op: 1e-6, - cost_per_scanned_byte: 1e-7, - cost_per_memory_byte_second: 0.0, - cost_per_provisioned_memory_byte_second: 0.0, - cost_per_provisioned_cpu_core_second: 0.0, - cost_per_disk_byte_second: 0.0, - }), - ..measured(operators) - } - } - - /// The candidate Stage 3 selects among `candidates` for one p99 every - /// minute over 10,000 rows per second. - fn selected(candidates: &[PhysicalWorkloadPlan], settings: &CostInputs) -> WholePlanSelection { - select_among_plans( - candidates, - &[every_minute()], - &arriving(10_000.0), - SelectionModels::builtin().with_cost_inputs(settings), - ) - .unwrap() - } - - /// The candidate cheapest when every node without an entry keeps its - /// formulas' price (no calibration), as Stage 3 priced before. - fn cheapest_uncalibrated(candidates: &[PhysicalWorkloadPlan], settings: &CostInputs) -> String { - let cost = |candidate: &PhysicalWorkloadPlan| { - plan_cost( - &candidate.dag, - &node_map(&candidate.dag), - &kept_tumbling_window_chains(&candidate.dag), - &[repeating(MINUTE)], - &[], - &arriving(10_000.0), - settings, - 1.0, - ) - .unwrap() - .0 - .cost_per_second - .unwrap() - }; - candidates - .iter() - .min_by(|a, b| cost(a).total_cmp(&cost(b))) - .unwrap() - .id - .clone() - } - - /// Calibration keeps the unmeasured nodes on the measured scale. A - /// table measuring the KLL as expensive (150 ns per row) and one exact - /// operator as cheap (the time range, 2 ns per row), but not the exact - /// plan's scan and aggregate, made the exact rebuild from raw data samples - /// look cheapest while the formulas kept its scan and aggregate at - /// their 1-ns scale; with the calibration factor the KLL maintained at - /// ingestion time (B1) is selected. With the full table, the exact - /// rebuild is selected when it really is cheaper (1 ns per scanned - /// sample and aggregated row), and B1 when raw reads and the exact - /// quantile cost 100 and 50 ns. - #[test] - fn calibration_keeps_unmeasured_nodes_on_the_measured_scale() { - let (candidates, kll) = exact_and_kll(); - let quantile = |ns_per_unit| OperatorCost { - measure: Some("quantile".to_string()), - ..plain_entry(OperatorKind::Aggregate, None, ns_per_unit) - }; - let kll_entries = || { - vec![ - sketch_entry(OperatorKind::SummaryTumblingWindowChain, kll.clone(), 150.0), - sketch_entry(OperatorKind::SummaryMerge, kll.clone(), 2_000.0), - sketch_entry(OperatorKind::SummaryEstimate, kll.clone(), 5_000.0), - plain_entry(OperatorKind::TimeRange, None, 2.0), - plain_entry(OperatorKind::Project, None, 1.0), - ] - }; - let partial = cpu_only(kll_entries()); - assert_eq!(cheapest_uncalibrated(&candidates, &partial), "EXACT"); - let selection = selected(&candidates, &partial); - assert_eq!(selection.selected, "P1-m1", "{:?}", selection.rejected); - let full = |scan_ns, aggregate_ns| { - let mut entries = kll_entries(); - entries.push(plain_entry(OperatorKind::Scan, None, scan_ns)); - entries.push(quantile(aggregate_ns)); - cpu_only(entries) - }; - assert_eq!(selected(&candidates, &full(1.0, 1.0)).selected, "EXACT"); - assert_eq!(selected(&candidates, &full(100.0, 50.0)).selected, "P1-m1"); - } - - /// The calibration factor is one heuristic, so a partial table can - /// still mis-rank plans: if the exact plan's scan and quantile really - /// cost 40 and 20 ns, the full table selects the exact rebuild, while - /// the partial table of - /// [`calibration_keeps_unmeasured_nodes_on_the_measured_scale`] prices - /// them at the KLL's ratio and selects B1, which then costs more than - /// the exact rebuild under the full table. - #[test] - fn a_partial_table_can_still_mis_rank_plans() { - let (candidates, kll) = exact_and_kll(); - let partial = vec![ - sketch_entry(OperatorKind::SummaryTumblingWindowChain, kll.clone(), 150.0), - sketch_entry(OperatorKind::SummaryMerge, kll.clone(), 2_000.0), - sketch_entry(OperatorKind::SummaryEstimate, kll.clone(), 5_000.0), - plain_entry(OperatorKind::TimeRange, None, 2.0), - plain_entry(OperatorKind::Project, None, 1.0), - ]; - let mut full = partial.clone(); - full.push(plain_entry(OperatorKind::Scan, None, 40.0)); - full.push(OperatorCost { - measure: Some("quantile".to_string()), - ..plain_entry(OperatorKind::Aggregate, None, 20.0) - }); - assert_eq!(selected(&candidates, &cpu_only(partial)).selected, "P1-m1"); - let full = selected(&candidates, &cpu_only(full)); - assert_eq!(full.selected, "EXACT"); - let cost = |id: &str| full.costs[id].cost_per_second.unwrap(); - assert!(cost("P1-m1") > cost("EXACT")); - } - - /// Reading raw data samples from the database holding them is the - /// deployment's cost. With every operator measured, the exact rebuild - /// from raw data samples at query time is the cheapest plan for a p99 every - /// minute while its raw reads are free (0 ns per sample fetched and - /// decoded at query time); priced at 200 ns per sample, it costs more - /// than the KLL maintained at ingestion time (B1), which is selected. - #[test] - fn external_raw_reads_make_the_query_time_raw_plan_costlier() { - let (candidates, kll) = exact_and_kll(); - let table = |raw_read_ns| { - cpu_only(vec![ - plain_entry( - OperatorKind::Scan, - Some(ExecutionTiming::IngestionTime), - 20.0, - ), - plain_entry( - OperatorKind::Scan, - Some(ExecutionTiming::QueryTime), - raw_read_ns, - ), - plain_entry(OperatorKind::TimeRange, None, 1.0), - plain_entry(OperatorKind::Project, None, 1.0), - OperatorCost { - measure: Some("quantile".to_string()), - ..plain_entry(OperatorKind::Aggregate, None, 10.0) - }, - sketch_entry(OperatorKind::SummaryTumblingWindowChain, kll.clone(), 30.0), - sketch_entry(OperatorKind::SummaryMerge, kll.clone(), 500.0), - sketch_entry(OperatorKind::SummaryEstimate, kll.clone(), 1_000.0), - ]) - }; - let free = selected(&candidates, &table(0.0)); - assert_eq!(free.selected, "EXACT", "{:?}", free.rejected); - let priced = selected(&candidates, &table(200.0)); - assert_eq!(priced.selected, "P1-m1", "{:?}", priced.rejected); - let cost = - |selection: &WholePlanSelection, id: &str| selection.costs[id].cost_per_second.unwrap(); - assert!(cost(&priced, "EXACT") > cost(&priced, "P1-m1")); - assert!(cost(&priced, "EXACT") > cost(&free, "EXACT")); - } - - /// A calibration factor below 1 (here a KLL build measured at 0.01 ns - /// per row against the formula's 1 ns) scales only the CPU of the - /// nodes without an entry, never their scanned bytes, so no node's CPU - /// goes negative: the query-time scan keeps its scanned bytes and - /// takes at least their scan time. - #[test] - fn a_calibration_factor_below_one_never_makes_cpu_negative() { - let (candidate, _, sketch) = kll_build(); - let before = price( - &candidate.dag, - &recurrences(1), - &data(), - &CostInputs::default(), - ) - .unwrap(); - let settings = measured(vec![sketch_entry(OperatorKind::SummaryAgg, sketch, 0.01)]); - let after = price(&candidate.dag, &recurrences(1), &data(), &settings).unwrap(); - assert!(after.usage.cpu_ops_per_second >= 0.0, "{:?}", after.usage); - for (id, node) in &after.per_node { - assert!( - node.cpu_core_seconds_per_second >= 0.0, - "node {id}: {}", - node.detail - ); - assert!(node.cost_per_second.unwrap() >= 0.0, "node {id}"); - } - assert_eq!( - after.usage.scanned_bytes_per_second, - before.usage.scanned_bytes_per_second - ); - } - - /// Each query request pays the table's `ns_per_evaluation` once: on - /// the node the query reads first, in its CPU and its latency. (Memory - /// is free here: the longer evaluation would hold its memory longer.) - #[test] - fn each_request_pays_the_per_evaluation_cost() { - let (candidate, _, _) = kll_build(); - let free_memory = unit_costs(|c| c.cost_per_memory_byte_second = 0.0); - let before = price(&candidate.dag, &recurrences(1), &data(), &free_memory).unwrap(); - let settings = CostInputs { - measured_operation_costs: MeasuredOperationCosts { - ns_per_evaluation: 1e6, - ..MeasuredOperationCosts::default() - }, - ..free_memory - }; - let after = price(&candidate.dag, &recurrences(1), &data(), &settings).unwrap(); - // 1 ms per request, a request every 10 s. - assert!(close( - after.usage.latency_per_query[0].query_ms - before.usage.latency_per_query[0].query_ms, - 1.0 - )); - let core_seconds = |cost: &PlanCost| { - cost.per_node - .values() - .map(|n| n.cpu_core_seconds_per_second) - .sum::() - }; - assert!(close( - core_seconds(&after) - core_seconds(&before), - 1e-3 / 10.0 - )); - assert!(close( - after.cost_per_second.unwrap() - before.cost_per_second.unwrap(), - 0.1 - )); - } - - // A literal query has no operator to own its fee, but still incurs one - // request's CPU, latency and price; range steps do not multiply that fee. - #[test] - fn literal_requests_pay_overhead_without_operator_nodes() { - use asap_types::workload::{DemandEstimate, Rate}; - let roots = [QueryRoot::Scalar(ScalarExpr::Literal( - ScalarValue::Float64(1.0), - ))]; - let recurrence = QueryRecurrence::Repeated(RepeatedDemand::EstimatedRate(DemandEstimate { - expected_rate: Rate(2000.0), - observed_at: None, - valid_for: None, - })); - let entries = [entry_with(AccuracyTarget::exact(), recurrence.clone())]; - let candidate = optimize_physical_asap("literal", &roots, &entries, &data(), false) - .unwrap() - .remove(0); - assert!(candidate.dag.nodes.is_empty()); - let mut settings = CostInputs::default(); - settings.measured_operation_costs.ns_per_evaluation = 2_000_000.0; - for range in [ - None, - Some(RangeEvaluation { - step_ms: 1000, - span_ms: 3000, - }), - ] { - let cost = self_calibrated_cost( - &candidate.dag, - &node_map(&candidate.dag), - &kept_tumbling_window_chains(&candidate.dag), - std::slice::from_ref(&recurrence), - &[range], - &data(), - &settings, - ) - .unwrap(); - let CostModel::UsageOverTime(prices) = &settings.cost_model else { - unreachable!() - }; - assert!(close( - cost.usage.cpu_ops_per_second, - 2.0 / settings.resource_speed.ms_per_cpu_op * 2000.0 - )); - assert!(close( - cost.cost_per_second.unwrap(), - prices.cost_per_cpu_op * cost.usage.cpu_ops_per_second - )); - assert!(cost.cost_per_second.unwrap() > 0.0); - assert_eq!(cost.usage.latency_per_query[0].query_ms, 2.0); - assert_eq!(cost.usage.peak_cpu_cores, 4); - assert!(cost.per_node.is_empty()); - assert_eq!(cost.source, MEASURED_COST_SOURCE); - } - // Cluster capacity must enforce overhead's throughput requirement too. - settings.cost_model = CostModel::ClusterCapacity(ClusterLimits { - memory_bytes: u64::MAX, - disk_bytes: u64::MAX, - cpu_cores: Some(3), - objective: ClusterObjective::LowestLatency, - }); - let facts = data(); - let models = SelectionModels { - cost_inputs: &settings, - ..SelectionModels::builtin() - }; - let mut selector = PlanSelector::new(&entries, &facts, models, false).unwrap(); - selector.set_calibration(CalibrationFit::default()); - selector.offer(&candidate).unwrap(); - assert!(matches!( - selector.finish(), - Err(ErrorSelectingPlan::NoValidCandidate(_)) - )); - // Finite input coefficients can still overflow when multiplied by demand. - settings.measured_operation_costs.ns_per_evaluation = f64::MAX; - assert!(matches!( - price(&candidate.dag, &[recurrence], &facts, &settings), - Err((_, AnalyticalCostError::Overflow)) - )); - } - - /// A measured `memory_bytes_per_state` sizes the state a build keeps, - /// in place of the bytes the formulas derive from its parameters. - #[test] - fn measured_state_bytes_size_the_kept_state() { - let candidates = tumbling_window_candidates(10_000.0); - let maintained = &candidates[1]; - let kll = maintained - .dag - .nodes - .iter() - .find_map(|n| match &n.payload { - Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { - family: FieldDataType::Sketch(kind, _), - .. - }) => Some(kind.params().clone()), - _ => None, - }) - .unwrap(); - let memory = |memory_bytes_per_state| { - let settings = measured(vec![OperatorCost { - memory_bytes_per_state, - ..sketch_entry(OperatorKind::SummaryTumblingWindowChain, kll.clone(), 1.0) - }]); - price( - &maintained.dag, - &[repeating(MINUTE)], - &arriving(10_000.0), - &settings, - ) - .unwrap() - .usage - .average_memory_bytes - }; - let formula = memory(None); - // The formula's state: each tumbling window sees its series' 60 samples (one - // minute of 10 000 rows per second over 10 000 series). - let samples = StateInputRows { - rows_per_state: Some(60), - non_negative: false, - }; - let half = memory(Some(sketch_shape(&kll, samples).1 as f64 / 2.0)); - assert!(half < formula && half > 0.0, "{half} vs {formula}"); - } - - // Oversized state configurations must be rejected as unrepresentable, - // never wrap to a cheap candidate or panic while pricing tumbling window choices. - #[test] - fn tumbling_window_state_byte_overflow_returns_a_cost_error() { - for candidate in tumbling_window_candidates(10_000.0) { - let kll = candidate - .dag - .nodes - .iter() - .find_map(|node| match &node.payload { - Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { - family: FieldDataType::Sketch(kind, _), - .. - }) => Some(kind.params().clone()), - _ => None, - }) - .unwrap(); - let settings = measured(vec![OperatorCost { - memory_bytes_per_state: Some(u64::MAX as f64), - ..sketch_entry(OperatorKind::SummaryTumblingWindowChain, kll, 1.0) - }]); - assert!(matches!( - price( - &candidate.dag, - &[repeating(MINUTE)], - &arriving(10_000.0), - &settings - ), - Err((_, AnalyticalCostError::Overflow)) - )); - } - } - - /// The reference pricer rejects a candidate it cannot price, as the - /// search does, rather than failing the whole selection. - #[test] - fn the_reference_pricer_rejects_an_unpriceable_candidate() { - let candidates = tumbling_window_candidates(10_000.0); - let kll = candidates[0] - .dag - .nodes - .iter() - .find_map(|node| match &node.payload { - Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { - family: FieldDataType::Sketch(kind, _), - .. - }) => Some(kind.params().clone()), - _ => None, - }) - .unwrap(); - let settings = measured(vec![OperatorCost { - memory_bytes_per_state: Some(u64::MAX as f64), - ..sketch_entry(OperatorKind::SummaryTumblingWindowChain, kll, 1.0) - }]); - let entries = [entry_with(p99_target(), repeating(MINUTE))]; - let models = SelectionModels { - cost_inputs: &settings, - ..SelectionModels::builtin() - }; - let Err(ErrorSelectingPlan::NoValidCandidate(rejected)) = - select_among_plans(&candidates, &entries, &arriving(10_000.0), models) - else { - panic!("every candidate overflows, so none is valid"); - }; - assert_eq!(rejected.len(), candidates.len()); - assert!(rejected - .iter() - .all(|r| r.cause == RejectionCause::CostNotRepresentable)); - } - - /// The table of earlier versions, a JSON array of sketch operations, - /// still loads: each entry becomes its operator's entry at both phases, - /// an update both a `SummaryAgg` and a `SummaryTumblingWindowChain` entry. - #[test] - fn the_earlier_table_format_still_loads() { - let settings: CostInputs = serde_json::from_str( - r#"{"measured_operation_costs": [ - {"sketch": {"Kll": {"k": 200}}, "operation": "update", "cpu_ns_per_operation": 120.0}, - {"sketch": {"Kll": {"k": 200}}, "operation": "merge", "cpu_ns_per_operation": 900.0}]}"#, - ) - .unwrap(); - let kll = SketchParams::Kll { - k: 200, - failure_probability: None, - }; - assert_eq!( - settings.measured_operation_costs.operators, - vec![ - sketch_entry(OperatorKind::SummaryAgg, kll.clone(), 120.0), - sketch_entry(OperatorKind::SummaryTumblingWindowChain, kll.clone(), 120.0), - sketch_entry(OperatorKind::SummaryMerge, kll, 900.0), - ] - ); - let table: MeasuredOperationCosts = serde_json::from_str( - r#"{"operators": [{"operator": "Scan", "phase": "query_time", "ns_per_unit": 150.0}], - "ns_per_evaluation": 20000.0, - "provenance": {"executor_version": "0.1.0", "machine": "x86_64, 16 cores"}}"#, - ) - .unwrap(); - assert_eq!( - table.operators, - [plain_entry( - OperatorKind::Scan, - Some(ExecutionTiming::QueryTime), - 150.0 - )] - ); - let round_trip: MeasuredOperationCosts = - serde_json::from_str(&serde_json::to_string(&table).unwrap()).unwrap(); - assert_eq!(round_trip, table); - assert!(serde_json::from_str::( - r#"{"operators": [{"operator": "Scan", "ns": 1.0}]}"# - ) - .is_err()); - } - - /// With no entry for any node's operator and family (a CMS that no - /// node builds, a KLL merge in a plan without merges), every node is - /// priced from the formulas, as without a table. - #[test] - fn entries_for_other_operators_or_families_change_nothing() { - let (candidate, _, sketch) = kll_build(); - let before = price( - &candidate.dag, - &recurrences(1), - &data(), - &CostInputs::default(), - ) - .unwrap(); - let other = measured(vec![ - sketch_entry( - OperatorKind::SummaryAgg, - SketchParams::Cms { - width: 1024, - depth: 3, - }, - 50.0, - ), - sketch_entry(OperatorKind::SummaryMerge, sketch, 50.0), - ]); - let after = price(&candidate.dag, &recurrences(1), &data(), &other).unwrap(); - assert_eq!(after, before); - } - - /// Bad input: a negative or non-finite time, an entry naming two - /// families, two entries for one operator, phase and family, or a - /// table with `ms_per_cpu_op` 0. - #[test] - fn invalid_measured_tables_are_rejected() { - let cms = SketchParams::Cms { - width: 1024, - depth: 3, - }; - let entry = |ns| sketch_entry(OperatorKind::SummaryAgg, cms.clone(), ns); - let validate = |operators, ms_per_cpu_op| { - let mut settings = measured(operators); - settings.resource_speed.ms_per_cpu_op = ms_per_cpu_op; - settings.validate() - }; - for ns in [-1.0, f64::NAN] { - assert!(matches!( - validate(vec![entry(ns)], 1e-6), - Err(AnalyticalCostError::InvalidCalibration(..)) - )); - } - let duplicate = validate(vec![entry(1.0), entry(2.0)], 1e-6).unwrap_err(); - assert!( - matches!(duplicate, AnalyticalCostError::DuplicateOperatorCost(_)), - "{duplicate}" - ); - assert!( - duplicate - .to_string() - .contains("SummaryAgg at both phases on Cms"), - "{duplicate}" - ); - let two_families = OperatorCost { - exact: Some(asap_types::ir::schema::ExactKind::Sum), - ..entry(1.0) - }; - assert!(matches!( - validate(vec![two_families], 1e-6), - Err(AnalyticalCostError::AmbiguousOperatorCost(_)) - )); - assert_eq!( - validate(vec![entry(1.0)], 0.0), - Err(AnalyticalCostError::MeasuredCostsWithoutCpuSpeed(0.0)) - ); - let mut empty = CostInputs::default(); - empty.resource_speed.ms_per_cpu_op = 0.0; - assert!( - empty.validate().is_ok(), - "no table, no positive speed needed" - ); - } -} diff --git a/crates/plan-selection/src/milp/completion.rs b/crates/plan-selection/src/milp/completion.rs deleted file mode 100644 index 86bcc1f44..000000000 --- a/crates/plan-selection/src/milp/completion.rs +++ /dev/null @@ -1,306 +0,0 @@ -//! The cost of a plan given as values of the MILP's binaries: the cheapest -//! values of the continuous variables, found without a solver. -//! -//! Every continuous variable of the model is one of: -//! - a lower bound that minimization makes tight: `v ≥ f(others)` over one or -//! more rows (reach `a`, latency `L`, held time `G`/`H`/`W`, AND variables, -//! rate components, raw-retention levels, …). Its cheapest value is the -//! largest right-hand side, or 0; -//! - an equality `N = Σ a` (approximate values read); -//! - free below its upper rows only (`t`, reach through): its value is the -//! smallest upper bound, which leaves the most room to the rows it -//! relaxes. -//! -//! A row defines the continuous variable it bounds from below: the one with a -//! negative coefficient in `expression ≤ 0` (or any nonzero one in an -//! equality) that was created last, since the model creates each variable -//! after the variables its defining rows read. Rows of the families in -//! [`CHECK_ONLY_FAMILIES`] (upper bounds of reach, reader constraints, and -//! valid inequalities a plan's values meet anyway) define nothing. The -//! values are computed in creation order, repeated until no value changes, -//! then every row and bound is checked. A plan that fails a check is -//! infeasible as far as this completion goes (HiGHS's LP completion could -//! find another one; the warm start then just skips that plan). -//! -//! Each value is the smallest its defining rows allow, so at a plan's -//! binaries the values are the plan's own reach, latencies, held times and -//! rates, and the objective is what the model charges the plan: with the -//! exact cost terms of [`super::cost_terms`], what `asap_cost::plan_cost` -//! says (the warm start's test checks it). -use std::collections::HashMap; - -use good_lp::{IntoAffineExpression, Variable}; - -use super::model::SelectionMilp; - -/// Families whose rows only check values (upper bounds of reach, reader -/// constraints, rows tests add) and define no variable. -const CHECK_ONLY_FAMILIES: [&str; 4] = [ - "6 reach from above", - "9 reader constraints", - "V (slowest rate)", - "test", -]; - -/// The most passes over the variables before giving up on a fixpoint. -const MOST_PASSES: usize = 64; - -/// One row in `Σ coefficient · column + constant ≤ 0` (or `= 0`). -struct Row { - /// Range into `columns` and `coefficients`. - start: usize, - end: usize, - constant: f64, - equality: bool, -} - -/// The model's rows, compiled for repeated completions. -pub(crate) struct Completion { - column_of: HashMap, - /// The variables, by column. - variables: Vec, - binary: Vec, - max: Vec, - rows: Vec, - columns: Vec, - coefficients: Vec, - /// Per column, the rows defining it. - defining_rows: Vec>, - /// Per column with no defining row, the rows bounding it from above. - upper_rows: Vec>, - /// The columns with defining rows, in creation order. - defined: Vec, - /// The columns with neither (free continuous variables), in creation - /// order. - free: Vec, - objective: Vec<(usize, f64)>, - /// The family of each row. - families: Vec<&'static str>, -} - -/// A completed plan. -pub(crate) struct Completed { - /// Every variable's value, in creation order. - pub(crate) columns: Vec, - /// The objective (cost per second without the request fees). - pub(crate) cost_per_second: f64, -} - -impl Completion { - pub(crate) fn new(milp: &SelectionMilp<'_>) -> Self { - let builder = &milp.builder; - let mut column_of = HashMap::new(); - let mut variables = Vec::new(); - let mut binary = Vec::new(); - let mut max = Vec::new(); - for (column, (variable, definition)) in - builder.variables.iter_variables_with_def().enumerate() - { - column_of.insert(variable, column); - variables.push(variable); - binary.push(definition.is_integer()); - max.push(definition.get_max()); - } - let count = binary.len(); - let mut rows = Vec::with_capacity(builder.rows.len()); - let mut columns = Vec::new(); - let mut coefficients = Vec::new(); - let mut defining_rows: Vec> = vec![Vec::new(); count]; - let mut positive_rows: Vec> = vec![Vec::new(); count]; - for (index, (row, family)) in builder.rows.iter().zip(&builder.row_families).enumerate() { - let expression = row.expression(); - let start = columns.len(); - let mut defines: Option = None; - for (variable, coefficient) in expression.linear_coefficients() { - if coefficient == 0.0 { - continue; - } - let column = column_of[&variable]; - columns.push(column as u32); - coefficients.push(coefficient); - if binary[column] { - continue; - } - if coefficient > 0.0 { - positive_rows[column].push(index as u32); - } - if (coefficient < 0.0 || row.is_equality()) - && defines.is_none_or(|known| column > known) - { - defines = Some(column); - } - } - if !CHECK_ONLY_FAMILIES.contains(family) { - if let Some(column) = defines { - defining_rows[column].push(index as u32); - } - } - rows.push(Row { - start, - end: columns.len(), - constant: expression.constant(), - equality: row.is_equality(), - }); - } - let mut defined = Vec::new(); - let mut free = Vec::new(); - let mut upper_rows: Vec> = vec![Vec::new(); count]; - for column in 0..count { - if binary[column] { - continue; - } - if defining_rows[column].is_empty() { - upper_rows[column] = std::mem::take(&mut positive_rows[column]); - free.push(column); - } else { - defined.push(column); - } - } - let objective = milp - .objective_terms - .iter() - .map(|&(variable, coefficient)| (column_of[&variable], coefficient)) - .collect(); - Self { - column_of, - variables, - binary, - max, - rows, - columns, - coefficients, - defining_rows, - upper_rows, - defined, - free, - objective, - families: builder.row_families.clone(), - } - } - - /// The completion of a solver's columns (in creation order): its - /// binaries rounded, every continuous variable recomputed. - pub(crate) fn complete_columns(&self, columns: &[f64]) -> Result { - let binaries: Vec<(Variable, f64)> = self - .variables - .iter() - .zip(columns) - .zip(&self.binary) - .filter(|(_, &binary)| binary) - .map(|((&variable, &value), _)| (variable, if value > 0.5 { 1.0 } else { 0.0 })) - .collect(); - self.complete(&binaries) - } - - /// Every variable's value in `completed`. - pub(crate) fn values(&self, completed: &Completed) -> Vec<(Variable, f64)> { - self.variables - .iter() - .copied() - .zip(completed.columns.iter().copied()) - .collect() - } - - /// `Σ coefficient · value + constant` of row `row`, without column - /// `skip`, and the coefficient of `skip`. - fn row_value(&self, row: &Row, values: &[f64], skip: usize) -> (f64, f64) { - let mut sum = row.constant; - let mut own = 0.0; - for at in row.start..row.end { - let column = self.columns[at] as usize; - if column == skip { - own += self.coefficients[at]; - } else { - sum += self.coefficients[at] * values[column]; - } - } - (sum, own) - } - - /// The cheapest completion of `binaries` (every binary not listed is 0), - /// or why it has none here: the row or bound it breaks. - pub(crate) fn complete(&self, binaries: &[(Variable, f64)]) -> Result { - let mut values = vec![0.0; self.binary.len()]; - for &(variable, value) in binaries { - let column = self.column_of[&variable]; - if self.binary[column] { - values[column] = value; - } - } - // Defined columns: the largest lower bound, to a fixpoint. - let mut converged = false; - for _ in 0..MOST_PASSES { - let mut changed = false; - for &column in &self.defined { - let mut value: f64 = 0.0; - for &row in &self.defining_rows[column] { - let row = &self.rows[row as usize]; - let (rest, own) = self.row_value(row, &values, column); - // own · v + rest ≤ 0 (or = 0) with own < 0 for an inequality. - let bound = -rest / own; - if row.equality { - value = bound; - break; - } - value = value.max(bound); - } - if (value - values[column]).abs() > 1e-12 * value.abs().max(1.0) { - changed = true; - } - values[column] = value; - } - if !changed { - converged = true; - break; - } - } - if !converged { - return Err("no fixpoint of the lower bounds".into()); - } - // Free columns: the smallest upper bound. - for &column in &self.free { - let mut value = self.max[column]; - for &row in &self.upper_rows[column] { - let row = &self.rows[row as usize]; - let (rest, own) = self.row_value(row, &values, column); - value = value.min(-rest / own); - } - values[column] = if value.is_finite() { - value.max(0.0) - } else { - 0.0 - }; - } - // Every row and bound. - for (index, row) in self.rows.iter().enumerate() { - let mut sum = row.constant; - let mut magnitude = row.constant.abs(); - for at in row.start..row.end { - let term = self.coefficients[at] * values[self.columns[at] as usize]; - sum += term; - magnitude += term.abs(); - } - let tolerance = 1e-7 * magnitude.max(1.0); - if sum > tolerance || (row.equality && sum < -tolerance) { - return Err(format!( - "row {index} ({}) off by {sum:e}", - self.families[index] - )); - } - } - for (column, &value) in values.iter().enumerate() { - if value > self.max[column] + 1e-7 * self.max[column].abs().max(1.0) { - return Err(format!("column {column} at {value:e} above its bound")); - } - } - let cost_per_second = self - .objective - .iter() - .map(|&(column, coefficient)| coefficient * values[column]) - .sum(); - Ok(Completed { - columns: values, - cost_per_second, - }) - } -} diff --git a/crates/plan-selection/src/milp/cost_terms.rs b/crates/plan-selection/src/milp/cost_terms.rs deleted file mode 100644 index 7814f7b3c..000000000 --- a/crates/plan-selection/src/milp/cost_terms.rs +++ /dev/null @@ -1,1204 +0,0 @@ -//! #1095 §3: the cost model 1 objective of the group selection MILP. Every -//! number is `asap-cost`'s: a candidate's usage per execution and standing -//! cost ([`CandidateUsage`], from [`asap_cost::node_usage`] in -//! [`crate::groups`]), the schedule groups -//! ([`asap_cost::execution_demand::schedule_groups`]) and rates -//! ([`asap_cost::execution_demand::evaluation_rate`]) and the unit costs of -//! [`asap_cost::ResourceUnitCost`]. This module only turns them into -//! coefficients and the auxiliary rows that linearize them. -//! -//! A **run** is a candidate computed in an execution context (`p[ctx, c]`), -//! or a shareable candidate's own run (`x[m]`, keyed by its own context). -//! A run's usage is one or more [`RunTerm`]s, each an indicator and the -//! numbers that count when it is 1: -//! - one term on the run's own indicator when its usage is plan -//! independent; -//! - per producer choice, when its usage depends on which candidate of an -//! input's slot the context picks (a summary state's configuration, a -//! scan's window): `AND(run, p[ctx, c'], producer gives that output)`; -//! - for a query-time scan read at several windows in one context: one -//! term per set `S` of window kinds read, `δ_S`, the exact indicator that -//! exactly the kinds in `S` are read (from `σ_W`, the exact OR of the -//! reads at kind `W`), costed at the union of `S`'s windows -//! (`PhysicalASAPDAG::scan_window`). This needs no monotonicity: every -//! union is costed by `asap-cost` itself. -//! -//! **Executions per second** of a run: `rate(q)` in query context `q`; 1 in -//! each `I_r` and for an ingestion-time shareable candidate; for a query-time -//! shareable candidate `m` and the runs in its context, `Σ rate · z[m, G, -//! ℓ]` over its schedule groups and their levels, `z ≥ a[q, m]` for each -//! reader at that level or faster (a group's rate is its fastest reader's). -//! A context's runs other than `m`'s plain own term are summed once, `E[m] -//! ≥ Σ cost · indicator`, and charged per component, `V ≥ E[m] − Ē[m] (x[m] -//! − z)`, with `Σ rate · V ≥ (slowest rate) · E[m]`. -//! -//! **Held memory**: working memory of a run in query context `q` for `q`'s -//! latency at `q`'s rate (`H_query ≥ L[q] − L̄[q] (1 − indicator)`); `m`'s -//! `MatAtQuery(Memory)` result and the working memory of the runs in its -//! context until `m`'s slowest reader finishes (`H[m] ≥ L[q] − L̄[q] (1 − -//! a[q, m])`, then `W ≥ H[m] − H̄[m] (slack − Σ indicators)` per rate -//! component). -//! -//! **Raw retention**: per source, levels at each raw-data size a query-time -//! scan run may keep, `v[src, ℓ] ≥` each scan term keeping at least that -//! much; charged `Δbytes · unit disk cost`. -//! -//! **Pane retention**: a kept pane chain's usage at the fewest panes any -//! reader needs on `x[m]`, and each larger retention's increase on `π[m, -//! ℓ] ≥` the reach (as `plan_cost` counts it: through query-time runs) of -//! each query needing at least that many panes. -//! -//! Terms `plan_cost` defines differently from the above are counted in -//! [`SelectionMilp::inexact`] by reason. -use std::collections::{BTreeMap, BTreeSet, HashMap}; - -use asap_cost::execution_demand::{evaluation_rate, schedule_groups}; -use asap_cost::CostModel; -use asap_types::ir::physical_export::ScanWindow; -use asap_types::ir::properties::ExecutionTiming; -use asap_types::ir::{NonASAPOp, Operator}; -use asap_types::physical::{Materialization, Medium}; -use asap_types::stages::PhysicalCandidateId; -use asap_types::workload::bounds::source_name; -use asap_types::workload::{QueryRecurrence, RepeatedDemand}; -use good_lp::{constraint, Expression, IntoAffineExpression, Variable}; - -use crate::groups::{ - CandidateUsage, ContextKind, ResultSlot, RunKind, UsageAndOutput, UsageContext, -}; - -use super::model::{Reader, Scope, SelectionMilp, SelectionMilpError}; - -/// The numbers of one run `asap-cost` computed, as the objective and the -/// latency rows read them. -#[derive(Debug, Clone, Copy, Default, PartialEq)] -pub(crate) struct NodeNumbers { - /// Cost of one execution. - pub(crate) execution_cost: f64, - /// Cost per second of what it keeps whatever runs it. - pub(crate) standing_cost: f64, - /// What it adds to a reaching query's latency, ms. - pub(crate) latency_ms: f64, - /// Query-time memory held while an evaluation reading it runs. - pub(crate) held_bytes: f64, - /// Raw data a query-time scan keeps on disk. - pub(crate) raw_kept_bytes: f64, -} - -impl NodeNumbers { - pub(crate) fn of(usage: &CandidateUsage) -> Self { - Self { - execution_cost: usage.cost_per_execution.unwrap_or(0.0), - standing_cost: usage.standing_cost_per_second.unwrap_or(0.0), - latency_ms: usage.own_latency_ms, - held_bytes: usage.held_bytes as f64, - raw_kept_bytes: usage.raw_data_kept_bytes as f64, - } - } - - /// `self − other`, each number at least 0, and whether one was below. - fn minus(self, other: Self) -> (Self, bool) { - let fields = [ - self.execution_cost - other.execution_cost, - self.standing_cost - other.standing_cost, - self.latency_ms - other.latency_ms, - self.held_bytes - other.held_bytes, - self.raw_kept_bytes - other.raw_kept_bytes, - ]; - let negative = fields - .iter() - .any(|&field| field < -1e-12 * field.abs().max(1.0)); - let [execution_cost, standing_cost, latency_ms, held_bytes, raw_kept_bytes] = - fields.map(|field| field.max(0.0)); - ( - Self { - execution_cost, - standing_cost, - latency_ms, - held_bytes, - raw_kept_bytes, - }, - negative, - ) - } -} - -/// One term of a run: its numbers count, and it gives its readers output -/// `output`, when `indicator` is 1. At integral binaries exactly one term -/// of a used run is 1, except a kept pane chain's retention increases, -/// which add to its base term. -#[derive(Debug, Clone, PartialEq)] -pub(crate) struct RunTerm { - pub(crate) indicator: Expression, - pub(crate) numbers: NodeNumbers, - /// Whether `indicator` is the run's own variable (`p[ctx, c]` or - /// `x[m]`). - pub(crate) plain: bool, - /// Its output (an index below `CandidateData::output_count`). - pub(crate) output: usize, -} - -/// Run `(context, candidate)`'s terms (module documentation). Empty when -/// the run does not exist. -pub(crate) fn run_terms( - milp: &mut SelectionMilp<'_>, - context: usize, - candidate: PhysicalCandidateId, -) -> Vec { - let Some(run) = milp.run_indicator(context, candidate) else { - return Vec::new(); - }; - let data = &milp.inputs.candidate_data[candidate.0]; - let Ok(base) = &data.usage else { - return Vec::new(); - }; - let base = *base; - for usage_context in data.usage_depends_on.clone() { - match usage_context { - UsageContext::ProducerOnACycle => milp.note_inexact("usage: a producer on a cycle"), - UsageContext::ProducerOutput - | UsageContext::ScanWindowOfReader - | UsageContext::PaneRetentionOfReachingQueries - | UsageContext::UncostableContext => {} - } - } - if milp.inputs.candidate_data[candidate.0].producer_usage_ambiguous { - milp.note_inexact("usage: a producer's output depends on its own producers"); - } - if !milp.inputs.candidate_data[candidate.0] - .usage_per_window - .is_empty() - { - if let Some(terms) = window_terms(milp, context, candidate, run) { - return terms; - } - } - if !milp.inputs.candidate_data[candidate.0] - .inputs_changing_usage - .is_empty() - { - return producer_terms(milp, context, candidate, run); - } - let mut terms = vec![RunTerm { - indicator: run.into(), - numbers: NodeNumbers::of(&base), - plain: true, - output: 0, - }]; - if !milp.inputs.candidate_data[candidate.0] - .usage_per_retained_panes - .is_empty() - { - pane_retention_terms(milp, context, candidate, &mut terms); - } - terms -} - -/// The indicator that, in `context`, the producer `producer` of a reader is -/// read and gives one of `outputs` (`None`: any output): `p[ctx, c']`, or -/// `AND(p[ctx, c'], t)` with `t ≥` the producer run's terms giving them. -/// `None` when the producer never gives them. -fn producer_gives( - milp: &mut SelectionMilp<'_>, - context: usize, - producer: PhysicalCandidateId, - outputs: Option<&BTreeSet>, -) -> Option { - let read = *milp.choice[context].get(&producer)?; - let Some(outputs) = outputs else { - return Some(read.into()); - }; - // The producer's run: computed here, its own run, or its run in an - // `I_r`. - let producer_run = match milp.kind(producer) { - _ if milp.computes(context, producer) => context, - RunKind::Shareable => milp.context_of(producer)?, - RunKind::Ingestion => milp.ingestion_context_read_by(context)?, - RunKind::PerContext => return None, - }; - if milp.terms_in_progress.contains(&(producer_run, producer)) { - milp.note_inexact("usage: a producer's output on a cycle of links (every output kept)"); - return Some(read.into()); - } - let producer_terms = milp.run_terms(producer_run, producer); - if producer_terms - .iter() - .all(|term| outputs.contains(&term.output)) - { - return Some(read.into()); - } - let gives: Vec<&RunTerm> = producer_terms - .iter() - .filter(|term| outputs.contains(&term.output)) - .collect(); - if gives.is_empty() { - return None; - } - let key = ( - producer_run, - producer, - outputs.iter().copied().collect::>(), - ); - let gives = match milp.producer_gives.get(&key) { - Some(&gives) => gives, - None => { - let sum: Expression = gives.iter().map(|term| term.indicator.clone()).sum(); - let variable = milp - .builder - .continuous("t_outputs (a producer gives outputs)", Some(1.0)); - milp.builder.row( - "t_outputs (a producer gives outputs)", - constraint!(variable >= sum), - ); - milp.producer_gives.insert(key, variable); - variable - } - }; - if milp.computes(context, producer) { - return Some(gives.into()); - } - Some( - milp.builder - .and_at_least("t_producers (AND)", &[read.into(), gives.into()]) - .into(), - ) -} - -/// A producer choice with the outputs it stands for (`None`: any output). -type OutputsOfProducer = (PhysicalCandidateId, Option>); - -/// Terms of a run whose usage or output depends on its producers (and -/// their outputs): one per choice, on `AND(run, producer gives)` over the -/// changing inputs. -fn producer_terms( - milp: &mut SelectionMilp<'_>, - context: usize, - candidate: PhysicalCandidateId, - run: Variable, -) -> Vec { - milp.terms_in_progress.insert((context, candidate)); - let data = &milp.inputs.candidate_data[candidate.0]; - let changing = data.inputs_changing_usage.clone(); - let base = *data.usage.as_ref().expect("costed"); - let same = |known: &Option, gives: &Option| match (known, gives) - { - (Some(known), Some(gives)) => known == gives, - (None, None) => true, - _ => false, - }; - let mut table: Vec<(Vec, Option)> = Vec::new(); - for (choices, gives) in &data.usage_per_producers { - let choices: Vec = choices - .iter() - .map(|choice| { - ( - choice.class, - choice.output.map(|output| BTreeSet::from([output])), - ) - }) - .collect(); - if changing.len() == 1 { - if let Some((known, _)) = table - .iter_mut() - .find(|(known, known_gives)| known[0].0 == choices[0].0 && same(known_gives, gives)) - { - if let (Some(outputs), Some(more)) = (&mut known[0].1, &choices[0].1) { - outputs.extend(more); - } - continue; - } - } - table.push((choices, *gives)); - } - let producers_here: Vec> = changing - .iter() - .map(|&input| { - milp.inputs.producer_candidates_per_input[candidate.0][input] - .iter() - .copied() - .filter(|producer| milp.choice[context].contains_key(producer)) - .filter(|producer| milp.slot(*producer) != milp.slot(candidate)) - .collect() - }) - .collect(); - let mut covered: Vec> = vec![BTreeSet::new(); changing.len()]; - let mut terms = Vec::new(); - for (choices, gives) in &table { - let mut per_input: Vec = Vec::with_capacity(choices.len()); - for (position, (producer, outputs)) in choices.iter().enumerate() { - if !producers_here[position].contains(producer) { - break; - } - covered[position].insert(*producer); - match producer_gives(milp, context, *producer, outputs.as_ref()) { - Some(indicator) => per_input.push(indicator), - None => break, - } - } - if per_input.len() < choices.len() { - continue; - } - let mut all: Vec = vec![run.into()]; - all.extend(per_input); - // A multi-slot input reads the choice's slot only through its link. - for (position, (producer, _)) in choices.iter().enumerate() { - if let Some(links) = milp - .slot_links - .get(&(context, candidate, changing[position])) - { - let slot = milp.slot(*producer); - if let Some(&(_, link)) = links.iter().find(|(at, _)| *at == slot) { - all.push(link.into()); - } - } - } - let count = all.len() as f64; - match gives { - None => { - let sum: Expression = all.into_iter().sum(); - milp.builder - .row("t_producers (AND)", constraint!(sum <= count - 1.0)); - } - Some((usage, output)) => { - let indicator = milp.builder.and_at_least("t_producers (AND)", &all); - terms.push(RunTerm { - indicator: indicator.into(), - numbers: NodeNumbers::of(usage), - plain: false, - output: *output, - }); - } - } - } - // A producer missing from the table: its reads carry the first costed - // usage. - let mut missing = false; - for (position, producers) in producers_here.iter().enumerate() { - for &producer in producers { - if covered[position].contains(&producer) { - continue; - } - missing = true; - let read = milp.choice[context][&producer]; - let indicator = milp - .builder - .and_at_least("t_producers (AND)", &[run.into(), read.into()]); - terms.push(RunTerm { - indicator: indicator.into(), - numbers: NodeNumbers::of(&base), - plain: false, - output: 0, - }); - } - } - if missing { - milp.note_inexact("usage: a producer missing from its usage table"); - } - if terms.is_empty() { - milp.builder - .row("t_producers (AND)", constraint!(run <= 0.0)); - } - milp.terms_in_progress.remove(&(context, candidate)); - terms -} - -/// The union of `windows`, as [`PhysicalASAPDAG::scan_window`] computes it -/// for one scan read at each: the nearest end and the farthest start. -pub(crate) fn window_union(windows: impl IntoIterator) -> ScanWindow { - let mut near = i64::MAX; - let mut far = i64::MIN; - for window in windows { - match window { - ScanWindow::Unbounded => return ScanWindow::Unbounded, - ScanWindow::Within { - end_offset_ms, - width_ms, - } => { - near = near.min(end_offset_ms); - far = far.max(end_offset_ms.saturating_add(width_ms)); - } - } - } - if near > far { - return ScanWindow::Unbounded; - } - ScanWindow::Within { - end_offset_ms: near, - width_ms: far.saturating_sub(near), - } -} - -/// The window a scan run read at `kinds` scans, as `asap_cost::plan_cost` -/// prices it: their union ([`window_union`]) when every kind is bounded; -/// with an unbounded read (a reader that is not a time range, a pane chain -/// or a time shift, or a read as an output), the trailing window of the -/// widest bounded kind (`plan_cost`'s `widest_window_read_ms`), or -/// `Unbounded` when there is none. -pub(crate) fn scan_window_read_at( - kinds: impl IntoIterator + Clone, -) -> ScanWindow { - let unbounded = kinds - .clone() - .into_iter() - .any(|kind| kind == ScanWindow::Unbounded); - if !unbounded { - return window_union(kinds); - } - let widest = kinds - .into_iter() - .filter_map(|kind| match kind { - ScanWindow::Within { width_ms, .. } => Some(width_ms), - ScanWindow::Unbounded => None, - }) - .max(); - match widest { - Some(width_ms) => ScanWindow::Within { - end_offset_ms: 0, - width_ms, - }, - None => ScanWindow::Unbounded, - } -} - -/// The windows reader `reader` reads scan `scan` at (one per path through -/// time shifts); `Unbounded` when unknown. -fn reader_windows( - milp: &SelectionMilp<'_>, - scan: PhysicalCandidateId, - reader: PhysicalCandidateId, -) -> Vec { - milp.inputs.candidate_data[scan.0] - .windows_per_reader - .iter() - .find(|(known, _)| *known == reader) - .map(|(_, read)| read.clone()) - .unwrap_or_else(|| vec![ScanWindow::Unbounded]) -} - -/// Terms of query-time scan run `(context, scan)` whose usage depends on -/// the window its readers read (module documentation). `None` when its -/// readers read one window kind and it has no other dependence handled -/// here: the caller then uses its other terms. -fn window_terms( - milp: &mut SelectionMilp<'_>, - context: usize, - scan: PhysicalCandidateId, - run: Variable, -) -> Option> { - if let Some(cached) = milp.window_cache.get(&(context, scan)) { - return cached.clone(); - } - // Each read: (reading context, window kind, reader's indicator). - let mut reads: Vec<(usize, ScanWindow, Expression)> = Vec::new(); - let reading_contexts: Vec = if milp.kind(scan) == RunKind::Shareable { - (0..milp.choice.len()) - .filter(|&other| milp.choice[other].contains_key(&scan)) - .collect() - } else { - vec![context] - }; - for &reading in &reading_contexts { - for reader in milp.readers[reading] - .get(&scan) - .cloned() - .unwrap_or_default() - { - let (window, indicator) = match reader { - Reader::Candidate(reader, _) => { - let windows = reader_windows(milp, scan, reader); - if windows.len() > 1 { - milp.note_inexact("a scan read at several windows by one reader (union)"); - } - ( - window_union(windows), - Expression::from(milp.choice[reading][&reader]), - ) - } - Reader::Output(_) => (ScanWindow::Unbounded, Expression::from(1.0)), - Reader::OwnInput(_) => { - let ContextKind::Shared(own) = milp.inputs.contexts[reading].kind else { - continue; - }; - // `m` itself reads the scan: at its own window. - ( - window_union(reader_windows(milp, scan, own)), - Expression::from(milp.selected[&own]), - ) - } - }; - reads.push((reading, window, indicator)); - } - } - let mut kinds: Vec = Vec::new(); - for (_, window, _) in &reads { - if !kinds.contains(window) { - kinds.push(*window); - } - } - let usage_at = |milp: &SelectionMilp<'_>, window: ScanWindow| { - let found = milp.inputs.candidate_data[scan.0] - .usage_per_window - .iter() - .find(|(known, _, _)| *known == window) - .map(|&(_, usage, output)| (NodeNumbers::of(&usage), output)); - if found.is_none() { - super::warm_start::debug(|| { - format!( - "scan {} has no usage at {window:?}; costed at {:?}", - scan.0, - milp.inputs.candidate_data[scan.0] - .usage_per_window - .iter() - .map(|(w, _, _)| *w) - .collect::>() - ) - }); - } - found - }; - let terms = match kinds.len() { - 0 => None, - 1 => match usage_at(milp, scan_window_read_at([kinds[0]])) { - Some((numbers, output)) => Some(vec![RunTerm { - indicator: run.into(), - numbers, - plain: true, - output, - }]), - None => { - // `asap-cost` cannot cost the scan so: no plan reads it so. - milp.builder - .row("δ (scan window sets)", constraint!(run <= 0.0)); - Some(Vec::new()) - } - }, - count => { - // `σ_W`: the exact OR, over the reading contexts, of "the - // context reads the scan and a reader at kind `W` is used" - // (`w`, exact from both sides at integral binaries). - let mut sigma = Vec::with_capacity(count); - for kind in &kinds { - let mut per_context: Vec = Vec::new(); - for &reading in &reading_contexts { - let at_kind: Vec<&Expression> = reads - .iter() - .filter(|(at, window, _)| *at == reading && window == kind) - .map(|(_, _, indicator)| indicator) - .collect(); - if at_kind.is_empty() { - continue; - } - let read_here = milp.choice[reading][&scan]; - let used = milp.builder.continuous("σ (scan window kinds)", Some(1.0)); - let mut upper = Expression::from(0.0); - for indicator in at_kind { - let indicator = indicator.clone(); - upper += indicator.clone(); - milp.builder.row( - "σ (scan window kinds)", - constraint!(used >= indicator + read_here - 1.0), - ); - } - milp.builder - .row("σ (scan window kinds)", constraint!(used <= read_here)); - milp.builder - .row("σ (scan window kinds)", constraint!(used <= upper)); - per_context.push(used); - } - let or = match per_context.as_slice() { - [one] => *one, - many => { - let or = milp.builder.continuous("σ (scan window kinds)", Some(1.0)); - let mut upper = Expression::from(0.0); - for &used in many { - milp.builder - .row("σ (scan window kinds)", constraint!(or >= used)); - upper += used; - } - milp.builder - .row("σ (scan window kinds)", constraint!(or <= upper)); - or - } - }; - sigma.push(or); - } - let mut terms = Vec::new(); - assert!( - count < usize::BITS as usize, - "a scan run with {count} window kinds has too many unions to encode" - ); - for set in 1usize..(1 << count) { - let window = scan_window_read_at( - (0..count) - .filter(|bit| set & (1 << bit) != 0) - .map(|bit| kinds[bit]), - ); - let members = set.count_ones() as f64; - let mut sum = Expression::from(0.0); - for (bit, &or) in sigma.iter().enumerate() { - if set & (1 << bit) != 0 { - sum += or; - } else { - sum -= or; - } - } - let Some((numbers, output)) = usage_at(milp, window) else { - // `asap-cost` cannot cost the scan at this union: no - // plan reads exactly these kinds. - milp.builder - .row("δ (scan window sets)", constraint!(sum <= members - 1.0)); - continue; - }; - let exactly = milp.builder.continuous("δ (scan window sets)", Some(1.0)); - milp.builder.row( - "δ (scan window sets)", - constraint!(exactly >= sum - (members - 1.0)), - ); - terms.push(RunTerm { - indicator: exactly.into(), - numbers, - plain: false, - output, - }); - } - Some(terms) - } - }; - milp.window_cache.insert((context, scan), terms.clone()); - terms -} - -/// For a shareable kept pane chain with usage depending on the panes it -/// retains: its base term at the fewest panes, plus one term per larger -/// retention, `π[m, ℓ] ≥` the reach of each query needing at least `ℓ`. -fn pane_retention_terms( - milp: &mut SelectionMilp<'_>, - context: usize, - chain: PhysicalCandidateId, - terms: &mut Vec, -) { - let data = &milp.inputs.candidate_data[chain.0]; - let mut levels = data.usage_per_retained_panes.clone(); - levels.sort_by_key(|(panes, _)| *panes); - let per_query = data.retained_panes_per_query.clone(); - let Some(&(_, base)) = levels.first() else { - return; - }; - terms[0].numbers = NodeNumbers::of(&base); - let at_query_time = milp.runs_at_query_time(context); - // Per query, the indicator that it reaches the chain as `plan_cost` - // counts reach (through query-time runs). - let mut reached: Vec<(usize, Vec)> = Vec::new(); - for &(query, _) in &per_query { - let mut indicators = Vec::new(); - if let Some(&reach) = milp.reach.get(&(query, chain)) { - if at_query_time - || milp.physical.candidates[chain.0].timing == ExecutionTiming::QueryTime - { - indicators.push(Expression::from(reach)); - reached.push((query, indicators)); - continue; - } - } - if let Some(&direct) = milp.choice[query].get(&chain) { - indicators.push(direct.into()); - } - for &reader in milp - .shared_readers - .get(&chain) - .cloned() - .unwrap_or_default() - .iter() - { - let reader_context = milp.inputs.context_of_shareable[&reader]; - if !milp.runs_at_query_time(reader_context) { - continue; - } - if let Some(&above) = milp.reach.get(&(query, reader)) { - let read = milp.choice[reader_context][&chain]; - let and = milp - .builder - .and_at_least("π (pane retention levels)", &[above.into(), read.into()]); - indicators.push(and.into()); - } - } - reached.push((query, indicators)); - } - let mut below = NodeNumbers::of(&base); - for &(panes, usage) in &levels[1..] { - let numbers = NodeNumbers::of(&usage); - let (increase, negative) = numbers.minus(below); - below = numbers; - if negative { - milp.note_inexact("a longer pane retention with a smaller number"); - } - if increase.execution_cost > 0.0 && at_query_time { - milp.note_inexact("pane retention changing a query-time execution cost"); - } - if increase.latency_ms > 0.0 || increase.held_bytes > 0.0 { - milp.note_inexact("pane retention changing latency or held memory (not encoded)"); - } - let level = milp - .builder - .continuous("π (pane retention levels)", Some(1.0)); - for (query, indicators) in &reached { - let needs = per_query - .iter() - .any(|&(q, retained)| q == *query && retained >= panes); - if !needs { - continue; - } - for indicator in indicators { - let indicator = indicator.clone(); - milp.builder - .row("π (pane retention levels)", constraint!(level >= indicator)); - } - } - terms.push(RunTerm { - indicator: level.into(), - numbers: NodeNumbers { - latency_ms: 0.0, - held_bytes: 0.0, - ..increase - }, - plain: false, - output: terms[0].output, - }); - } -} - -/// Each ingestion-time shareable candidate's compaction term for its -/// readers' latency, ms, where `plan_cost` charges one: `(k − 1) · states · -/// merge time`, `k` the cores its ingestion work takes. With at most one -/// second of ingestion work per second below it, `k = 1` and the term is 0; -/// a larger subtree is not encoded (counted as inexact). -pub(crate) fn compaction_ms(milp: &mut SelectionMilp<'_>) -> HashMap { - let inputs = milp.inputs; - let mut compaction = HashMap::new(); - for &shareable in inputs.context_of_shareable.keys() { - if milp.physical.candidates[shareable.0].timing != ExecutionTiming::IngestionTime { - continue; - } - let mut seen = BTreeSet::new(); - let mut stack = vec![shareable]; - let mut most_work_ms = 0.0; - while let Some(class) = stack.pop() { - if !seen.insert(class) { - continue; - } - if let Ok(usage) = &inputs.candidate_data[class.0].usage { - most_work_ms += usage.work_ms_per_execution; - } - stack.extend( - inputs.producer_candidates_per_input[class.0] - .iter() - .flatten() - .copied(), - ); - } - if most_work_ms > 1_000.0 { - milp.note_inexact("compaction with more than one ingest worker (not encoded)"); - } - compaction.insert(shareable, 0.0); - } - compaction -} - -/// A schedule group's readers with their own rates, and its rate levels. -type GroupLevels = (Vec<(usize, f64)>, Vec); - -/// How the runs of a context execute per second. -#[derive(Debug, Clone)] -enum ContextRate { - /// A fixed rate (a query's, 1 at ingestion time, or a query-time - /// shareable candidate whose possible readers run it at one rate). - Fixed(f64), - /// `Σ rate · z` over a query-time shareable candidate's schedule-group - /// levels. - Components(Vec<(Variable, f64)>), -} - -/// Each context's rate (module documentation), and the queries' own -/// rates. -fn context_rates( - milp: &mut SelectionMilp<'_>, -) -> Result<(Vec, Vec), SelectionMilpError> { - let horizon = milp.cost_inputs.non_repeating_query_horizon_seconds; - let cost = |error: asap_cost::analytical::AnalyticalCostError| { - SelectionMilpError::Cost(error.to_string()) - }; - let query_rate: Vec = milp - .entries - .iter() - .map(|entry| evaluation_rate([(&entry.recurrence, entry.range_evaluation)], horizon)) - .collect::>() - .map_err(cost)?; - let mut rates = Vec::with_capacity(milp.inputs.contexts.len()); - milp.builder.scope = Scope::SharedContexts; - for context in 0..milp.inputs.contexts.len() { - let rate = match milp.inputs.contexts[context].kind { - ContextKind::Query(query) => ContextRate::Fixed(query_rate[query]), - ContextKind::Ingestion(_) => ContextRate::Fixed(1.0), - ContextKind::Shared(m) => { - let per_request = match milp.physical.candidates[m.0].materialization { - Materialization::MatAtIngestion(_) | Materialization::NotMaterialized => { - rates.push(ContextRate::Fixed(1.0)); - continue; - } - Materialization::MatAtQuery(_) => false, - Materialization::MatAtQueryKeptAcrossEvaluations(_) => true, - }; - let readers = milp.queries_reaching(m); - let demand = |query: usize| { - let entry = &milp.entries[query]; - ( - &entry.recurrence, - if per_request { - None - } else { - entry.range_evaluation - }, - ) - }; - if unknown_phase_joins_several( - readers.iter().map(|&query| &milp.entries[query].recurrence), - ) { - milp.note_inexact("schedule: an unknown phase among several known ones"); - } - let groups = schedule_groups(readers.iter().map(|&query| demand(query)), horizon) - .map_err(cost)?; - let mut levels_per_group: Vec = Vec::new(); - for group in groups { - let members: Vec = group - .readers - .iter() - .map(|&position| readers[position]) - .collect(); - let alone: Vec = members - .iter() - .map(|&query| evaluation_rate([demand(query)], horizon)) - .collect::>() - .map_err(cost)?; - let fastest = alone.iter().copied().fold(0.0, f64::max); - let full = group.evaluations_per_second; - let mut levels: Vec = alone.clone(); - if (full - fastest).abs() > 1e-9 * full.abs().max(1e-12) { - milp.note_inexact("schedule: a group's rate is not its fastest reader's"); - levels = vec![full]; - } - levels.sort_by(f64::total_cmp); - levels.dedup(); - levels_per_group.push((members.into_iter().zip(alone).collect(), levels)); - } - if let [(_, levels)] = levels_per_group.as_slice() { - if let [level] = levels.as_slice() { - rates.push(ContextRate::Fixed(*level)); - continue; - } - } - let mut components = Vec::new(); - for (members, levels) in levels_per_group { - let mut below = 0.0; - for &level in &levels { - let z = milp.builder.continuous("z (schedule groups)", Some(1.0)); - for &(query, own) in &members { - if own >= level || levels.len() == 1 { - let reach = milp.reach[&(query, m)]; - milp.builder - .row("z (schedule groups)", constraint!(z >= reach)); - } - } - components.push((z, level - below)); - below = level; - } - } - ContextRate::Components(components) - } - }; - rates.push(rate); - } - milp.builder.scope = Scope::Global; - Ok((rates, query_rate)) -} - -/// Whether some interval has readers of unknown phase and readers of two -/// or more known phases. -fn unknown_phase_joins_several<'r>(recurrences: impl Iterator) -> bool { - let mut phases: BTreeMap)> = BTreeMap::new(); - for recurrence in recurrences { - match recurrence { - QueryRecurrence::Repeated(RepeatedDemand::FixedInterval(interval)) => { - phases.entry(interval.0).or_default().0 = true; - } - QueryRecurrence::Repeated(RepeatedDemand::FixedIntervalAt { - interval, - evaluation_phase, - }) => { - phases - .entry(interval.0) - .or_default() - .1 - .insert(evaluation_phase.0 % u64::from(interval.0.max(1))); - } - _ => {} - } - } - phases - .values() - .any(|(unknown, known)| *unknown && known.len() > 1) -} - -/// The objective (cost model 1, module documentation) and its rows. -pub(crate) fn encode_objective(milp: &mut SelectionMilp<'_>) -> Result<(), SelectionMilpError> { - let CostModel::UsageOverTime(unit_cost) = milp.cost_inputs.cost_model.clone() else { - return Err(SelectionMilpError::UnsupportedCostModel("not cost model 1")); - }; - let horizon = milp.cost_inputs.non_repeating_query_horizon_seconds; - let cost = |error: asap_cost::analytical::AnalyticalCostError| { - SelectionMilpError::Cost(error.to_string()) - }; - let request_ms = milp.cost_inputs.measured_operation_costs.ns_per_evaluation * 1e-6; - if request_ms > 0.0 { - for entry in milp.entries { - let requests = evaluation_rate([(&entry.recurrence, None)], horizon).map_err(cost)?; - milp.request_cost_per_second += unit_cost.cost_per_cpu_op * request_ms - / milp.cost_inputs.resource_speed.ms_per_cpu_op - * requests; - } - } - let (rates, query_rate) = context_rates(milp)?; - let memory = unit_cost.cost_per_memory_byte_second; - let mut objective: Vec<(Variable, f64)> = Vec::new(); - let mut charge = |expression: &Expression, coefficient: f64| { - if coefficient != 0.0 { - for (variable, factor) in expression.linear_coefficients() { - objective.push((variable, factor * coefficient)); - } - } - }; - // Per query-time shareable context: the execution cost of its runs - // other than `m`'s plain own term, and per slot (`None`: `m`'s own run) - // the most it can be. - let mut context_execution: BTreeMap, f64>)> = - BTreeMap::new(); - // `H[m]` per query-time shareable context. - let mut held_time: HashMap = HashMap::new(); - let runs = milp.runs(); - for (context, candidate) in runs { - let terms = milp.run_terms(context, candidate); - milp.builder.scope = milp.scope_of(context); - let candidate_data = &milp.physical.candidates[candidate.0]; - let (timing, materialization) = (candidate_data.timing, candidate_data.materialization); - let own = milp.inputs.contexts[context].kind == ContextKind::Shared(candidate); - // The most this run adds to its context's execution cost: all its - // terms at 1. - let mut run_execution = 0.0; - for term in &terms { - let numbers = term.numbers; - charge(&term.indicator, numbers.standing_cost); - match &rates[context] { - ContextRate::Fixed(rate) => charge(&term.indicator, numbers.execution_cost * rate), - ContextRate::Components(components) => { - if numbers.execution_cost == 0.0 { - } else if term.plain && own { - for &(component, rate) in components { - charge(&component.into(), numbers.execution_cost * rate); - } - } else { - let (sum, _) = context_execution - .entry(context) - .or_insert_with(|| (Expression::from(0.0), BTreeMap::new())); - *sum += numbers.execution_cost * term.indicator.clone(); - run_execution += numbers.execution_cost; - } - } - } - // Held memory. - if numbers.held_bytes <= 0.0 || timing != ExecutionTiming::QueryTime { - continue; - } - // Per byte held per latency unit. - let per_byte_ms = memory * numbers.held_bytes / 1_000.0 * milp.ms_per_latency_unit; - match milp.inputs.contexts[context].kind { - ContextKind::Query(query) => { - let latency = milp.latency[query]; - let most = milp.latency_max[query]; - let held = milp - .builder - .continuous("H_query (held for a query's latency)", None); - let indicator = term.indicator.clone(); - milp.builder.row( - "H_query (held for a query's latency)", - constraint!(held >= latency - most * (1.0 - indicator)), - ); - charge(&held.into(), per_byte_ms * query_rate[query]); - } - ContextKind::Shared(m) => { - if own && materialization != Materialization::MatAtQuery(Medium::Memory) { - milp.note_inexact("held bytes of a shareable candidate not in memory"); - } - let (longest, most_held) = *held_time.entry(context).or_insert_with(|| { - let longest = milp - .builder - .continuous("H (held until m's slowest reader)", None); - let mut most_held: f64 = 0.0; - for query in 0..milp.latency.len() { - if let Some(&reach) = milp.reach.get(&(query, m)) { - let latency = milp.latency[query]; - let most = milp.latency_max[query]; - most_held = most_held.max(most); - milp.builder.row( - "H (held until m's slowest reader)", - constraint!(longest >= latency - most * (1.0 - reach)), - ); - } - } - (longest, most_held) - }); - let components: Vec<(Option, f64)> = match &rates[context] { - ContextRate::Fixed(rate) => vec![(None, *rate)], - ContextRate::Components(components) => components - .iter() - .map(|&(component, rate)| (Some(component), rate)) - .collect(), - }; - for (component, rate) in components { - let mut both = Expression::from(0.0); - let mut slack = 0.0; - if let Some(component) = component { - both += component; - slack += 1.0; - } - if !(term.plain && own) { - both += term.indicator.clone(); - slack += 1.0; - } - if slack == 0.0 { - charge(&longest.into(), per_byte_ms * rate); - continue; - } - let held = milp - .builder - .continuous("W (held time × schedule group)", None); - milp.builder.row( - "W (held time × schedule group)", - constraint!(held >= longest - most_held * (slack - both)), - ); - charge(&held.into(), per_byte_ms * rate); - } - } - ContextKind::Ingestion(_) => {} - } - } - if let Some((_, most_per_slot)) = context_execution.get_mut(&context) { - let slot = (!own).then(|| milp.inputs.slot[candidate.0]); - let entry = most_per_slot.entry(slot).or_default(); - *entry = entry.max(run_execution); - } - } - // A context's execution at its varying rate: `V ≥ E − Ē (x[m] − z)` per - // component `z`, `Ē` the most `E` can be: per slot, its costliest run. - milp.builder.scope = Scope::SharedContexts; - for (context, (sum, most_per_slot)) in context_execution { - let most: f64 = if super::model::tests_loose_big_m() { - // Before: every term's cost. - sum.clone() - .linear_coefficients() - .map(|(_, cost)| cost) - .sum() - } else { - most_per_slot.values().sum() - }; - let ContextRate::Components(components) = &rates[context] else { - continue; - }; - let ContextKind::Shared(m) = milp.inputs.contexts[context].kind else { - continue; - }; - let region = milp - .builder - .continuous("V (context execution at a rate component)", None); - milp.builder.row( - "V (context execution at a rate component)", - constraint!(region >= sum), - ); - let active = milp.selected[&m]; - let slowest = components - .iter() - .map(|&(_, rate)| rate) - .fold(f64::INFINITY, f64::min); - let mut at_rates = Expression::from(0.0); - for &(component, rate) in components { - let at_rate = milp - .builder - .continuous("V (context execution at a rate component)", None); - milp.builder.row( - "V (context execution at a rate component)", - constraint!(at_rate >= region - most * (active - component)), - ); - charge(&at_rate.into(), rate); - at_rates += rate * at_rate; - } - if slowest.is_finite() && slowest > 0.0 { - milp.builder.row( - "V (slowest rate)", - constraint!(at_rates >= slowest * region), - ); - } - } - milp.builder.scope = Scope::Global; - raw_retention(milp, unit_cost.cost_per_disk_byte_second, &mut charge); - milp.objective_terms = objective; - Ok(()) -} - -/// Raw retention: per source, the largest raw data a query-time scan run of -/// the plan keeps, by levels. -fn raw_retention( - milp: &mut SelectionMilp<'_>, - disk: f64, - charge: &mut impl FnMut(&Expression, f64), -) { - let mut kept: BTreeMap> = BTreeMap::new(); - for (context, candidate) in milp.runs() { - let Operator::NonASAP(NonASAPOp::Scan { source, .. }) = - &milp.physical.physical_node(candidate).operator - else { - continue; - }; - if milp.physical.candidates[candidate.0].timing != ExecutionTiming::QueryTime { - continue; - } - let source = source_name(source).to_string(); - for term in milp.run_terms(context, candidate) { - let bytes = term.numbers.raw_kept_bytes; - if bytes > 0.0 { - kept.entry(source.clone()) - .or_default() - .push((bytes, term.indicator)); - } - } - } - for (_, scans) in kept { - let mut levels: Vec = scans.iter().map(|(bytes, _)| *bytes).collect(); - levels.sort_by(f64::total_cmp); - levels.dedup(); - let mut below = 0.0; - for level in levels { - let at_least = milp - .builder - .continuous("v (raw retention levels)", Some(1.0)); - for (bytes, indicator) in &scans { - if *bytes >= level { - let indicator = indicator.clone(); - milp.builder.row( - "v (raw retention levels)", - constraint!(at_least >= indicator), - ); - } - } - charge(&at_least.into(), disk * (level - below)); - below = level; - } - } -} diff --git a/crates/plan-selection/src/milp/decode.rs b/crates/plan-selection/src/milp/decode.rs deleted file mode 100644 index 5b8b53466..000000000 --- a/crates/plan-selection/src/milp/decode.rs +++ /dev/null @@ -1,162 +0,0 @@ -//! #1095 Step 6 (§2.0, "Decode and check"): the group selection MILP's raw -//! solution as a [`DagOfSelectedCandidates`], one node per candidate run. -//! -//! - A candidate computed in a context (`p[ctx, c] = 1`, a `NotMaterialized` -//! query-time candidate in a query's or a shareable candidate's context, -//! or a `NotMaterialized` ingestion-time candidate in an `I_r`) is one node per -//! context: selected in two contexts, it is two nodes; read by two -//! readers in one context, it is one. -//! - A selected shareable candidate (`x[m] = 1`) is one node, whoever reads -//! it. -//! - In each context, input `n` of a computed node (or of `m`'s own node in -//! `m`'s context) reads the context's selected candidate among its -//! producers: the node computed there, the shareable candidate's node, or -//! the candidate's node in `I_r(m)` (read in ingestion-time `m`'s context). -//! - Output reference `r` is the selected output candidate in `r`'s query -//! context. -//! -//! Only the nodes the outputs reach are kept. The MILP never selects an -//! unread candidate (row 4), so this drops nothing at a solution whose -//! objective equals `plan_cost`. - -use std::collections::{BTreeMap, BTreeSet}; - -use asap_types::stages::{ - output_references, DagOfSelectedCandidates, PhysicalASAPMemoCandidates, PhysicalCandidateId, - SelectedEdge, SelectedNodeId, -}; - -use crate::groups::{MilpInputs, ResultSlot, RunKind}; - -/// One run of a candidate in the solution. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] -enum Run { - /// Candidate `.1` computed in context `.0`. - Computed(usize, PhysicalCandidateId), - /// Shareable candidate `.0`. - Shared(PhysicalCandidateId), -} - -/// The selected candidates of a solution: per context, `p[ctx, c] = 1`, and -/// the shareable candidates with `x[m] = 1`. -#[derive(Debug, Clone, PartialEq, Eq, Default)] -pub(crate) struct ChosenCandidates { - pub(crate) chosen: Vec>, - pub(crate) selected: BTreeSet, - /// Per multi-slot input `(context, reader, input)`, the slot it reads. - pub(crate) slot_reads: BTreeMap<(usize, PhysicalCandidateId, usize), ResultSlot>, -} - -/// The selection as a DAG of candidate runs over `physical` (module -/// documentation), or why it is not one (an output or an input with no -/// selected producer). The caller validates it -/// ([`DagOfSelectedCandidates::validate`]). -pub(crate) fn decode( - physical: &PhysicalASAPMemoCandidates, - inputs: &MilpInputs, - chosen: &ChosenCandidates, -) -> Result { - // The node a read of `producer` in `context` resolves to. - let run_of = |context: usize, producer: PhysicalCandidateId| -> Option { - if !chosen.chosen[context].contains(&producer) { - return None; - } - let kind = inputs.kind[producer.0]; - if inputs.contexts[context].computes(kind) { - return Some(Run::Computed(context, producer)); - } - match kind { - RunKind::Shareable => chosen - .selected - .contains(&producer) - .then_some(Run::Shared(producer)), - RunKind::Ingestion => inputs - .ingestion_context_read_by(context) - .filter(|&ingestion| chosen.chosen[ingestion].contains(&producer)) - .map(|ingestion| Run::Computed(ingestion, producer)), - RunKind::PerContext => None, - } - }; - // The run's context for its inputs, and its candidate. - let inputs_of = |run: Run| -> Result, String> { - let (context, candidate) = match run { - Run::Computed(context, candidate) => (context, candidate), - Run::Shared(candidate) => ( - *inputs - .context_of_shareable - .get(&candidate) - .ok_or_else(|| format!("shareable candidate {} has no context", candidate.0))?, - candidate, - ), - }; - let mut reads = Vec::new(); - for (input, producers) in inputs.producer_candidates_per_input[candidate.0] - .iter() - .enumerate() - { - let slot_read = match run { - Run::Computed(..) => chosen.slot_reads.get(&(context, candidate, input)), - Run::Shared(_) => None, - }; - let producer = producers - .iter() - .filter(|producer| { - matches!(run, Run::Shared(_)) - || inputs.slot[producer.0] != inputs.slot[candidate.0] - }) - .filter(|producer| slot_read.is_none_or(|&slot| inputs.slot[producer.0] == slot)) - .find_map(|&producer| run_of(context, producer)) - .ok_or_else(|| { - format!( - "input {input} of candidate {} in context {context} has no selected producer", - candidate.0 - ) - })?; - reads.push((input, producer)); - } - Ok(reads) - }; - let outputs: Vec = output_references(&physical.query_roots) - .into_iter() - .enumerate() - .map(|(reference, (query, _))| { - inputs.output_candidates[reference] - .iter() - .find_map(|&candidate| run_of(query, candidate)) - .ok_or_else(|| format!("output reference {reference} has no selected candidate")) - }) - .collect::>()?; - // Number the runs the outputs reach, depth first from the outputs. - let mut node_of: BTreeMap = BTreeMap::new(); - let mut reads_of: BTreeMap> = BTreeMap::new(); - let mut candidate_of_node = Vec::new(); - let mut stack: Vec = outputs.iter().rev().copied().collect(); - while let Some(run) = stack.pop() { - if node_of.contains_key(&run) { - continue; - } - node_of.insert(run, SelectedNodeId(candidate_of_node.len())); - candidate_of_node.push(match run { - Run::Computed(_, candidate) | Run::Shared(candidate) => candidate, - }); - let reads = inputs_of(run)?; - stack.extend(reads.iter().map(|&(_, producer)| producer)); - reads_of.insert(run, reads); - } - let mut edges = Vec::new(); - for (run, &consumer) in &node_of { - for &(input, producer) in &reads_of[run] { - edges.push(SelectedEdge { - producer: node_of[&producer], - consumer, - input, - }); - } - } - edges.sort_by_key(|edge| (edge.consumer.0, edge.input)); - Ok(DagOfSelectedCandidates { - candidate_of_node, - edges, - node_of_output_reference: outputs.iter().map(|run| node_of[run]).collect(), - }) -} diff --git a/crates/plan-selection/src/milp/lp_diagnosis.rs b/crates/plan-selection/src/milp/lp_diagnosis.rs deleted file mode 100644 index d8bdc113f..000000000 --- a/crates/plan-selection/src/milp/lp_diagnosis.rs +++ /dev/null @@ -1,302 +0,0 @@ -//! #1095 §6.1: where the selection MILP's LP relaxation loses its bound -//! (measurement only, behind the `lp-diagnosis` feature). With -//! `ASAP_STAGE3_LP_DIAGNOSIS=`, each solve appends one JSON line: -//! - `lp`: the LP relaxation of the model as built, and `root`: HiGHS's -//! bound after its root node (presolve, cuts), both from the warm start; -//! - `lp_without`: the LP with one row family dropped, per family: how much -//! of the LP bound that family's rows hold up; -//! - `lp_tightened`: the LP with valid rows added: `Σ terms ≥ run` for each -//! run charged only on its configuration or window terms (at integral -//! binaries one of them is 1); -//! - `objective_by_family`: the objective split by the family of each -//! charged variable, at the LP's solution, at the warm start and at the -//! incumbent, so `incumbent − lp` splits into one share per family; -//! - `runs`: the charged `p` and `x` variables with the largest difference -//! between the incumbent and the LP, with their values in both. -//! -//! A diagnosed run's wall time is not comparable with an undiagnosed one. -//! The LPs and the root node run without a time limit: an LP solve and a -//! one-node MIP solve end on their own. -use std::collections::BTreeMap; -use std::time::Instant; - -use good_lp::solvers::highs::highs; -use good_lp::{constraint, variable, Constraint, Expression, ProblemVariables, SolverModel}; - -use super::model::{objective_scale, SelectionMilp}; - -/// The environment variable naming the file the diagnosis appends to. -const DIAGNOSIS_ENV: &str = "ASAP_STAGE3_LP_DIAGNOSIS"; - -/// The runs listed per solve. -const RUNS_SHOWN: usize = 40; - -/// One LP (or root node) solved. -#[derive(Debug, Clone, serde::Serialize)] -struct Bound { - /// The bound, cost per second (`None` when HiGHS gave none). - per_second: Option, - status: String, - ms: u64, -} - -/// The LP solves, before the MILP's. -pub(crate) struct Diagnosis { - path: String, - lp: Bound, - root: Bound, - lp_without: BTreeMap<&'static str, Bound>, - lp_tightened: Bound, - tightening_rows: usize, - /// The LP's columns. - lp_columns: Vec, - /// The objective, by column. - objective: Vec<(usize, f64)>, - column_families: Vec<&'static str>, - /// Per `p` and `x` column, what it stands for. - labels: BTreeMap, -} - -impl Diagnosis { - /// The LP solves of `milp` from `start` (columns), when the diagnosis is - /// asked for. - pub(crate) fn run(milp: &SelectionMilp<'_>, start: Option<&[f64]>) -> Option { - let path = std::env::var(DIAGNOSIS_ENV).ok()?; - let definitions: Vec<_> = milp - .builder - .variables - .iter_variables_with_def() - .map(|(variable, definition)| (variable, definition.clone())) - .collect(); - let column_of: std::collections::HashMap<_, usize> = definitions - .iter() - .enumerate() - .map(|(column, (variable, _))| (*variable, column)) - .collect(); - let objective: Vec<(usize, f64)> = milp - .objective_terms - .iter() - .map(|&(variable, coefficient)| (column_of[&variable], coefficient)) - .collect(); - let start_values: Option> = start.map(|start| { - definitions - .iter() - .map(|(variable, _)| *variable) - .zip(start.iter().copied()) - .collect() - }); - let scale = objective_scale(&milp.objective_terms, start_values.as_deref()); - // The relaxation: the same variables (same indices), none integral. - let relaxed = || { - let mut variables = ProblemVariables::new(); - for (_, definition) in &definitions { - variables.add( - variable() - .min(definition.get_min()) - .max(definition.get_max()), - ); - } - variables - }; - let objective_expression: Expression = milp - .objective_terms - .iter() - .map(|&(variable, coefficient)| coefficient * scale * variable) - .sum(); - let solve = |variables: ProblemVariables, rows: Vec, root: bool| { - let started = Instant::now(); - let problem = variables - .minimise(objective_expression.clone()) - .using(highs) - .with_all(rows); - let mut model = problem.try_into_inner().expect("a HiGHS model"); - model.set_option("output_flag", false); - model.set_option("threads", milp.highs_threads); - if root { - // The root node, then stop: its bound is the root bound. - model.set_option("mip_max_nodes", 1); - model.set_option("mip_rel_gap", 0.0); - model.set_option("mip_abs_gap", 0.0); - if let Some(start) = start { - let _ = model.try_set_solution(Some(start), None, None, None); - } - } - let Ok(solved) = model.try_solve() else { - return ( - Bound { - per_second: None, - status: "solver error".into(), - ms: started.elapsed().as_millis() as u64, - }, - Vec::new(), - ); - }; - let value = if root { - solved.double_info_value(c"mip_dual_bound").ok() - } else if solved.status() == highs::HighsModelStatus::Optimal { - Some(solved.objective_value()) - } else { - None - }; - let columns = if root { - Vec::new() - } else { - solved.get_solution().columns().to_vec() - }; - ( - Bound { - per_second: value.filter(|v| v.is_finite()).map(|v| v / scale), - status: format!("{:?}", solved.status()), - ms: started.elapsed().as_millis() as u64, - }, - columns, - ) - }; - let rows = &milp.builder.rows; - let families = &milp.builder.row_families; - let (lp, lp_columns) = solve(relaxed(), rows.clone(), false); - let (root, _) = solve(milp.builder.variables.clone(), rows.clone(), true); - let mut lp_without = BTreeMap::new(); - let present: std::collections::BTreeSet<&'static str> = families.iter().copied().collect(); - for family in present { - let kept: Vec = rows - .iter() - .zip(families) - .filter(|(_, &f)| f != family) - .map(|(row, _)| row.clone()) - .collect(); - lp_without.insert(family, solve(relaxed(), kept, false).0); - } - let tightening = cover_rows(milp); - let tightening_rows = tightening.len(); - let mut tightened = rows.clone(); - tightened.extend(tightening); - let (lp_tightened, _) = solve(relaxed(), tightened, false); - let candidate = |c: asap_types::stages::PhysicalCandidateId| { - let data = &milp.physical.candidates[c.0]; - format!( - "c{} {} {:?} {:?}", - c.0, - milp.physical.physical_node(c).operator.kind_name(), - data.timing, - data.materialization - ) - }; - let mut labels = BTreeMap::new(); - for (context, choices) in milp.choice.iter().enumerate() { - let place = match milp.inputs.contexts[context].kind { - crate::groups::ContextKind::Query(query) => format!("q{query}"), - crate::groups::ContextKind::Ingestion(pipeline) => match pipeline { - crate::groups::IngestionPipeline::Window(window) => { - format!("I[{}ms@{}]", window.width_ms, window.phase_ms) - } - crate::groups::IngestionPipeline::Alone(m) => format!("I[alone m{}]", m.0), - }, - crate::groups::ContextKind::Shared(m) => format!("m{}", m.0), - }; - for (&c, variable) in choices { - labels.insert(column_of[variable], format!("p[{place}] {}", candidate(c))); - } - } - for (&m, variable) in &milp.selected { - labels.insert(column_of[variable], format!("x {}", candidate(m))); - } - Some(Self { - labels, - path, - lp, - root, - lp_without, - lp_tightened, - tightening_rows, - lp_columns, - objective, - column_families: milp.builder.column_families.clone(), - }) - } - - /// Appends the line, with the warm start's and the incumbent's columns - /// (completed) and the MILP's results. - pub(crate) fn write( - self, - start: Option<&[f64]>, - incumbent: Option<&[f64]>, - objective_per_second: f64, - dual_bound_per_second: f64, - ) { - let by_family = |columns: Option<&[f64]>| -> Option> { - let columns = columns.filter(|columns| !columns.is_empty())?; - let mut split: BTreeMap<&'static str, f64> = BTreeMap::new(); - for &(column, coefficient) in &self.objective { - *split.entry(self.column_families[column]).or_default() += - coefficient * columns[column]; - } - Some(split) - }; - // Per labeled column: (LP charge, incumbent charge, LP value, - // incumbent value). - let mut runs: Vec<(String, f64, f64, f64, f64)> = Vec::new(); - if let Some(incumbent) = incumbent.filter(|columns| !columns.is_empty()) { - let mut charge: BTreeMap = BTreeMap::new(); - for &(column, coefficient) in &self.objective { - if self.labels.contains_key(&column) { - *charge.entry(column).or_default() += coefficient; - } - } - for (column, coefficient) in charge { - let (lp, at) = (self.lp_columns[column], incumbent[column]); - runs.push(( - self.labels[&column].clone(), - coefficient * lp, - coefficient * at, - lp, - at, - )); - } - runs.sort_by(|a, b| (b.2 - b.1).abs().total_cmp(&(a.2 - a.1).abs())); - runs.truncate(RUNS_SHOWN); - } - let line = serde_json::json!({ - "runs": runs, - "variables": self.column_families.len(), - "objective_per_second": objective_per_second, - "dual_bound_per_second": dual_bound_per_second, - "lp": self.lp, - "root": self.root, - "lp_without": self.lp_without, - "lp_tightened": self.lp_tightened, - "tightening_rows": self.tightening_rows, - "objective_by_family": { - "lp": by_family(Some(&self.lp_columns)), - "warm_start": by_family(start), - "incumbent": by_family(incumbent), - }, - }); - use std::io::Write; - if let Ok(mut file) = std::fs::OpenOptions::new() - .create(true) - .append(true) - .open(&self.path) - { - let _ = writeln!(file, "{line}"); - } - } -} - -/// `Σ terms ≥ run` for each run whose terms are all configuration or -/// window terms (none on the run's own indicator): at integral binaries a -/// used run has one term at 1. -fn cover_rows(milp: &SelectionMilp<'_>) -> Vec { - let mut rows = Vec::new(); - for (&(context, candidate), terms) in &milp.run_terms_cache { - if terms.is_empty() || terms.iter().any(|term| term.plain) { - continue; - } - let Some(run) = milp.run_indicator(context, candidate) else { - continue; - }; - let sum: Expression = terms.iter().map(|term| term.indicator.clone()).sum(); - rows.push(constraint!(sum >= run)); - } - rows -} diff --git a/crates/plan-selection/src/milp/mod.rs b/crates/plan-selection/src/milp/mod.rs index 7c0e9ab2f..9a2d7f7e3 100644 --- a/crates/plan-selection/src/milp/mod.rs +++ b/crates/plan-selection/src/milp/mod.rs @@ -1,26 +1,3 @@ -//! #1095 Steps 3–5: the group selection MILP (§2.0) over -//! [`MilpInputs`](crate::groups::MilpInputs): one candidate per result slot -//! per execution context, solved once with HiGHS from a warm start. -//! -//! - [`model`]: the variables and the structural rows (coverage, slots, -//! validity, used only if read, sharing, reach, reader constraints, -//! latency, accuracy), the solve and the raw solution; -//! - [`cost_terms`]: the cost model 1 objective, every coefficient a -//! number `asap-cost` computed (§3); -//! - [`warm_start`]: each query's cheapest plan, then greedy sharing, -//! HiGHS's first incumbent; -//! - [`completion`]: the exact objective of a plan given as binaries, the -//! cheapest values of the continuous variables found without a solver; -//! - [`decode`]: the solution as a `DagOfSelectedCandidates`, one node per -//! candidate run; -//! - `lp_diagnosis` (feature `lp-diagnosis`, measurement only): where the -//! LP relaxation loses its bound. -//! -//! [`crate::select`] runs them, realizes the plan once and checks it. -pub(crate) mod completion; -pub(crate) mod cost_terms; -pub(crate) mod decode; -#[cfg(feature = "lp-diagnosis")] -pub(crate) mod lp_diagnosis; +//! Stage 3's selection MILP (concepts doc, sections 3–5): its variables, +//! one function per constraint family, and the execution contexts. pub(crate) mod model; -pub(crate) mod warm_start; diff --git a/crates/plan-selection/src/milp/model.rs b/crates/plan-selection/src/milp/model.rs index ffd268768..e976556c0 100644 --- a/crates/plan-selection/src/milp/model.rs +++ b/crates/plan-selection/src/milp/model.rs @@ -1,2185 +1,134 @@ -//! #1095 §2.0: the group selection MILP's variables and rows over execution -//! contexts ([`crate::groups`]), one HiGHS solve, and the raw solution. +//! Stage 3's selection MILP, named after the formulation (concepts doc, +//! sections 3–5). //! -//! ## Variables +//! | Family | Variables | Constraints | +//! |---|---|---| +//! | baseline (section 3) | $x_v$, $y_{u,c,i}$, $z_{q,v}$, $t_v$ ([`SelectionVariables`]) | (A) [`input_satisfaction_constraints`], (B) [`producer_selection_constraints`], (C)/(D) [`query_coverage_constraints`], (E) [`acyclicity_constraints`], (F) [`utilization_constraints`]; (G), (H): $y$ exists only on admissible edges | +//! | execution contexts (section 5) | $x^{\mathrm{mat}}_v$, $x_{\kappa,v}$, $y_{\kappa,u,c,i}$, $h_v$ ([`ExecutionContextVariables`]) | (R) [`retention_constraints`] | +//! | accuracy (section 4) | $E_c$, $F_{c,i}$, $D_c$, $H_{c,i}$ ([`AccuracyVariables`]) | (K) [`accuracy_constraints`] | //! -//! | Family | Meaning | -//! |---|---| -//! | `p[ctx, c]` | in context `ctx`, candidate `c` gives its result slot (binary): computed there, or read across the boundary (a shareable candidate, or a candidate of an ingestion pipeline `I_r`) | -//! | `x[m]` | shareable candidate `m` is selected (binary) | -//! | `a[q, m]` | query `q` depends on shareable `m` (continuous, exact at integral binaries) | -//! | `y[q, m, c]` | `AND(a[q, m], p[m, c])`, for reach from above and accuracy (continuous) | -//! | `L[ctx]`, `T[q, m]`, `K[q]` | latency of a query or of a query-time shareable candidate's run and context; `m`'s latency in `q`'s; compaction (continuous) | -//! | `N[q]`, `A[q, parts]` | approximate values `q` depends on; whether it depends on one sized for `parts` shares (continuous) | -//! | `g[m, c]` | highest shared materialization: `c` is read in context `m`, which at most one query reaches (binary) | -//! | `rank` | order ranks on a context's cyclic links (continuous) | -//! -//! The cost families are [`super::cost_terms`]'s. -//! -//! ## Rows (numbered as in #1095 §2.0) -//! -//! 1. Coverage: `Σ_{c ∈ Out(r)} p[q(r), c] = 1` per output reference. -//! 2. One candidate per result slot per context: `Σ_{c ∈ slot} p[ctx, c] ≤ 1`. -//! 3. Validity: `Σ_{c' ∈ compat(c, n)} p[ctx, c'] ≥ p[ctx, c]` for each `c` -//! computed in `ctx`; `≥ x[m]` for `m`'s own inputs in its context; -//! `p[ctx, m] ≤ x[m]`, `p[m, c] ≤ p[I_r(m), c]` for reads across a -//! boundary (`r(m)` the refresh window of ingestion-time `m`). -//! 4. Used only if read, per context; a candidate of `I_r` is also read -//! by `p[m, c]` for each `m` with `r(m) = r`. -//! 5. Sharing: `x[m] ≤ Σ_ctx p[ctx, m]`. -//! 6. Reach: `a[q, m] ≥ p[q, m]`, `a[q, c] ≥ a[q, m] + p[m, c] − 1`; from -//! above, where a reader constraint counts queries. -//! 7. Latency (`plan_cost`'s: each query-time run it reaches counts once). -//! 8. Accuracy (A1), per run a query depends on. -//! 9. Reader constraints. -//! -//! A link from a candidate to one of its own result slot is dropped: with -//! one candidate per slot per context it can only be a self-loop (a -//! widened scan's `Project` lists its own memo group). Links inside a -//! larger cyclic component of a context get order ranks. -use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; -use std::time::Instant; - -use asap_cost::{CostInputs, CostModel}; -use asap_types::stages::{ - output_references, PhysicalASAPMemoCandidates, PhysicalCandidateId, ReaderConstraint, -}; -use asap_types::workload::QueryWorkloadEntry; -use good_lp::solvers::highs::highs; -use good_lp::{ - constraint, variable, Constraint, Expression, IntoAffineExpression, ProblemVariables, - SolverModel, Variable, -}; - -use crate::groups::{AccuracyWeight, ContextKind, MilpInputs, ResultSlot, RunKind}; - -use super::cost_terms::{self, RunTerm}; -use super::warm_start; - -/// Why the selection MILP was not built or solved. -#[derive(Debug, Clone, PartialEq)] -pub(crate) enum SelectionMilpError { - /// The MILP implements cost model 1 only (§3); the name of the model given. - UnsupportedCostModel(&'static str), - /// `entries` and the query roots differ in number. - EntryCount { roots: usize, entries: usize }, - /// No plan meets every row. - Infeasible, - /// A rate or another number `asap-cost` could not compute. - Cost(String), - /// HiGHS failed. - Solver(String), -} - -/// How the MILP is built. -#[derive(Debug, Clone, Copy)] -pub(crate) struct SelectionMilpOptions { - /// Write the accuracy rows (row 8); off when the facade ignores - /// accuracy guarantees. - pub(crate) enforce_accuracy: bool, - /// Solve each independent part alone ([`independent_parts`]); off only - /// to compare with one solve in tests. - pub(crate) split_into_parts: bool, - /// HiGHS's threads (its `threads` option). One thread makes ties among - /// plans of equal cost break the same way on every run. - pub(crate) highs_threads: i32, -} +//! Every function body is `todo!()` until #1207 PRs 6 (baseline), 7 +//! (contexts) and 8 (accuracy) fill them in; until then nothing calls +//! them, hence the `dead_code` allowance below. +#![allow(dead_code)] -impl Default for SelectionMilpOptions { - fn default() -> Self { - Self { - enforce_accuracy: true, - split_into_parts: true, - highs_threads: 1, - } - } -} - -/// Which part of the model a variable or row belongs to, for the size -/// report: the query contexts, the `I_r`, the shareable candidates' contexts -/// (with their `x`, reach and schedule terms), or the rest. -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, serde::Serialize)] -pub(crate) enum Scope { - Queries, - Ingestion, - SharedContexts, - Global, -} - -/// The size of one family (or scope) of variables and rows. -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, serde::Serialize)] -pub(crate) struct FamilySize { - pub(crate) variables: usize, - pub(crate) binaries: usize, - pub(crate) rows: usize, - /// Nonzero coefficients of its rows. - pub(crate) nonzeros: usize, -} - -impl FamilySize { - fn add(&mut self, other: FamilySize) { - self.variables += other.variables; - self.binaries += other.binaries; - self.rows += other.rows; - self.nonzeros += other.nonzeros; - } -} +use std::collections::HashMap; -/// Variables and rows, counted per family and per scope. -pub(crate) struct ModelBuilder { - pub(crate) variables: ProblemVariables, - pub(crate) rows: Vec, - /// The family of each row of `rows`. - pub(crate) row_families: Vec<&'static str>, - pub(crate) families: BTreeMap<&'static str, FamilySize>, - pub(crate) scopes: BTreeMap, - /// Where new variables and rows are counted. - pub(crate) scope: Scope, - /// The family of each variable, in creation order. - #[cfg(feature = "lp-diagnosis")] - pub(crate) column_families: Vec<&'static str>, -} - -impl ModelBuilder { - fn new() -> Self { - Self { - variables: ProblemVariables::new(), - rows: Vec::new(), - row_families: Vec::new(), - families: BTreeMap::new(), - scopes: BTreeMap::new(), - scope: Scope::Global, - #[cfg(feature = "lp-diagnosis")] - column_families: Vec::new(), - } - } - - fn count(&mut self, family: &'static str, size: FamilySize) { - #[cfg(feature = "lp-diagnosis")] - if size.variables > 0 { - self.column_families.push(family); - } - self.families.entry(family).or_default().add(size); - self.scopes.entry(self.scope).or_default().add(size); - } - - pub(crate) fn binary(&mut self, family: &'static str) -> Variable { - self.count( - family, - FamilySize { - variables: 1, - binaries: 1, - ..FamilySize::default() - }, - ); - self.variables.add(variable().binary()) - } +use good_lp::{Constraint, Variable}; - /// A continuous variable in `[0, max]` (unbounded above for `None`). - pub(crate) fn continuous(&mut self, family: &'static str, max: Option) -> Variable { - self.count( - family, - FamilySize { - variables: 1, - ..FamilySize::default() - }, - ); - let definition = variable().min(0.0); - self.variables.add(match max { - Some(max) => definition.max(max), - None => definition, - }) - } - - pub(crate) fn row(&mut self, family: &'static str, row: Constraint) { - let nonzeros = row.expression().linear_coefficients().count(); - self.count( - family, - FamilySize { - rows: 1, - nonzeros, - ..FamilySize::default() - }, - ); - self.rows.push(row); - self.row_families.push(family); - } - - /// `and ≥ Σ terms − (n − 1)`, `and ∈ [0, 1]`: the AND of `n` - /// indicators, tight under a cost or a `≤` row that charges it. - pub(crate) fn and_at_least(&mut self, family: &'static str, terms: &[Expression]) -> Variable { - let and = self.continuous(family, Some(1.0)); - let sum: Expression = terms.iter().cloned().sum(); - let count = terms.len() as f64; - self.row(family, constraint!(and >= sum - (count - 1.0))); - and - } -} +use asap_types::stages::{ + ExecutionContextId, PhysicalASAPMemoCandidates, PhysicalCandidateId, PhysicalEdgeId, +}; -/// One way a candidate's input is read in a context: `(reader, input)` -/// with the reader computed in that context, the query's output reference, -/// or the shareable candidate's own input. +/// A query, by index in the workload. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub(crate) enum Reader { - Candidate(PhysicalCandidateId, usize), - Output(usize), - OwnInput(usize), -} - -/// One input of a candidate: `(reader, input)`. -type InputOf = (PhysicalCandidateId, usize); - -/// The slot links of one multi-slot input: `(slot, e)`. -type SlotLinks = Vec<(ResultSlot, Variable)>; - -/// The built model, before the solve. -pub(crate) struct SelectionMilp<'a> { - pub(crate) builder: ModelBuilder, - pub(crate) physical: &'a PhysicalASAPMemoCandidates, - pub(crate) inputs: &'a MilpInputs, - pub(crate) entries: &'a [QueryWorkloadEntry], - pub(crate) cost_inputs: &'a CostInputs, - /// Per context, `p[ctx, c]` per option. - pub(crate) choice: Vec>, - /// Per context, per option, the reads of it in that context. - pub(crate) readers: Vec>>, - /// `x[m]` per shareable candidate with a context. - pub(crate) selected: BTreeMap, - /// `a[q, m]`, for each query that may depend on shareable `m`. - pub(crate) reach: HashMap<(usize, PhysicalCandidateId), Variable>, - /// Per shareable candidate, the shareable candidates whose contexts may - /// read it. - pub(crate) shared_readers: BTreeMap>, - /// `L[q]`. - pub(crate) latency: Vec, - /// An upper bound of each `L[q]` (its bound, or the most it can be), in - /// latency units. - pub(crate) latency_max: Vec, - /// Milliseconds per latency unit: every latency variable (`L`, `T`, `K` - /// and the held times) counts in these units, chosen so the largest - /// possible latency is about 1,000 units (with sub-microsecond work per - /// execution, milliseconds fall below HiGHS's absolute tolerances). - pub(crate) ms_per_latency_unit: f64, - /// The cost model 1 objective, in cost per second, without the request - /// fees (constant for every plan, added after selection). - pub(crate) objective_terms: Vec<(Variable, f64)>, - /// The request fees per second (the same for every plan). - pub(crate) request_cost_per_second: f64, - /// Terms encoded inexactly, by reason. - pub(crate) inexact: BTreeMap<&'static str, usize>, - /// The terms of each run `(context, candidate)`, cached; a shareable - /// candidate's own run is keyed by its own context. - pub(crate) run_terms_cache: HashMap<(usize, PhysicalCandidateId), Vec>, - /// Runs whose terms are being built (cycles of links). - pub(crate) terms_in_progress: HashSet<(usize, PhysicalCandidateId)>, - /// Per run and outputs, the variable `≥` its terms giving them. - pub(crate) producer_gives: HashMap<(usize, PhysicalCandidateId, Vec), Variable>, - /// Per query-time scan run, its window terms - /// ([`cost_terms::window_terms`]). - pub(crate) window_cache: HashMap<(usize, PhysicalCandidateId), Option>>, - /// For each input whose producers lie in several result slots (a - /// factored root over the private nodes of several implementations), - /// per context: `e[ctx, r, n, slot]`, the slot input `n` of computed - /// reader `r` reads (binary). Row 4 counts these reads, so a selected - /// producer in the other slot is not justified by `r`. - pub(crate) slot_links: - BTreeMap<(usize, PhysicalCandidateId, usize), Vec<(ResultSlot, Variable)>>, - /// [`SelectionMilpOptions::split_into_parts`]. - pub(crate) split_into_parts: bool, - /// [`SelectionMilpOptions::highs_threads`]. - pub(crate) highs_threads: i32, -} - -/// Variables and rows of one model. -#[derive(Debug, Clone, Default, PartialEq, Eq, serde::Serialize)] -pub(crate) struct SelectionMilpSize { - pub(crate) variables: usize, - pub(crate) binaries: usize, - pub(crate) rows: usize, - pub(crate) nonzeros: usize, - pub(crate) families: BTreeMap<&'static str, FamilySize>, - pub(crate) scopes: BTreeMap, - pub(crate) inexact: BTreeMap<&'static str, usize>, -} - -/// The raw solution of a solve run to proven optimality, ready to decode. -#[derive(Debug, Clone, PartialEq)] -pub(crate) struct SelectionMilpSolution { - /// The objective: cost per second without the request fees. - pub(crate) cost_per_second: f64, - /// The request fees per second, the same for every plan. - pub(crate) request_cost_per_second: f64, - /// HiGHS's dual bound on `cost_per_second`. - pub(crate) cost_lower_bound_per_second: f64, - /// Per context, the options with `p[ctx, c] = 1`. - pub(crate) chosen: Vec>, - /// The shareable candidates with `x[m] = 1`. - pub(crate) selected: BTreeSet, - /// Per multi-slot input `(context, reader, input)` of a used reader, the - /// slot it reads. - pub(crate) slot_reads: BTreeMap<(usize, PhysicalCandidateId, usize), ResultSlot>, - pub(crate) size: SelectionMilpSize, - /// Per independent part, how its solve went, in part order. - pub(crate) parts: Vec, - pub(crate) build_ms: u64, - /// Building the warm start, HiGHS's first incumbent. - pub(crate) warm_start_ms: u64, - /// The warm start's objective (like `cost_per_second`), when its - /// completion costed it. - pub(crate) warm_start_cost_per_second: Option, - pub(crate) solve_ms: u64, -} - -impl<'a> SelectionMilp<'a> { - pub(crate) fn query_count(&self) -> usize { - self.physical.query_roots.len() - } - - /// The context where `context` reads a candidate of an `I_r` - /// ([`MilpInputs::ingestion_context_read_by`]). - pub(crate) fn ingestion_context_read_by(&self, context: usize) -> Option { - self.inputs.ingestion_context_read_by(context) - } - - /// The context of shareable candidate `m`. - pub(crate) fn context_of(&self, shareable: PhysicalCandidateId) -> Option { - self.inputs.context_of_shareable.get(&shareable).copied() - } - - pub(crate) fn kind(&self, candidate: PhysicalCandidateId) -> RunKind { - self.inputs.kind[candidate.0] - } - - pub(crate) fn slot(&self, candidate: PhysicalCandidateId) -> ResultSlot { - self.inputs.slot[candidate.0] - } - - /// Whether `candidate` is computed in context `context`. - pub(crate) fn computes(&self, context: usize, candidate: PhysicalCandidateId) -> bool { - self.inputs.contexts[context].computes(self.kind(candidate)) - } - - /// The indicator of run `(context, candidate)`: `p[ctx, c]` for a - /// candidate computed there, `x[m]` for a shareable candidate's own run - /// in its own context. - pub(crate) fn run_indicator( - &self, - context: usize, - candidate: PhysicalCandidateId, - ) -> Option { - if self.inputs.contexts[context].kind == ContextKind::Shared(candidate) { - return self.selected.get(&candidate).copied(); - } - if !self.computes(context, candidate) { - return None; - } - self.choice[context].get(&candidate).copied() - } - - /// The context where run `(context, candidate)` reads its inputs: its - /// own (a shareable candidate's own run reads in its context too). - pub(crate) fn runs_at_query_time(&self, context: usize) -> bool { - self.inputs.contexts[context].at_query_time - } - - pub(crate) fn size(&self) -> SelectionMilpSize { - let families = self.builder.families.clone(); - SelectionMilpSize { - variables: families.values().map(|f| f.variables).sum(), - binaries: families.values().map(|f| f.binaries).sum(), - rows: families.values().map(|f| f.rows).sum(), - nonzeros: families.values().map(|f| f.nonzeros).sum(), - families, - scopes: self.builder.scopes.clone(), - inexact: self.inexact.clone(), - } - } - - pub(crate) fn note_inexact(&mut self, reason: &'static str) { - *self.inexact.entry(reason).or_default() += 1; - } - - /// The scope of context `context`. - pub(crate) fn scope_of(&self, context: usize) -> Scope { - match self.inputs.contexts[context].kind { - ContextKind::Query(_) => Scope::Queries, - ContextKind::Ingestion(_) => Scope::Ingestion, - ContextKind::Shared(_) => Scope::SharedContexts, - } - } - - /// Run `(context, candidate)`'s terms ([`cost_terms::run_terms`]), - /// cached. - pub(crate) fn run_terms( - &mut self, - context: usize, - candidate: PhysicalCandidateId, - ) -> Vec { - if let Some(terms) = self.run_terms_cache.get(&(context, candidate)) { - return terms.clone(); - } - let scope_here = self.scope_of(context); - let scope = std::mem::replace(&mut self.builder.scope, scope_here); - let terms = cost_terms::run_terms(self, context, candidate); - self.builder.scope = scope; - self.run_terms_cache - .insert((context, candidate), terms.clone()); - terms - } - - /// Every run of the model: `(context, candidate)` for each candidate - /// computed in a context, and each shareable candidate's own run. - pub(crate) fn runs(&self) -> Vec<(usize, PhysicalCandidateId)> { - let mut runs = Vec::new(); - for (context, options) in self.choice.iter().enumerate() { - for &candidate in options.keys() { - if self.computes(context, candidate) { - runs.push((context, candidate)); - } - } - if let ContextKind::Shared(own) = self.inputs.contexts[context].kind { - if self.selected.contains_key(&own) { - runs.push((context, own)); - } - } - } - runs - } - - /// The query contexts that may depend on shareable `m`, ascending. - pub(crate) fn queries_reaching(&self, shareable: PhysicalCandidateId) -> Vec { - let mut queries: Vec = (0..self.query_count()) - .filter(|&query| self.reach.contains_key(&(query, shareable))) - .collect(); - queries.sort_unstable(); - queries - } -} - -/// Builds the MILP over `inputs` (§2.0): `entries[q]` is query `q`'s -/// workload entry. -pub(crate) fn build_selection_milp<'a>( - physical: &'a PhysicalASAPMemoCandidates, - inputs: &'a MilpInputs, - entries: &'a [QueryWorkloadEntry], - cost_inputs: &'a CostInputs, - options: SelectionMilpOptions, -) -> Result, SelectionMilpError> { - match cost_inputs.cost_model { - CostModel::UsageOverTime(_) => {} - CostModel::ProvisionedCapacity(_) => { - return Err(SelectionMilpError::UnsupportedCostModel( - "ProvisionedCapacity (cost model 2)", - )) - } - CostModel::ClusterCapacity(_) => { - return Err(SelectionMilpError::UnsupportedCostModel( - "ClusterCapacity (cost model 3)", - )) - } - } - let query_count = physical.query_roots.len(); - if entries.len() != query_count { - return Err(SelectionMilpError::EntryCount { - roots: query_count, - entries: entries.len(), - }); - } - let mut milp = SelectionMilp { - builder: ModelBuilder::new(), - physical, - inputs, - entries, - cost_inputs, - choice: Vec::new(), - readers: Vec::new(), - selected: BTreeMap::new(), - reach: HashMap::new(), - shared_readers: BTreeMap::new(), - latency: Vec::new(), - latency_max: Vec::new(), - ms_per_latency_unit: 1.0, - objective_terms: Vec::new(), - request_cost_per_second: 0.0, - inexact: BTreeMap::new(), - run_terms_cache: HashMap::new(), - terms_in_progress: HashSet::new(), - producer_gives: HashMap::new(), - window_cache: HashMap::new(), - slot_links: BTreeMap::new(), - split_into_parts: options.split_into_parts, - highs_threads: options.highs_threads, - }; - choice_variables(&mut milp); - structural_rows(&mut milp); - acyclic_rows(&mut milp); - acyclic_context_rows(&mut milp); - reach_rows(&mut milp); - reader_constraint_rows(&mut milp); - latency_rows(&mut milp)?; - if options.enforce_accuracy { - accuracy_rows(&mut milp); - } - cost_terms::encode_objective(&mut milp)?; - Ok(milp) -} - -/// `p`, `x` and, per context, the reads of each option. -fn choice_variables(milp: &mut SelectionMilp<'_>) { - let inputs = milp.inputs; - let output_refs = output_references(&milp.physical.query_roots); - for (context, data) in inputs.contexts.iter().enumerate() { - milp.builder.scope = milp.scope_of(context); - let options: BTreeSet = data.options.iter().copied().collect(); - let mut readers: BTreeMap> = BTreeMap::new(); - let mut choices = BTreeMap::new(); - for &option in &options { - choices.insert(option, milp.builder.binary("p (choices)")); - } - for &reader in &options { - if !data.computes(inputs.kind[reader.0]) { - continue; - } - for (input, producers) in inputs.producer_candidates_per_input[reader.0] - .iter() - .enumerate() - { - for &producer in producers { - // A read of its own result slot is a self-loop. - if options.contains(&producer) - && inputs.slot[producer.0] != inputs.slot[reader.0] - { - readers - .entry(producer) - .or_default() - .push(Reader::Candidate(reader, input)); - } - } - } - } - match data.kind { - ContextKind::Query(query) => { - for (reference, (q, _)) in output_refs.iter().enumerate() { - if *q != query { - continue; - } - for &out in &inputs.output_candidates[reference] { - if options.contains(&out) { - readers - .entry(out) - .or_default() - .push(Reader::Output(reference)); - } - } - } - } - ContextKind::Shared(own) => { - for (input, producers) in inputs.producer_candidates_per_input[own.0] - .iter() - .enumerate() - { - for &producer in producers { - if options.contains(&producer) { - readers - .entry(producer) - .or_default() - .push(Reader::OwnInput(input)); - } - } - } - } - ContextKind::Ingestion(_) => {} - } - for &reader in &options { - if !data.computes(inputs.kind[reader.0]) { - continue; - } - for (input, producers) in inputs.producer_candidates_per_input[reader.0] - .iter() - .enumerate() - { - let slots: BTreeSet = producers - .iter() - .filter(|producer| options.contains(producer)) - .map(|producer| inputs.slot[producer.0]) - .filter(|&slot| slot != inputs.slot[reader.0]) - .collect(); - if slots.len() > 1 { - let links = slots - .into_iter() - .map(|slot| (slot, milp.builder.binary("e (slot an input reads)"))) - .collect(); - milp.slot_links.insert((context, reader, input), links); - } - } - } - milp.choice.push(choices); - milp.readers.push(readers); - } - milp.builder.scope = Scope::SharedContexts; - for &shareable in inputs.context_of_shareable.keys() { - let selected = milp.builder.binary("x (shareable candidates)"); - milp.selected.insert(shareable, selected); - } - for (context, data) in inputs.contexts.iter().enumerate() { - if let ContextKind::Shared(reader) = data.kind { - for &option in &data.options { - if inputs.kind[option.0] == RunKind::Shareable { - milp.shared_readers.entry(option).or_default().push(reader); - } - } - let _ = context; - } - } - milp.builder.scope = Scope::Global; -} - -/// Rows 1–5. -fn structural_rows(milp: &mut SelectionMilp<'_>) { - let inputs = milp.inputs; - let output_refs = output_references(&milp.physical.query_roots); - // Per `I_r`, the contexts of the ingestion-time shareable candidates of - // refresh window `r`. - let mut readers_of_ingestion: BTreeMap> = BTreeMap::new(); - for (m, &at) in &inputs.ingestion_context_of_shareable { - readers_of_ingestion - .entry(at) - .or_default() - .push(inputs.context_of_shareable[m]); - } - // 1. Coverage. - milp.builder.scope = Scope::Queries; - for (reference, (query, _)) in output_refs.iter().enumerate() { - let sum: Expression = inputs.output_candidates[reference] - .iter() - .filter_map(|out| milp.choice[*query].get(out).copied()) - .map(Expression::from) - .sum(); - milp.builder.row("1 coverage", constraint!(sum == 1.0)); - } - for context in 0..inputs.contexts.len() { - milp.builder.scope = milp.scope_of(context); - let data = &inputs.contexts[context]; - // 2. One candidate per slot. - let mut per_slot: BTreeMap> = BTreeMap::new(); - for (&option, &variable) in &milp.choice[context] { - per_slot - .entry(inputs.slot[option.0]) - .or_default() - .push(variable); - } - for variables in per_slot.values().filter(|v| v.len() > 1) { - let sum: Expression = variables.iter().copied().map(Expression::from).sum(); - milp.builder - .row("2 one candidate per slot", constraint!(sum <= 1.0)); - } - // 3. Validity. - let choices = milp.choice[context].clone(); - let producers_sum = |reader: PhysicalCandidateId, input: usize| -> Expression { - inputs.producer_candidates_per_input[reader.0][input] - .iter() - .filter(|producer| inputs.slot[producer.0] != inputs.slot[reader.0]) - .filter_map(|producer| choices.get(producer).copied()) - .map(Expression::from) - .sum() - }; - for (&option, &variable) in &choices { - let kind = inputs.kind[option.0]; - if data.computes(kind) { - for input in 0..inputs.producer_candidates_per_input[option.0].len() { - let sum = producers_sum(option, input); - milp.builder.row("3 validity", constraint!(sum >= variable)); - } - } else { - match kind { - RunKind::Shareable => { - let selected = milp.selected[&option]; - milp.builder - .row("3 validity (boundary)", constraint!(variable <= selected)); - } - RunKind::Ingestion => { - let source = milp - .ingestion_context_read_by(context) - .and_then(|ingestion| milp.choice[ingestion].get(&option).copied()); - match source { - Some(source) => milp - .builder - .row("3 validity (boundary)", constraint!(variable <= source)), - None => milp - .builder - .row("3 validity (boundary)", constraint!(variable <= 0.0)), - } - } - RunKind::PerContext => milp - .builder - .row("3 validity (boundary)", constraint!(variable <= 0.0)), - } - } - } - if let ContextKind::Shared(own) = data.kind { - let selected = milp.selected[&own]; - for input in 0..inputs.producer_candidates_per_input[own.0].len() { - let sum: Expression = inputs.producer_candidates_per_input[own.0][input] - .iter() - .filter_map(|producer| choices.get(producer).copied()) - .map(Expression::from) - .sum(); - milp.builder.row("3 validity", constraint!(sum >= selected)); - } - } - // The slot each multi-slot input reads: one, and only a slot the - // context gives. - let links: Vec<(InputOf, SlotLinks)> = milp - .slot_links - .range((context, PhysicalCandidateId(0), 0)..(context + 1, PhysicalCandidateId(0), 0)) - .map(|(&(_, reader, input), links)| ((reader, input), links.clone())) - .collect(); - for ((reader, input), links) in links { - let used = choices[&reader]; - let sum: Expression = links.iter().map(|&(_, link)| Expression::from(link)).sum(); - milp.builder - .row("3 validity (slot links)", constraint!(sum == used)); - for (slot, link) in links { - let given: Expression = inputs.producer_candidates_per_input[reader.0][input] - .iter() - .filter(|producer| inputs.slot[producer.0] == slot) - .filter_map(|producer| choices.get(producer).copied()) - .map(Expression::from) - .sum(); - milp.builder - .row("3 validity (slot links)", constraint!(link <= given)); - } - } - // 4. Used only if read. - for (&option, &variable) in &choices { - let mut reads = Expression::from(0.0); - for reader in milp.readers[context].get(&option).into_iter().flatten() { - match *reader { - Reader::Candidate(reader, input) => { - match milp.slot_links.get(&(context, reader, input)) { - Some(links) => { - let slot = inputs.slot[option.0]; - for &(at, link) in links { - if at == slot { - reads += link; - } - } - } - None => reads += choices[&reader], - } - } - Reader::Output(_) => reads += 1.0, - Reader::OwnInput(_) => { - if let ContextKind::Shared(own) = data.kind { - reads += milp.selected[&own]; - } - } - } - } - // A candidate of `I_r` is also read by the contexts of the - // ingestion-time shareable candidates of refresh window `r`. - for &other in readers_of_ingestion.get(&context).into_iter().flatten() { - if let Some(&read) = milp.choice[other].get(&option) { - reads += read; - } - } - milp.builder - .row("4 used only if read", constraint!(variable <= reads)); - } - } - // 5. Sharing. - milp.builder.scope = Scope::SharedContexts; - let mut reads_of: BTreeMap = BTreeMap::new(); - for choices in &milp.choice { - for (&option, &variable) in choices { - if inputs.kind[option.0] == RunKind::Shareable { - *reads_of.entry(option).or_default() += variable; - } - } - } - let selected = milp.selected.clone(); - for (shareable, selected) in selected { - let reads = reads_of.remove(&shareable).unwrap_or_default(); - milp.builder - .row("5 sharing", constraint!(selected <= reads)); - } - milp.builder.scope = Scope::Global; -} - -/// Order ranks on the links inside each cyclic component of a context's -/// links between candidates it computes (none when the slot graph is -/// acyclic). -fn acyclic_rows(milp: &mut SelectionMilp<'_>) { - for context in 0..milp.choice.len() { - milp.builder.scope = milp.scope_of(context); - let computed: Vec = milp.choice[context] - .keys() - .copied() - .filter(|&c| milp.computes(context, c)) - .collect(); - let index: HashMap = - computed.iter().enumerate().map(|(i, &c)| (c, i)).collect(); - let mut edges: Vec<(usize, usize)> = Vec::new(); - for (&producer, readers) in &milp.readers[context] { - let Some(&from) = index.get(&producer) else { - continue; - }; - for reader in readers { - if let Reader::Candidate(reader, _) = reader { - edges.push((from, index[reader])); - } - } - } - let components = strongly_connected_components(computed.len(), &edges); - let mut rank: HashMap = HashMap::new(); - for &(producer, reader) in &edges { - let size = components.size_of(reader); - if components.of[producer] != components.of[reader] || size < 2 { - continue; - } - let n = size as f64; - let [producer_rank, reader_rank] = [producer, reader].map(|node| { - *rank - .entry(node) - .or_insert_with(|| milp.builder.continuous("acyclic ranks", Some(n - 1.0))) - }); - // Both used: the reader ranks above the producer. (Within one - // context a used reader reads the used candidate of the slot.) - let (p, r) = ( - milp.choice[context][&computed[producer]], - milp.choice[context][&computed[reader]], - ); - milp.builder.row( - "acyclic ranks", - constraint!(producer_rank + 1.0 <= reader_rank + n * (2.0 - p - r)), - ); - } - } - milp.builder.scope = Scope::Global; -} - -/// Order ranks across contexts: on the graph whose nodes are the shareable -/// candidates' own runs and the runs of each `I_r`, with an edge from a run -/// to each shareable candidate or candidate of an `I_r` its context may -/// read (for a run of `I_r`, its producers there). Rows 3–5 alone admit a -/// cycle of reads across contexts that justifies itself with no output -/// above it (two shareable candidates of one memo group, each listed among -/// the other's producers); only edges inside a cyclic component get a row. -fn acyclic_context_rows(milp: &mut SelectionMilp<'_>) { - let inputs = milp.inputs; - // Nodes: (context of the run, candidate). - let mut nodes: Vec<(usize, PhysicalCandidateId)> = Vec::new(); - for (&m, &context) in &inputs.context_of_shareable { - nodes.push((context, m)); - } - for (context, data) in inputs.contexts.iter().enumerate() { - if let ContextKind::Ingestion(_) = data.kind { - for &c in milp.choice[context].keys() { - if milp.computes(context, c) { - nodes.push((context, c)); - } - } - } - } - let index: HashMap<(usize, PhysicalCandidateId), usize> = - nodes.iter().enumerate().map(|(i, &run)| (run, i)).collect(); - // The node a read of `option` in `context` resolves to. - let node_read = |context: usize, option: PhysicalCandidateId| -> Option { - let run_context = match inputs.kind[option.0] { - RunKind::Shareable => milp.context_of(option)?, - RunKind::Ingestion => milp.ingestion_context_read_by(context)?, - RunKind::PerContext => return None, - }; - index.get(&(run_context, option)).copied() - }; - // Edges with their activity: `(from, to, indicators that must all be 1)`. - let mut edges: Vec<(usize, usize, Vec)> = Vec::new(); - for (from, &(context, candidate)) in nodes.iter().enumerate() { - if let ContextKind::Ingestion(_) = inputs.contexts[context].kind { - let reader = milp.choice[context][&candidate]; - for producers in &inputs.producer_candidates_per_input[candidate.0] { - for &producer in producers { - if let (Some(to), Some(&read)) = ( - node_read(context, producer), - milp.choice[context].get(&producer), - ) { - if to != from { - edges.push((from, to, vec![reader, read])); - } - } - } - } - } else { - for (&option, &read) in &milp.choice[context] { - if let Some(to) = node_read(context, option) { - edges.push((from, to, vec![read])); - } - } - } - } - let plain: Vec<(usize, usize)> = edges.iter().map(|&(from, to, _)| (from, to)).collect(); - let components = strongly_connected_components(nodes.len(), &plain); - milp.builder.scope = Scope::SharedContexts; - let mut rank: HashMap = HashMap::new(); - for (from, to, active) in edges { - let size = components.size_of(from); - if components.of[from] != components.of[to] || size < 2 { - continue; - } - let n = size as f64; - let [reader_rank, read_rank] = [from, to].map(|node| { - *rank.entry(node).or_insert_with(|| { - milp.builder - .continuous("acyclic ranks (contexts)", Some(n - 1.0)) - }) - }); - let slack = active.len() as f64; - let on: Expression = active.into_iter().map(Expression::from).sum(); - milp.builder.row( - "acyclic ranks (contexts)", - constraint!(read_rank + 1.0 <= reader_rank + n * (slack - on)), - ); - } - milp.builder.scope = Scope::Global; -} - -/// The strongly connected components of a directed graph. -pub(crate) struct Components { - /// Per node, its component. - pub(crate) of: Vec, - /// Per component, its number of nodes. - pub(crate) sizes: Vec, -} - -impl Components { - pub(crate) fn size_of(&self, node: usize) -> usize { - self.sizes[self.of[node]] - } -} - -/// The strongly connected components of the graph on nodes `0..count` with -/// `edges` `(from, to)` (Tarjan's algorithm, without recursion). -pub(crate) fn strongly_connected_components(count: usize, edges: &[(usize, usize)]) -> Components { - let mut successors = vec![Vec::new(); count]; - for &(from, to) in edges { - successors[from].push(to); - } - const UNVISITED: usize = usize::MAX; - let mut order = vec![UNVISITED; count]; - let mut lowest = vec![0; count]; - let mut on_stack = vec![false; count]; - let mut stack = Vec::new(); - let mut of = vec![UNVISITED; count]; - let mut sizes = Vec::new(); - let mut next = 0; - for root in 0..count { - if order[root] != UNVISITED { - continue; - } - let mut path = vec![(root, 0)]; - order[root] = next; - lowest[root] = next; - next += 1; - stack.push(root); - on_stack[root] = true; - while let Some(&mut (node, ref mut successor)) = path.last_mut() { - if let Some(&to) = successors[node].get(*successor) { - *successor += 1; - if order[to] == UNVISITED { - order[to] = next; - lowest[to] = next; - next += 1; - stack.push(to); - on_stack[to] = true; - path.push((to, 0)); - } else if on_stack[to] { - lowest[node] = lowest[node].min(order[to]); - } - continue; - } - path.pop(); - if let Some(&(parent, _)) = path.last() { - lowest[parent] = lowest[parent].min(lowest[node]); - } - if lowest[node] == order[node] { - let component = sizes.len(); - let mut size = 0; - loop { - let member = stack.pop().expect("the node is on the stack"); - on_stack[member] = false; - of[member] = component; - size += 1; - if member == node { - break; - } - } - sizes.push(size); - } - } - } - Components { of, sizes } -} - -/// The shareable candidates in order: each before every shareable -/// candidate its context may read (readers first), so `a[q, m]` exists -/// before the reach rows of what `m` reads. A cycle among contexts is -/// broken at the first candidate met. -pub(crate) fn shareables_readers_first(milp: &SelectionMilp<'_>) -> Vec { - let inputs = milp.inputs; - let mut order = Vec::new(); - let mut state: BTreeMap = BTreeMap::new(); - // Post-order over "m reads c" (children = what m reads), reversed. - for &start in inputs.context_of_shareable.keys() { - if state.contains_key(&start) { - continue; - } - let mut stack: Vec<(PhysicalCandidateId, bool)> = vec![(start, false)]; - while let Some((node, done)) = stack.pop() { - if done { - state.insert(node, 2); - order.push(node); - continue; - } - if state.contains_key(&node) { - continue; - } - state.insert(node, 1); - stack.push((node, true)); - let context = inputs.context_of_shareable[&node]; - for &option in &inputs.contexts[context].options { - if inputs.kind[option.0] == RunKind::Shareable && !state.contains_key(&option) { - stack.push((option, false)); - } - } - } - } - order.reverse(); - order -} - -/// Row 6: reach, from below for every `(q, m)`, from above where a reader -/// constraint counts queries. -fn reach_rows(milp: &mut SelectionMilp<'_>) { - let inputs = milp.inputs; - milp.builder.scope = Scope::SharedContexts; - let order = shareables_readers_first(milp); - // Which queries may depend on each shareable candidate: read in their - // context, or in the context of one they may depend on. - let mut may_reach: BTreeMap> = BTreeMap::new(); - for query in 0..milp.query_count() { - for &option in &inputs.contexts[query].options { - if inputs.kind[option.0] == RunKind::Shareable { - may_reach.entry(option).or_default().insert(query); - } - } - } - for &shareable in &order { - let queries = may_reach.get(&shareable).cloned().unwrap_or_default(); - let context = inputs.context_of_shareable[&shareable]; - for &option in &inputs.contexts[context].options { - if inputs.kind[option.0] == RunKind::Shareable { - may_reach - .entry(option) - .or_default() - .extend(queries.iter().copied()); - } - } - } - for &shareable in &order { - for &query in may_reach.get(&shareable).into_iter().flatten() { - let reach = milp.builder.continuous("a (reach)", Some(1.0)); - milp.reach.insert((query, shareable), reach); - } - } - for &shareable in &order { - for &query in may_reach.get(&shareable).into_iter().flatten() { - let reach = milp.reach[&(query, shareable)]; - if let Some(&direct) = milp.choice[query].get(&shareable) { - milp.builder - .row("6 reach from below", constraint!(reach >= direct)); - } - for &reader in milp.shared_readers.get(&shareable).into_iter().flatten() { - let Some(&above) = milp.reach.get(&(query, reader)) else { - continue; - }; - let context = inputs.context_of_shareable[&reader]; - let read = milp.choice[context][&shareable]; - milp.builder.row( - "6 reach from below", - constraint!(reach >= above + read - 1.0), - ); - } - } - } - // From above, where needed. - let exact = shareables_needing_exact_reach(milp); - for &shareable in &order { - if !exact.contains(&shareable) { - continue; - } - for &query in may_reach.get(&shareable).into_iter().flatten() { - let reach = milp.reach[&(query, shareable)]; - let mut sources = Expression::from(0.0); - if let Some(&direct) = milp.choice[query].get(&shareable) { - sources += direct; - } - for &reader in milp - .shared_readers - .get(&shareable) - .cloned() - .unwrap_or_default() - .iter() - { - let Some(&above) = milp.reach.get(&(query, reader)) else { - continue; - }; - let context = inputs.context_of_shareable[&reader]; - let read = milp.choice[context][&shareable]; - let through = milp.builder.continuous("y (reach through)", Some(1.0)); - milp.builder - .row("6 reach from above", constraint!(through <= above)); - milp.builder - .row("6 reach from above", constraint!(through <= read)); - sources += through; - } - milp.builder - .row("6 reach from above", constraint!(reach <= sources)); - } - } - milp.builder.scope = Scope::Global; -} - -/// The reader constraints of each candidate. -pub(crate) fn reader_constraints_of_candidates<'a>( - milp: &SelectionMilp<'a>, -) -> BTreeMap> { - let physical: &'a PhysicalASAPMemoCandidates = milp.physical; - let mut of_candidate: BTreeMap> = - BTreeMap::new(); - for constraint in &physical.reader_constraints { - of_candidate - .entry(constraint.candidate()) - .or_default() - .push(constraint); - } - of_candidate -} - -/// The shareable candidates whose `a` must be exact from above: those a -/// counting reader constraint (minimum readers, highest shared -/// materialization) names or whose context holds a constrained candidate, -/// and transitively every shareable candidate whose context reads one. -fn shareables_needing_exact_reach(milp: &SelectionMilp<'_>) -> BTreeSet { - let inputs = milp.inputs; - let constraints = reader_constraints_of_candidates(milp); - let mut stack: Vec = Vec::new(); - for (candidate, constraints) in &constraints { - let counted = constraints.iter().any(|constraint| { - matches!( - constraint, - ReaderConstraint::MinimumQueryReaders { .. } - | ReaderConstraint::HighestSharedQueryMaterialization { .. } - ) - }); - if !counted { - continue; - } - if inputs.kind[candidate.0] == RunKind::Shareable { - stack.push(*candidate); - // Highest shared materialization reads the reach of its - // reading contexts. - stack.extend(milp.shared_readers.get(candidate).into_iter().flatten()); - } - for (context, data) in inputs.contexts.iter().enumerate() { - if let ContextKind::Shared(own) = data.kind { - if milp.choice[context].contains_key(candidate) { - stack.push(own); - } - } - } - } - let mut exact = BTreeSet::new(); - while let Some(shareable) = stack.pop() { - if exact.insert(shareable) { - stack.extend(milp.shared_readers.get(&shareable).into_iter().flatten()); - } - } - exact -} - -/// The queries of `query_outputs` buckets (one per query, each its output -/// references). -fn bucket_queries(query_of_reference: &[usize], query_outputs: &[Vec]) -> Vec { - query_outputs - .iter() - .filter_map(|bucket| { - bucket - .first() - .map(|&reference| query_of_reference[reference]) - }) - .collect() -} - -/// Row 9: reader constraints, on every node of the constrained candidate. -/// A node in a query's context is reached by that query alone; a node in -/// shareable `m`'s context, and `m` itself, by the queries that depend on -/// `m`. Reads into an `I_r` and per-output-reference reach inside one query -/// are not modeled: the constraint is then applied to every reference of -/// the query (conservative, counted as inexact). -fn reader_constraint_rows(milp: &mut SelectionMilp<'_>) { - let inputs = milp.inputs; - let query_of_reference: Vec = output_references(&milp.physical.query_roots) - .into_iter() - .map(|(query, _)| query) - .collect(); - let constraints = reader_constraints_of_candidates(milp); - let query_count = milp.query_count(); - milp.builder.scope = Scope::Global; - let reach_count = |milp: &SelectionMilp<'_>, queries: &[usize], m: PhysicalCandidateId| { - let mut count = Expression::from(0.0); - let mut most = 0.0f64; - for &query in queries { - if let Some(&reach) = milp.reach.get(&(query, m)) { - count += reach; - most += 1.0; - } - } - (count, most) - }; - for (candidate, constraints) in constraints { - let shareable = inputs.kind[candidate.0] == RunKind::Shareable; - // Its uses: (context, p) per context holding it as an option it - // computes. - let uses: Vec<(usize, Variable)> = (0..milp.choice.len()) - .filter(|&context| milp.computes(context, candidate)) - .filter_map(|context| { - milp.choice[context] - .get(&candidate) - .map(|&variable| (context, variable)) - }) - .collect(); - for constraint in constraints { - match constraint { - ReaderConstraint::MinimumQueryReaders { - query_outputs, - minimum, - .. - } => { - let queries = bucket_queries(&query_of_reference, query_outputs); - let minimum = *minimum as f64; - if shareable { - if let Some(&selected) = milp.selected.get(&candidate) { - let (count, _) = reach_count(milp, &queries, candidate); - milp.builder.row( - "9 reader constraints", - constraint!(count >= minimum * selected), - ); - } - } - for &(context, used) in &uses { - let count = match inputs.contexts[context].kind { - ContextKind::Query(_) => Expression::from(1.0), - ContextKind::Shared(m) => reach_count(milp, &queries, m).0, - ContextKind::Ingestion(_) => { - milp.note_inexact( - "minimum readers of a candidate of an I_r (forbidden)", - ); - Expression::from(0.0) - } - }; - milp.builder - .row("9 reader constraints", constraint!(count >= minimum * used)); - } - } - ReaderConstraint::HighestSharedQueryMaterialization { query_outputs, .. } => { - let queries = bucket_queries(&query_of_reference, query_outputs); - if shareable { - let Some(&selected) = milp.selected.get(&candidate) else { - continue; - }; - // Allowed: a read in a query's context (its readers - // there are reached by that query alone), or a read - // in the context of `m` that at most one query - // reaches. - let mut allowed = Expression::from(0.0); - for query in 0..query_count { - if let Some(&read) = milp.choice[query].get(&candidate) { - allowed += read; - } - } - for &reader in milp - .shared_readers - .get(&candidate) - .cloned() - .unwrap_or_default() - .iter() - { - let context = inputs.context_of_shareable[&reader]; - let read = milp.choice[context][&candidate]; - let (count, most) = reach_count(milp, &queries, reader); - let single = milp.builder.binary("g (single-query readers)"); - milp.builder - .row("9 reader constraints", constraint!(single <= read)); - let slack = (most - 1.0).max(0.0); - milp.builder.row( - "9 reader constraints", - constraint!(count <= 1.0 + slack * (1.0 - single)), - ); - allowed += single; - } - milp.builder - .row("9 reader constraints", constraint!(selected <= allowed)); - } - for &(context, used) in &uses { - match inputs.contexts[context].kind { - ContextKind::Query(_) => {} - ContextKind::Shared(m) => { - let (count, most) = reach_count(milp, &queries, m); - let slack = (most - 1.0).max(0.0); - milp.builder.row( - "9 reader constraints", - constraint!(count <= 1.0 + slack * (1.0 - used)), - ); - } - ContextKind::Ingestion(_) => { - milp.note_inexact( - "highest shared materialization in an I_r (forbidden)", - ); - milp.builder - .row("9 reader constraints", constraint!(used <= 0.0)); - } - } - } - } - ReaderConstraint::AllowedOutputs { outputs, .. } => { - for query in 0..query_count { - let references: Vec = (0..query_of_reference.len()) - .filter(|&reference| query_of_reference[reference] == query) - .collect(); - let disallowed = references - .iter() - .filter(|reference| !outputs.contains(reference)) - .count(); - if disallowed == 0 { - continue; - } - if disallowed < references.len() { - milp.note_inexact("AllowedOutputs over part of a query's outputs"); - } - forbid_reach(milp, query, candidate, &uses); - } - } - ReaderConstraint::IncompatibleOutputs { left, right, .. } => { - let (left, right) = (query_of_reference[*left], query_of_reference[*right]); - if left == right { - milp.note_inexact("IncompatibleOutputs within one query (forbidden)"); - forbid_reach(milp, left, candidate, &uses); - continue; - } - if shareable { - if let (Some(&l), Some(&r)) = ( - milp.reach.get(&(left, candidate)), - milp.reach.get(&(right, candidate)), - ) { - milp.builder - .row("9 reader constraints", constraint!(l + r <= 1.0)); - } - } - for &(context, used) in &uses { - if let ContextKind::Shared(m) = inputs.contexts[context].kind { - if let (Some(&l), Some(&r)) = - (milp.reach.get(&(left, m)), milp.reach.get(&(right, m))) - { - milp.builder - .row("9 reader constraints", constraint!(l + r + used <= 2.0)); - } - } - } - } - } - } - } -} - -/// Query `query` reaches no node of `candidate`. -fn forbid_reach( - milp: &mut SelectionMilp<'_>, - query: usize, - candidate: PhysicalCandidateId, - uses: &[(usize, Variable)], -) { - if let Some(&reach) = milp.reach.get(&(query, candidate)) { - milp.builder - .row("9 reader constraints", constraint!(reach <= 0.0)); - } else if let Some(&read) = milp.choice[query].get(&candidate) { - milp.builder - .row("9 reader constraints", constraint!(read <= 0.0)); - } - for &(context, used) in uses { - match milp.inputs.contexts[context].kind { - ContextKind::Query(q) if q == query => { - milp.builder - .row("9 reader constraints", constraint!(used <= 0.0)); - } - ContextKind::Query(_) => {} - ContextKind::Shared(m) => { - if let Some(&reach) = milp.reach.get(&(query, m)) { - milp.builder - .row("9 reader constraints", constraint!(reach + used <= 1.0)); - } - } - ContextKind::Ingestion(_) => { - milp.note_inexact("a reader constraint on a candidate of an I_r (forbidden)"); - milp.builder - .row("9 reader constraints", constraint!(used <= 0.0)); - } - } - } -} - -/// Query `query`'s latency bound: the smaller of its `max_latency_ms` and -/// its fixed repeat interval (`checks::latency_violation`). -pub(crate) fn latency_bound_ms(entry: &QueryWorkloadEntry) -> Option { - let interval = entry.recurrence.fixed_interval_ms().map(|ms| ms as f64); - match (entry.requirements.max_latency_ms, interval) { - (Some(bound), Some(interval)) => Some(bound.min(interval)), - (bound, interval) => bound.or(interval), - } -} - -/// The latency terms of context `context`'s computed runs (and of `m`'s -/// own run in its own context): `Σ d · indicator`, and the most it can be. -fn context_work(milp: &mut SelectionMilp<'_>, context: usize) -> (Expression, f64) { - let mut work = Expression::from(0.0); - let mut most = 0.0; - let mut runs: Vec = milp.choice[context] - .keys() - .copied() - .filter(|&c| milp.computes(context, c)) - .collect(); - if let ContextKind::Shared(own) = milp.inputs.contexts[context].kind { - runs.push(own); - } - // Per slot, only the slowest candidate can count (one per slot). - let mut most_per_slot: BTreeMap = BTreeMap::new(); - for candidate in runs { - let mut run_most = 0.0; - for term in milp.run_terms(context, candidate) { - if term.numbers.latency_ms > 0.0 { - let units = term.numbers.latency_ms / milp.ms_per_latency_unit; - work += units * term.indicator.clone(); - run_most = f64::max(run_most, units); - } - } - let slot = milp.slot(candidate); - let entry = most_per_slot.entry(slot).or_default(); - *entry = entry.max(run_most); - } - most += most_per_slot.values().sum::(); - (work, most) -} - -/// Row 7: each query's latency, `asap_cost::plan_cost`'s definition: its -/// request time, plus the work of every query-time run it reaches, each -/// once (its own context's, and each query-time shareable candidate's run -/// and context through `T[q, m]`), plus the longest compaction among the -/// ingestion-time states it reaches. -fn latency_rows(milp: &mut SelectionMilp<'_>) -> Result<(), SelectionMilpError> { - let inputs = milp.inputs; - let query_count = milp.query_count(); - let request_ms = milp.cost_inputs.measured_operation_costs.ns_per_evaluation * 1e-6; - // The latency unit: the most any query's latency can be (every - // candidate's work once, plus the request) is about 1,000 units. - let most_ms = request_ms - + inputs - .candidate_data - .iter() - .filter_map(|data| data.usage.as_ref().ok()) - .map(|usage| usage.own_latency_ms) - .sum::(); - if most_ms.is_finite() && most_ms > 0.0 { - milp.ms_per_latency_unit = most_ms / 1_000.0; - } - let unit = milp.ms_per_latency_unit; - let request = request_ms / unit; - // `L[m]` per query-time shareable candidate. - milp.builder.scope = Scope::SharedContexts; - let mut shared_latency: BTreeMap = BTreeMap::new(); - let shareables: Vec<(PhysicalCandidateId, usize)> = inputs - .context_of_shareable - .iter() - .map(|(&m, &context)| (m, context)) - .collect(); - for (m, context) in shareables { - if !milp.runs_at_query_time(context) { - continue; - } - let (work, most) = context_work(milp, context); - milp.builder.scope = Scope::SharedContexts; - if most <= 0.0 { - continue; - } - let latency = milp.builder.continuous("L (latency of m)", None); - milp.builder.row("7 latency", constraint!(latency >= work)); - shared_latency.insert(m, (latency, most)); - } - let compaction = cost_terms::compaction_ms(milp); - for query in 0..query_count { - let (mut work, mut most) = context_work(milp, query); - milp.builder.scope = Scope::Queries; - work += request; - most += request; - let mut compactions = Vec::new(); - let mut longest_compaction = 0.0f64; - for (&m, &(shared, shared_most)) in &shared_latency { - let Some(&reach) = milp.reach.get(&(query, m)) else { - continue; - }; - let through = milp.builder.continuous("T (latency of m in q)", None); - milp.builder.row( - "7 latency", - constraint!(through >= shared - shared_most * (1.0 - reach)), - ); - work += through; - most += shared_most; - } - for (&m, &ms) in &compaction { - if let Some(&reach) = milp.reach.get(&(query, m)) { - if ms > 0.0 { - compactions.push((ms / unit, reach)); - longest_compaction = longest_compaction.max(ms / unit); - } - } - } - if !compactions.is_empty() { - let longest = milp.builder.continuous("K (compaction)", None); - for (ms, reach) in compactions { - milp.builder - .row("7 latency", constraint!(longest >= ms * reach)); - } - work += longest; - most += longest_compaction; - } - let latency = milp.builder.continuous("L (latency of q)", None); - let bound = latency_bound_ms(&milp.entries[query]) - .map(|bound| bound / unit) - .filter(|&bound| bound < most); - // A binding bound's rows are divided by it, so HiGHS's absolute - // feasibility tolerance is relative to the bound. - let per_ms = bound.map_or(1.0, |bound| 1.0 / bound.max(1e-9)); - milp.builder - .row("7 latency", constraint!(per_ms * latency >= per_ms * work)); - if let Some(bound) = bound { - milp.builder - .row("7 latency", constraint!(per_ms * latency <= per_ms * bound)); - } - let bound = latency_bound_ms(&milp.entries[query]).map(|bound| bound / unit); - milp.latency.push(latency); - milp.latency_max - .push(bound.map_or(most, |bound| bound.min(most))); - } - milp.builder.scope = Scope::Global; - Ok(()) -} - -/// Row 8: accuracy (A1, revised 2026-10-31), per run `q` depends on: a run -/// unusable for `q` is never depended on, two whose pair is excluded are -/// not depended on together, and the shares depended on are at most each -/// such run's `parts`. -fn accuracy_rows(milp: &mut SelectionMilp<'_>) { - let inputs = milp.inputs; - milp.builder.scope = Scope::Global; - let approximate_runs: Vec<(usize, PhysicalCandidateId)> = milp - .runs() - .into_iter() - .filter(|&(_, candidate)| inputs.candidate_data[candidate.0].approximate) - .collect(); - for query in 0..milp.query_count() { - let exact_target = milp.entries[query].requirements.accuracy.is_exact(); - // (candidate, dep, parts) per run `q` may depend on. - let mut counted: Vec<(PhysicalCandidateId, Expression, usize)> = Vec::new(); - // Where each counted run gives its value: (context, slot), `None` - // for a shareable candidate's own run. One value per place. - let mut places: BTreeSet<(usize, Option)> = BTreeSet::new(); - for &(context, candidate) in &approximate_runs { - let Some(weight) = inputs.accuracy_weights[query].get(&candidate).copied() else { - continue; - }; - let dep: Option = match inputs.contexts[context].kind { - ContextKind::Query(q) if q == query => { - milp.choice[context].get(&candidate).map(|&p| p.into()) - } - ContextKind::Query(_) => None, - ContextKind::Shared(own) if own == candidate => { - milp.reach.get(&(query, candidate)).map(|&a| a.into()) - } - ContextKind::Shared(own) => match milp.reach.get(&(query, own)) { - Some(&above) => { - let p = milp.choice[context][&candidate]; - Some( - milp.builder - .and_at_least( - "y (depends on a run in m)", - &[above.into(), p.into()], - ) - .into(), - ) - } - None => None, - }, - ContextKind::Ingestion(_) => { - milp.note_inexact("an approximate candidate of an I_r (forbidden)"); - let p = milp.choice[context][&candidate]; - milp.builder.row("8 accuracy", constraint!(p <= 0.0)); - None - } - }; - let Some(dep) = dep else { - continue; - }; - let usable = match weight { - _ if exact_target => false, - AccuracyWeight::One => true, - AccuracyWeight::Unusable(_) => false, - }; - if usable { - places.insert(match inputs.contexts[context].kind { - ContextKind::Shared(own) if own == candidate => (context, None), - _ => (context, Some(inputs.slot[candidate.0])), - }); - counted.push(( - candidate, - dep, - inputs.candidate_data[candidate.0].accuracy_budget_parts, - )); - } else { - milp.builder.row("8 accuracy", constraint!(dep <= 0.0)); - } - } - let mut pairs: Vec<(Expression, Expression)> = Vec::new(); - for &(first, second) in &inputs.accuracy_excluded_pairs[query] { - for (_, one, _) in counted.iter().filter(|(c, _, _)| *c == first) { - for (_, other, _) in counted.iter().filter(|(c, _, _)| *c == second) { - pairs.push((one.clone(), other.clone())); - } - } - } - for (one, other) in pairs { - milp.builder - .row("8 accuracy (pair)", constraint!(one + other <= 1.0)); - } - // The most values `q` can depend on together. - let most = if tests_loose_big_m() { - counted.len() as f64 - } else { - places.len() as f64 - }; - if counted.iter().all(|&(_, _, parts)| parts as f64 >= most) { - continue; - } - let shares = milp.builder.continuous("N (approximate values)", None); - let sum: Expression = counted.iter().map(|(_, dep, _)| dep.clone()).sum(); - milp.builder.row("8 accuracy", constraint!(shares == sum)); - let mut by_parts: BTreeMap> = BTreeMap::new(); - for (_, dep, parts) in &counted { - if (*parts as f64) < most { - by_parts.entry(*parts).or_default().push(dep.clone()); - } - } - for (parts, deps) in by_parts { - let reads = milp - .builder - .continuous("A (depends at a parts value)", Some(1.0)); - for dep in deps { - milp.builder.row("8 accuracy", constraint!(reads >= dep)); - } - let slack = most - parts as f64; - milp.builder.row( - "8 accuracy", - constraint!(shares <= parts as f64 + slack * (1.0 - reads)), - ); - } - } -} - -/// One independent part of the model (#1095 §5, 3a): a connected component -/// of the graph joining the variables of each row, or all components -/// without a query together. Parts share no variable and no row, and the -/// objective is a sum over variables, so solving each alone and joining -/// their solutions solves the whole model. -#[derive(Debug, Clone, PartialEq, Eq)] -pub(crate) struct IndependentPart { - /// The queries whose output references it covers, ascending. - pub(crate) queries: Vec, - /// Its columns (variables in creation order), ascending. - pub(crate) columns: Vec, - /// Its rows, ascending. - pub(crate) rows: Vec, -} - -/// The independent parts of `milp`, in order of their first query (the -/// part without a query last). Queries fall into one part when they may -/// share a shareable candidate, a run of an `I_r`, a source's raw retention or -/// anything else a row joins. -pub(crate) fn independent_parts(milp: &SelectionMilp<'_>) -> Vec { - let column_of: HashMap = milp - .builder - .variables - .iter_variables_with_def() - .enumerate() - .map(|(column, (variable, _))| (variable, column)) - .collect(); - let count = column_of.len(); - // Union-find over the columns, joined along each row. - let mut parent: Vec = (0..count).collect(); - fn root(parent: &mut [usize], mut node: usize) -> usize { - while parent[node] != node { - parent[node] = parent[parent[node]]; - node = parent[node]; - } - node - } - for row in &milp.builder.rows { - let mut first: Option = None; - for (variable, _) in row.expression().linear_coefficients() { - let column = root(&mut parent, column_of[&variable]); - match first { - None => first = Some(column), - Some(known) if known != column => parent[column] = known, - Some(_) => {} - } - } - } - let output_refs = output_references(&milp.physical.query_roots); - let mut query_of_component: BTreeMap = BTreeMap::new(); - for (reference, &(query, _)) in output_refs.iter().enumerate() { - for out in &milp.inputs.output_candidates[reference] { - if let Some(variable) = milp.choice[query].get(out) { - let component = root(&mut parent, column_of[variable]); - query_of_component.entry(component).or_insert(query); - } - } - } - // The part of each component: its first query's, or the last. - let part_of = |component: usize| query_of_component.get(&component).copied(); - let mut parts: BTreeMap, IndependentPart> = BTreeMap::new(); - let mut part_of_column = Vec::with_capacity(count); - for column in 0..count { - let key = part_of(root(&mut parent, column)); - part_of_column.push(key); - parts - .entry(key) - .or_insert_with(|| IndependentPart { - queries: Vec::new(), - columns: Vec::new(), - rows: Vec::new(), - }) - .columns - .push(column); - } - for (reference, &(query, _)) in output_refs.iter().enumerate() { - let Some(variable) = milp.inputs.output_candidates[reference] - .iter() - .find_map(|out| milp.choice[query].get(out)) - else { - continue; - }; - let key = part_of_column[column_of[variable]]; - let queries = &mut parts.get_mut(&key).expect("its column's part").queries; - if !queries.contains(&query) { - queries.push(query); - } - } - for (index, row) in milp.builder.rows.iter().enumerate() { - // A row without a variable is a constant check: kept in the part - // without a query when there is one (`None` sorts first), else in - // the first query's part. - let key = row - .expression() - .linear_coefficients() - .next() - .map(|(variable, _)| part_of_column[column_of[&variable]]) - .unwrap_or_else(|| *parts.keys().next().expect("a part")); - parts.get_mut(&key).expect("a part").rows.push(index); - } - // `None` sorts first; the part without a query goes last. - let mut parts: Vec<(Option, IndependentPart)> = parts.into_iter().collect(); - parts.sort_by_key(|(key, _)| key.unwrap_or(usize::MAX)); - parts - .into_iter() - .map(|(_, mut part)| { - part.queries.sort_unstable(); - part - }) - .collect() -} - -/// How one part's solve went, for the report. -#[derive(Debug, Clone, PartialEq, serde::Serialize)] -pub(crate) struct PartSolve { - pub(crate) queries: usize, - pub(crate) variables: usize, - pub(crate) binaries: usize, - pub(crate) rows: usize, - pub(crate) solve_ms: u64, - /// HiGHS's dual bound on the part's objective (cost per second). - pub(crate) dual_bound_per_second: f64, -} - -/// One part's model ready for HiGHS: its variables, rows and objective -/// terms, renamed to its own columns when the model has several parts. -struct PartModel { - variables: ProblemVariables, - rows: Vec, - objective_terms: Vec<(Variable, f64)>, - /// Per column of the part, the column in the whole model. - columns: Vec, - queries: usize, -} - -/// A part solved to proven optimality: its columns' values, and the HiGHS -/// objective and dual bound (cost per second). -struct SolvedPart { - columns: Vec, - objective_per_second: f64, - dual_bound_per_second: f64, - report: PartSolve, -} - -/// Solves `milp` with HiGHS, one solve per independent part -/// ([`independent_parts`]) when `split_into_parts`, each part from the warm -/// start and run to proven optimality (no time limit). -pub(crate) fn solve_selection_milp( - milp: SelectionMilp<'_>, - build_ms: u64, -) -> Result { - let size = milp.size(); - let highs_threads = milp.highs_threads; - let warm_started = Instant::now(); - let completion = super::completion::Completion::new(&milp); - let start = warm_start::warm_start(&milp, &completion); - let warm_start_ms = warm_started.elapsed().as_millis() as u64; - let warm_start_cost_per_second = start.as_ref().map(|start| start.cost_per_second); - #[cfg(feature = "lp-diagnosis")] - let diagnosis = { - let column_of: HashMap = milp - .builder - .variables - .iter_variables_with_def() - .enumerate() - .map(|(column, (variable, _))| (variable, column)) - .collect(); - let start_columns = start.as_ref().map(|start| { - let mut columns = vec![0.0; column_of.len()]; - for &(variable, value) in &start.values { - columns[column_of[&variable]] = value; - } - columns - }); - super::lp_diagnosis::Diagnosis::run(&milp, start_columns.as_deref()) - .map(|diagnosis| (diagnosis, start_columns)) - }; - let parts = if milp.split_into_parts { - independent_parts(&milp) - } else { - Vec::new() - }; - let SelectionMilp { - builder, - choice, - selected, - slot_links, - request_cost_per_second, - objective_terms, - .. - } = milp; - let ModelBuilder { - variables, rows, .. - } = builder; - let column_of: HashMap = variables - .iter_variables_with_def() - .enumerate() - .map(|(column, (variable, _))| (variable, column)) - .collect(); - let column_count = column_of.len(); - let start_columns: Option> = start.map(|start| { - let mut columns = vec![0.0; column_count]; - for (variable, value) in start.values { - columns[column_of[&variable]] = value; - } - columns - }); - let models: Vec = if parts.len() <= 1 { - vec![PartModel { - variables, - rows, - objective_terms, - columns: (0..column_count).collect(), - queries: parts.first().map_or(0, |part| part.queries.len()), - }] - } else { - let definitions: Vec<_> = variables - .iter_variables_with_def() - .map(|(_, definition)| definition.clone()) - .collect(); - let mut part_of_column = vec![0; column_count]; - let mut renamed: Vec> = vec![None; column_count]; - let mut models: Vec = parts - .iter() - .enumerate() - .map(|(index, part)| { - let mut variables = ProblemVariables::new(); - for &column in &part.columns { - part_of_column[column] = index; - renamed[column] = Some(variables.add(definitions[column].clone())); - } - PartModel { - variables, - rows: Vec::with_capacity(part.rows.len()), - objective_terms: Vec::new(), - columns: part.columns.clone(), - queries: part.queries.len(), - } - }) - .collect(); - let rename = |variable: Variable| renamed[column_of[&variable]].expect("in a part"); - for (index, part) in parts.iter().enumerate() { - for &row in &part.rows { - let row = &rows[row]; - let mut expression = Expression::from(row.expression().constant()); - for (variable, coefficient) in row.expression().linear_coefficients() { - expression.add_mul(coefficient, rename(variable)); - } - models[index].rows.push(if row.is_equality() { - constraint!(expression == 0.0) - } else { - constraint!(expression <= 0.0) - }); - } - } - for (variable, coefficient) in objective_terms { - let column = column_of[&variable]; - models[part_of_column[column]] - .objective_terms - .push((rename(variable), coefficient)); - } - models - }; - let mut columns = vec![0.0; column_count]; - let mut solved_parts: Vec = Vec::with_capacity(models.len()); - let started = Instant::now(); - for model in models { - let start = start_columns.as_ref().map(|start| { - model - .columns - .iter() - .map(|&column| start[column]) - .collect::>() - }); - let part_columns = model.columns.clone(); - let solved = solve_part(model, start, highs_threads)?; - for (&column, &value) in part_columns.iter().zip(&solved.columns) { - columns[column] = value; - } - solved_parts.push(solved); - } - let solve_ms = started.elapsed().as_millis() as u64; - let highs_objective: f64 = solved_parts - .iter() - .map(|part| part.objective_per_second) - .sum(); - let dual_bound: f64 = solved_parts - .iter() - .map(|part| part.dual_bound_per_second) - .sum(); - // The objective of the integral plan: the incumbent's binaries, - // rounded, with every continuous variable at its cheapest value (the - // solver's own value carries its tolerances, relative to the scaled - // objective). HiGHS's objective when the completion does not apply. - let objective_value = match completion.complete_columns(&columns) { - Ok(completed) => completed.cost_per_second, - Err(reason) => { - warm_start::debug(|| format!("completion of the incumbent failed: {reason}")); - highs_objective - } - }; - warm_start::debug(|| { - format!("objective: HiGHS {highs_objective:e}, completed {objective_value:e}") - }); - #[cfg(feature = "lp-diagnosis")] - if let Some((diagnosis, start_columns)) = diagnosis { - let incumbent = completion.complete_columns(&columns).ok(); - diagnosis.write( - start_columns.as_deref(), - incumbent - .as_ref() - .map(|completed| completed.columns.as_slice()), - objective_value, - dual_bound, - ); - } - let on = |variable: Variable| columns[column_of[&variable]] > 0.5; - Ok(SelectionMilpSolution { - cost_per_second: objective_value, - request_cost_per_second, - cost_lower_bound_per_second: dual_bound, - chosen: choice - .iter() - .map(|options| { - options - .iter() - .filter(|(_, &variable)| on(variable)) - .map(|(&candidate, _)| candidate) - .collect() - }) - .collect(), - selected: selected - .iter() - .filter(|(_, &variable)| on(variable)) - .map(|(&candidate, _)| candidate) - .collect(), - slot_reads: slot_links - .iter() - .filter_map(|(&key, links)| { - links - .iter() - .find(|(_, link)| on(*link)) - .map(|&(slot, _)| (key, slot)) - }) - .collect(), - size, - parts: solved_parts.into_iter().map(|part| part.report).collect(), - build_ms, - warm_start_ms, - warm_start_cost_per_second, - solve_ms, - }) -} - -/// Solves one part once with HiGHS to proven optimality, from `start` (its -/// columns' values) when given, on `highs_threads` threads. -fn solve_part( - model: PartModel, - start: Option>, - highs_threads: i32, -) -> Result { - let PartModel { - variables, - rows, - objective_terms, - columns: part_columns, - queries, - } = model; - let start_values: Option> = start.as_ref().map(|start| { - variables - .iter_variables_with_def() - .map(|(variable, _)| variable) - .zip(start.iter().copied()) - .collect() - }); - let scale = objective_scale(&objective_terms, start_values.as_deref()); - let objective: Expression = objective_terms - .iter() - .map(|&(variable, coefficient)| coefficient * scale * variable) - .sum(); - let binaries = variables - .iter_variables_with_def() - .filter(|(_, definition)| definition.is_integer()) - .count(); - let row_count = rows.len(); - let problem = variables.minimise(objective).using(highs).with_all(rows); - let mut model = problem - .try_into_inner() - .map_err(|error| SelectionMilpError::Solver(error.to_string()))?; - model.set_option("mip_rel_gap", 0.0); - model.set_option("mip_abs_gap", 0.0); - model.set_option("output_flag", false); - // HiGHS's MIP search is mostly serial; a thread per core costs - // gigabytes of address space on a large host (#1036 under 8 GB). - model.set_option("threads", highs_threads); - if let Some(start) = &start { - model - .try_set_solution(Some(start), None, None, None) - .map_err(|status| SelectionMilpError::Solver(format!("warm start: {status:?}")))?; - } - let started = Instant::now(); - let solved = model - .try_solve() - .map_err(|status| SelectionMilpError::Solver(format!("{status:?}")))?; - let solve_ms = started.elapsed().as_millis() as u64; - use highs::{HighsModelStatus, HighsSolutionStatus}; - // No limit is set, so HiGHS stops only at proven optimality or - // infeasibility; any other status is a solver failure. - match solved.status() { - HighsModelStatus::Optimal => {} - HighsModelStatus::Infeasible | HighsModelStatus::UnboundedOrInfeasible => { - return Err(SelectionMilpError::Infeasible) - } - other => return Err(SelectionMilpError::Solver(format!("{other:?}"))), - } - if solved.primal_solution_status() != HighsSolutionStatus::Feasible { - return Err(SelectionMilpError::Solver( - "HiGHS reported an optimal status without a feasible solution".into(), - )); - } - // HiGHS leaves `mip_dual_bound` at 0 for a part without binaries (an - // LP): its optimal objective is its bound. - let dual_bound = if binaries > 0 { - solved - .double_info_value(c"mip_dual_bound") - .unwrap_or(f64::NAN) - } else { - solved.objective_value() - } / scale; - Ok(SolvedPart { - columns: solved.get_solution().columns().to_vec(), - objective_per_second: solved.objective_value() / scale, - dual_bound_per_second: dual_bound, - report: PartSolve { - queries, - variables: part_columns.len(), - binaries, - rows: row_count, - solve_ms, - dual_bound_per_second: dual_bound, - }, - }) +pub struct QueryId(pub usize); + +/// The baseline variables (section 3). +struct SelectionVariables { + /// $x_v$, by [`PhysicalCandidateId`]. + x: Vec, + /// $y_{u,c,i}$, on admissible edges only. + y: HashMap, + /// $z_{q,v}$, for $v \in G_q$. + z: HashMap<(QueryId, PhysicalCandidateId), Variable>, + /// $t_v$, by [`PhysicalCandidateId`]. + t: Vec, +} + +/// The execution-context variables (section 5), which replace $x$ and the +/// unindexed $y$ of the baseline. +struct ExecutionContextVariables { + /// $x^{\mathrm{mat}}_v$, for an `AcrossEvaluations` candidate. + x_materialized: HashMap, + /// $x_{\kappa,v}$, for a candidate with $\ell$ = None or `OneEvaluation`. + x_in_execution_context: HashMap<(ExecutionContextId, PhysicalCandidateId), Variable>, + /// $y_{\kappa,u,c,i}$, indexed by the consumer's context. + y_in_execution_context: HashMap<(ExecutionContextId, PhysicalEdgeId), Variable>, + /// $h_v$, for an `AcrossEvaluations` producer. + retention_amount: HashMap, +} + +/// The accuracy variables of constraint (K) (section 4), all continuous. +struct AccuracyVariables { + /// $E_c$, by [`PhysicalCandidateId`]. + error_bound: Vec, + /// $F_{c,i}$: the chosen producer's error bound on input $i$ of $c$. + error_bound_of_input: HashMap<(PhysicalCandidateId, usize), Variable>, + /// $D_c$, by [`PhysicalCandidateId`]. + failure_probability: Vec, + /// $H_{c,i}$: the chosen producer's failure probability on input $i$ of $c$. + failure_probability_of_input: HashMap<(PhysicalCandidateId, usize), Variable>, +} + +/// (A) $\sum_{u \in P_{c,i}} y_{u,c,i} = x_c$. +fn input_satisfaction_constraints( + physical: &PhysicalASAPMemoCandidates, + variables: &SelectionVariables, +) -> Vec { + let _ = (physical, variables); + todo!("#1207 PR 6: constraint (A)") +} + +/// (B) $y_{u,c,i} \le x_u$. +fn producer_selection_constraints( + physical: &PhysicalASAPMemoCandidates, + variables: &SelectionVariables, +) -> Vec { + let _ = (physical, variables); + todo!("#1207 PR 6: constraint (B)") +} + +/// (C) $\sum_{v \in G_q} z_{q,v} = 1$; (D) $z_{q,v} \le x_v$. +fn query_coverage_constraints( + physical: &PhysicalASAPMemoCandidates, + variables: &SelectionVariables, +) -> Vec { + let _ = (physical, variables); + todo!("#1207 PR 6: constraints (C) and (D)") +} + +/// (E) $t_u + 1 \le t_c + |\mathcal C|(1 - y_{u,c,i})$. +fn acyclicity_constraints( + physical: &PhysicalASAPMemoCandidates, + variables: &SelectionVariables, +) -> Vec { + let _ = (physical, variables); + todo!("#1207 PR 6: constraint (E)") +} + +/// (F) $x_v \le \sum_{(v,c,i)} y_{v,c,i} + \sum_q z_{q,v}$. +fn utilization_constraints( + physical: &PhysicalASAPMemoCandidates, + variables: &SelectionVariables, +) -> Vec { + let _ = (physical, variables); + todo!("#1207 PR 6: constraint (F)") +} + +/// (R) $h_v \ge \operatorname{need}_{\kappa,v,c,i}\, y_{\kappa,v,c,i}$. +/// +/// Assumptions (decided 2026-11-06): an edge's need is the time horizon +/// the queries it serves read, a constant per edge (a 5m range over 10 s +/// windows needs 30 windows), so (R) is linear; query latency is ignored +/// until the latency design: a result counts as read when its evaluation +/// starts. +fn retention_constraints( + physical: &PhysicalASAPMemoCandidates, + variables: &ExecutionContextVariables, +) -> Vec { + let _ = (physical, variables); + todo!("#1207 PR 7: constraint (R)") +} + +/// (K): $E_c$ and $D_c$ per candidate, big-M on selected edges, and the +/// root checks $E_v \le \varepsilon_q + M_q(1 - z_{q,v})$, +/// $D_v \le \delta_q + (1 - z_{q,v})$. +fn accuracy_constraints( + physical: &PhysicalASAPMemoCandidates, + selection: &SelectionVariables, + accuracy: &AccuracyVariables, +) -> Vec { + let _ = (physical, selection, accuracy); + todo!("#1207 PR 8: constraint (K)") } - -/// The factor the objective is multiplied by for HiGHS: the warm start's -/// cost becomes about 1,000, its largest coefficient at most 10⁷; without -/// a warm start cost, the largest coefficient becomes 1,000. -pub(crate) fn objective_scale(terms: &[(Variable, f64)], start: Option<&[(Variable, f64)]>) -> f64 { - let largest = terms - .iter() - .map(|(_, coefficient)| coefficient.abs()) - .fold(0.0, f64::max); - if !(largest > 0.0 && largest.is_finite()) { - return 1.0; - } - let started: HashMap = start.into_iter().flatten().copied().collect(); - let start_cost: f64 = terms - .iter() - .map(|(variable, coefficient)| coefficient * started.get(variable).copied().unwrap_or(0.0)) - .sum(); - if start_cost > 0.0 && start_cost.is_finite() { - (1_000.0 / start_cost).min(1e7 / largest) - } else { - 1_000.0 / largest - } -} - -/// Whether this thread builds the big-M bounds as before #1095 step 1 -/// (tests only: to compare the tightened bounds with the looser ones). -#[cfg(test)] -pub(crate) fn tests_loose_big_m() -> bool { - tests::LOOSE_BIG_M.with(|loose| loose.get()) -} - -#[cfg(not(test))] -pub(crate) fn tests_loose_big_m() -> bool { - false -} - -#[cfg(test)] -pub(crate) mod tests; diff --git a/crates/plan-selection/src/milp/model/tests.rs b/crates/plan-selection/src/milp/model/tests.rs deleted file mode 100644 index a8ec9f4a0..000000000 --- a/crates/plan-selection/src/milp/model/tests.rs +++ /dev/null @@ -1,1229 +0,0 @@ -use std::collections::{BTreeMap, BTreeSet}; - -use super::*; -use crate::groups::{prepare_milp_inputs, AccuracyWeight, IngestionPipeline}; -use crate::milp::decode::{decode, ChosenCandidates}; -use asap_cost::{node_map, plan_cost, PlanCost, ResourceUnitCost}; -use asap_types::ir::properties::ExecutionTiming; -use asap_types::ir::{OperatorNode, QueryRoot}; -use asap_types::physical::{Materialization, Medium}; -use asap_types::stages::{AssignmentRealizer, DagOfSelectedCandidates, PhysicalCandidate}; -use asap_types::workload::{DataWorkload, DurationMs, RangeEvaluation}; - -// --------------------------------------------------------------------------- -// Through Stages 1 and 2 -// --------------------------------------------------------------------------- - -/// A workload through the real Stages 1 and 2, with the roots the -/// realizer rebuilds plans from. -pub(crate) struct Fixture { - pub(crate) physical: PhysicalASAPMemoCandidates, - pub(crate) roots: Vec, - pub(crate) entries: Vec, - pub(crate) data: DataWorkload, - pub(crate) query_time_results_retained: bool, -} - -/// One query of a fixture: PromQL over metric `m`, its repeat interval in -/// ms, its relative error target (`None`: exact), its latency bound and -/// its range evaluation. -#[derive(Clone, Copy)] -pub(crate) struct Query<'a> { - promql: &'a str, - interval_ms: u32, - max_error: Option, - max_latency_ms: Option, - range_evaluation: Option, -} - -pub(crate) fn query(promql: &str, interval_ms: u32) -> Query<'_> { - Query { - promql, - interval_ms, - max_error: None, - max_latency_ms: None, - range_evaluation: None, - } -} - -pub(crate) fn approximate(promql: &str, interval_ms: u32, max_error: f64) -> Query<'_> { - Query { - max_error: Some(max_error), - ..query(promql, interval_ms) - } -} - -/// `queries` through Stage 1 and Stage 2 as the facade runs them, over the -/// complete pane-width domain, metric `m` with 100 series sampled every -/// second. -pub(crate) fn through_stages(queries: &[Query<'_>], raw_data_retained: bool) -> Fixture { - through_stages_with(queries, raw_data_retained, false) -} - -/// [`through_stages`] over metric `m` declared non-negative (so relative -/// errors compose through quotients, and Pass 1 keeps DDSketch). -pub(crate) fn through_stages_non_negative(queries: &[Query<'_>]) -> Fixture { - through_stages_over(queries, &["m"], true, false, true) -} - -/// [`through_stages`], with rows arriving (100 per second) when -/// `ingesting`, so Stage 2 offers ingestion-time candidates. -pub(crate) fn through_stages_with( - queries: &[Query<'_>], - raw_data_retained: bool, - ingesting: bool, -) -> Fixture { - through_stages_over(queries, &["m"], raw_data_retained, ingesting, false) -} - -/// The fixture builder behind [`through_stages`], over the metrics -/// `metrics`, each with 100 series sampled every second. -fn through_stages_over( - queries: &[Query<'_>], - metrics: &[&str], - raw_data_retained: bool, - ingesting: bool, - values_non_negative: bool, -) -> Fixture { - use asap_logical_asap_optimization::generate_logical_asap_memo_candidates; - use asap_logical_asap_optimization::generate_per_computation_logical_memo_candidates; - use asap_logical_asap_optimization::test_support::{entry_with, lower_promql}; - use asap_types::ir::properties::AccuracyTarget; - use asap_types::workload::{ - Predictability, QueryRecurrence, RepeatedDemand, RepetitionInterval, - }; - let target = |query: &Query<'_>| match query.max_error { - Some(error) => AccuracyTarget::max_error(error).failure_probability(0.01), - None => AccuracyTarget::exact(), - }; - let roots: Vec = queries - .iter() - .map(|query| QueryRoot::Operator(lower_promql(query.promql, target(query)))) - .collect(); - let entries: Vec = queries - .iter() - .map(|query| { - let mut entry = entry_with( - target(query), - QueryRecurrence::Repeated(RepeatedDemand::FixedInterval(RepetitionInterval( - query.interval_ms, - ))), - ); - entry.predictability = Predictability::Predictable { known_at: None }; - entry.requirements.max_latency_ms = query.max_latency_ms; - entry.range_evaluation = query.range_evaluation; - entry - }) - .collect(); - let mut data = DataWorkload::sampled_every( - metrics.iter().map(|metric| metric.to_string()), - DurationMs(1_000), - ); - for metric in metrics { - let source = data.sources.get_mut(*metric).expect("declared"); - source.series_count = Some(100); - source.values_non_negative = values_non_negative; - } - if ingesting { - data.arrival = asap_types::workload::DataArrival::Ingesting { - rows_per_second: 100.0, - }; - } - let targets: Vec = queries.iter().map(target).collect(); - let pass1 = generate_per_computation_logical_memo_candidates( - asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&roots), - &targets, - &asap_accuracy::BuiltinAccuracyModel, - &asap_accuracy::NoAccuracyEvidence, - &data, - None, - ) - .unwrap(); - let logical = generate_logical_asap_memo_candidates(pass1, &entries).unwrap(); - let physical = asap_physical_asap_optimization::generate_physical_asap_memo_candidates( - &logical, - &entries, - &data, - false, - raw_data_retained, - ) - .unwrap(); - Fixture { - physical, - roots, - entries, - data, - query_time_results_retained: false, - } -} - -/// What one checked solve gives. -pub(crate) struct Checked { - pub(crate) solution: SelectionMilpSolution, - pub(crate) plan: PlanCost, - pub(crate) dag: DagOfSelectedCandidates, -} - -/// `fixture`'s MILP under `cost_inputs`, with `restrict` adding rows first: -/// solved, decoded, realized by Stage 2 and costed by -/// `asap_cost::plan_cost`. The objective plus the request fees must equal -/// `plan_cost`'s cost. -pub(crate) fn solve_and_check( - fixture: &Fixture, - cost_inputs: &CostInputs, - restrict: impl FnOnce(&mut SelectionMilp<'_>), -) -> Result { - solve_and_check_by(fixture, cost_inputs, restrict, true) -} - -/// [`solve_and_check`] with the independent parts solved apart when -/// `split_into_parts`. -fn solve_and_check_by( - fixture: &Fixture, - cost_inputs: &CostInputs, - restrict: impl FnOnce(&mut SelectionMilp<'_>), - split_into_parts: bool, -) -> Result { - let inputs = prepare_milp_inputs( - &fixture.physical, - &fixture.entries, - &fixture.data, - cost_inputs, - ); - let mut milp = build_selection_milp( - &fixture.physical, - &inputs, - &fixture.entries, - cost_inputs, - SelectionMilpOptions { - split_into_parts, - ..SelectionMilpOptions::default() - }, - )?; - assert!(milp.inexact.is_empty(), "inexact terms: {:?}", milp.inexact); - restrict(&mut milp); - let solution = solve_selection_milp(milp, 0)?; - let dag = decode( - &fixture.physical, - &inputs, - &ChosenCandidates { - chosen: solution.chosen.clone(), - selected: solution.selected.clone(), - slot_reads: solution.slot_reads.clone(), - }, - ) - .expect("the solution decodes"); - dag.validate(&fixture.physical) - .expect("the decoded selection is valid"); - let realizer = asap_physical_asap_optimization::Stage2AssignmentRealizer { - original_roots: &fixture.roots, - entries: &fixture.entries, - data: &fixture.data, - query_time_results_retained: fixture.query_time_results_retained, - }; - let realized = realizer - .realize(&fixture.physical, &dag) - .expect("the realizer accepts the decoded plan"); - let plan_dag = &realized.plan.dag; - let recurrence: Vec<_> = fixture - .entries - .iter() - .map(|entry| entry.recurrence.clone()) - .collect(); - let ranges: Vec<_> = fixture - .entries - .iter() - .map(|entry| entry.range_evaluation) - .collect(); - let (plan, _) = plan_cost( - plan_dag, - &node_map(plan_dag), - &asap_cost::kept_tumbling_window_chains(plan_dag), - &recurrence, - &ranges, - &fixture.data, - cost_inputs, - cost_inputs.frozen_calibration_factor.unwrap_or(1.0), - ) - .expect("plan_cost costs the plan"); - let expected = plan.cost_per_second.expect("cost model 1") - solution.request_cost_per_second; - let tolerance = 1e-6 * expected.abs().max(1e-12); - assert!( - (solution.cost_per_second - expected).abs() <= tolerance, - "MILP objective {:e} != plan_cost {:e} (without {:e} of request fees); per node: {:#?}", - solution.cost_per_second, - expected, - solution.request_cost_per_second, - plan.per_node - ); - Ok(Checked { - solution, - plan, - dag, - }) -} - -pub(crate) fn cost_inputs() -> CostInputs { - CostInputs { - frozen_calibration_factor: Some(1.0), - ..CostInputs::default() - } -} - -/// The `aws-fargate-us-east-1` cost model of the memo examples. -fn aws_fargate_cost_inputs() -> CostInputs { - let defaults = cost_inputs(); - CostInputs { - cost_model: asap_cost::CostModel::UsageOverTime(ResourceUnitCost::aws_fargate_us_east_1( - defaults.resource_speed, - )), - ..defaults - } -} - -/// The candidates of `milp` that satisfy `wanted`. -fn candidates_where( - milp: &SelectionMilp<'_>, - wanted: impl Fn(&PhysicalCandidate, &OperatorNode) -> bool, -) -> Vec { - (0..milp.physical.candidates.len()) - .map(PhysicalCandidateId) - .filter(|&candidate| { - wanted( - &milp.physical.candidates[candidate.0], - milp.physical.physical_node(candidate), - ) - }) - .collect() -} - -/// Adds `Σ x[m] ≥ 1` over the shareable `candidates`: one is selected. -fn require_one_of(milp: &mut SelectionMilp<'_>, candidates: &[PhysicalCandidateId]) { - let sum: Expression = candidates - .iter() - .filter_map(|candidate| milp.selected.get(candidate).copied()) - .map(Expression::from) - .sum(); - milp.builder.row("test", constraint!(sum >= 1.0)); -} - -/// Adds `x[m] = 0` and `p[ctx, c] = 0` everywhere for `candidates`. -fn forbid(milp: &mut SelectionMilp<'_>, candidates: &[PhysicalCandidateId]) { - for candidate in candidates { - if let Some(&selected) = milp.selected.get(candidate) { - milp.builder.row("test", constraint!(selected <= 0.0)); - } - for context in 0..milp.choice.len() { - if let Some(&choice) = milp.choice[context].get(candidate) { - milp.builder.row("test", constraint!(choice <= 0.0)); - } - } - } -} - -/// The candidates that are pane chains or run at ingestion time. -fn pane_chains_and_ingestion(milp: &SelectionMilp<'_>) -> Vec { - candidates_where(milp, |candidate, node| { - candidate.timing == ExecutionTiming::IngestionTime - || matches!( - node.asap(), - Some(asap_types::ir::ASAPOp::SummaryTumblingWindowChain { .. }) - ) - }) -} - -/// How many nodes of `dag` run `candidate`. -fn runs_of(dag: &DagOfSelectedCandidates, candidate: PhysicalCandidateId) -> usize { - dag.candidate_of_node - .iter() - .filter(|&&c| c == candidate) - .count() -} - -/// `sum(m)` repeating every minute and `sum(m) + max(m)` every 30 s: the -/// second query does more work, so its evaluations take longer. -pub(crate) fn shared_sum() -> Fixture { - through_stages( - &[query("sum(m)", 60_000), query("sum(m) + max(m)", 30_000)], - true, - ) -} - -/// `sum_over_time(m[5m])` and `sum_over_time(m[1h])` over raw data. -pub(crate) fn two_windows() -> Fixture { - through_stages( - &[ - query("sum_over_time(m[5m])", 60_000), - query("sum_over_time(m[1h])", 60_000), - ], - true, - ) -} - -// --------------------------------------------------------------------------- -// Cost terms equal `plan_cost` (§3) -// --------------------------------------------------------------------------- - -// The optimum's objective equals `plan_cost` of the plan it decodes to. -#[test] -fn the_optimum_costs_what_plan_cost_says() { - for costs in [cost_inputs(), aws_fargate_cost_inputs()] { - solve_and_check(&shared_sum(), &costs, |_| {}).unwrap(); - } -} - -// Held memory and schedule groups: a `MatAtQuery(Memory)` result both -// queries read (every 60 s and every 30 s) is held until the slower reader -// finishes, once per execution at the schedule groups' rates. -#[test] -fn held_memory_of_a_result_readers_of_different_latency_share() { - let fixture = shared_sum(); - let checked = solve_and_check(&fixture, &cost_inputs(), |milp| { - let shared: Vec = candidates_where(milp, |candidate, _| { - candidate.materialization == Materialization::MatAtQuery(Medium::Memory) - }) - .into_iter() - .filter(|&candidate| milp.queries_reaching(candidate).len() == 2) - .collect(); - assert!(!shared.is_empty()); - require_one_of(milp, &shared); - }) - .unwrap(); - let latencies: Vec = checked - .plan - .usage - .latency_per_query - .iter() - .map(|latency| latency.total_ms()) - .collect(); - assert!(latencies[0] != latencies[1], "{latencies:?}"); - assert!(checked - .plan - .per_node - .values() - .any(|node| node.detail.contains("holds"))); -} - -// Raw retention: with one scan per window, the source keeps the longer -// window's raw data once. -#[test] -fn raw_retention_keeps_the_longest_window() { - let checked = solve_and_check(&two_windows(), &cost_inputs(), |milp| { - let excluded = pane_chains_and_ingestion(milp); - forbid(milp, &excluded); - }) - .unwrap(); - assert!(checked - .plan - .per_node - .values() - .any(|node| node.detail.contains("of raw data on disk"))); -} - -// A scan read through two offsets in one query evaluation: the scan run -// reads the union of the two windows, and the MILP charges exactly that -// union (`δ` over the window kinds read), as `plan_cost` does. -#[test] -fn a_scan_read_through_two_offsets_is_costed_at_their_union() { - let fixture = through_stages( - &[query( - "sum_over_time(m[5m]) + sum_over_time(m[5m] offset 1h)", - 60_000, - )], - true, - ); - let checked = solve_and_check(&fixture, &cost_inputs(), |milp| { - let excluded = pane_chains_and_ingestion(milp); - forbid(milp, &excluded); - }) - .unwrap(); - assert!( - checked - .solution - .size - .families - .contains_key("δ (scan window sets)"), - "{:?}", - checked.solution.size.families.keys().collect::>() - ); -} - -// A kept pane chain whose readers have different range-evaluation spans: -// it keeps the panes of the longest span, charged by retention levels. -#[test] -fn a_kept_pane_chain_keeps_the_longest_readers_span() { - let span = |span_ms| { - Some(RangeEvaluation { - step_ms: 60_000, - span_ms, - }) - }; - let fixture = through_stages_with( - &[ - Query { - range_evaluation: span(600_000), - ..query("sum_over_time(m[1h])", 60_000) - }, - Query { - range_evaluation: span(3_600_000), - ..query("sum_over_time(m[1h])", 60_000) - }, - ], - false, - true, - ); - let checked = solve_and_check(&fixture, &cost_inputs(), |milp| { - let chains = candidates_where(milp, |candidate, node| { - matches!( - node.asap(), - Some(asap_types::ir::ASAPOp::SummaryTumblingWindowChain { .. }) - ) && matches!( - candidate.materialization, - Materialization::MatAtIngestion(_) - ) - }); - require_one_of(milp, &chains); - }) - .unwrap(); - assert!( - checked - .solution - .size - .families - .contains_key("π (pane retention levels)"), - "{:?}", - checked.solution.size.families.keys().collect::>() - ); -} - -// A `NotMaterialized` input of a `MatAtQuery` candidate both queries read -// (`rate(m[5m])` under `sum` and `max`, one schedule group) runs once, in -// the candidate's context, and is costed as `plan_cost` costs it. -#[test] -fn a_shared_results_not_materialized_input_runs_once() { - let fixture = through_stages( - &[ - query("sum(rate(m[5m]))", 60_000), - query("max(rate(m[5m]))", 60_000), - ], - true, - ); - let mut shared = Vec::new(); - let checked = solve_and_check(&fixture, &cost_inputs(), |milp| { - shared = candidates_where(milp, |candidate, _| { - matches!(candidate.materialization, Materialization::MatAtQuery(_)) - }) - .into_iter() - .filter(|&m| milp.queries_reaching(m).len() == 2) - .filter(|&m| { - let context = milp.context_of(m).expect("a context"); - milp.choice[context] - .keys() - .any(|&c| milp.computes(context, c)) - }) - .collect(); - assert!(!shared.is_empty(), "a shared result over a computed input"); - require_one_of(milp, &shared); - }) - .unwrap(); - let m = *checked - .solution - .selected - .iter() - .find(|m| shared.contains(m)) - .expect("selected"); - assert_eq!(runs_of(&checked.dag, m), 1); - let node = checked - .dag - .candidate_of_node - .iter() - .position(|&c| c == m) - .expect("a node"); - for edge in checked.dag.edges.iter().filter(|e| e.consumer.0 == node) { - let input = checked.dag.candidate_of_node[edge.producer.0]; - let readers = checked - .dag - .edges - .iter() - .filter(|e| e.producer == edge.producer) - .count(); - assert_eq!(readers, 1, "m's input {input:?} is read by m alone"); - } -} - -// One ingestion scan read by sketches of two configurations (two quantile -// targets) runs once in its `I_r` and is costed once. -#[test] -fn one_ingestion_scan_shared_by_two_sketch_configurations_is_charged_once() { - let fixture = through_stages_with( - &[ - approximate("quantile_over_time(0.5, m[5m])", 60_000, 0.01), - approximate("quantile_over_time(0.9, m[5m])", 60_000, 0.05), - ], - true, - true, - ); - let checked = solve_and_check(&fixture, &cost_inputs(), |milp| { - let sketches = candidates_where(milp, |candidate, node| { - candidate.timing == ExecutionTiming::IngestionTime - && candidate.materialization != Materialization::NotMaterialized - && matches!( - node.asap(), - Some(asap_types::ir::ASAPOp::SummaryAgg { .. }) - | Some(asap_types::ir::ASAPOp::SummaryTumblingWindowChain { .. }) - ) - }); - let sum: Expression = sketches - .iter() - .filter_map(|c| milp.selected.get(c).copied()) - .map(Expression::from) - .sum(); - milp.builder.row("test", constraint!(sum >= 2.0)); - // Every query-time scan is excluded: both read ingestion state. - let scans = candidates_where(milp, |candidate, node| { - candidate.timing == ExecutionTiming::QueryTime && node.operator.kind_name() == "Scan" - }); - forbid(milp, &scans); - }) - .unwrap(); - let ingestion_scans: Vec = (0..checked.dag.candidate_of_node.len()) - .filter(|&node| { - let candidate = checked.dag.candidate_of_node[node]; - fixture.physical.candidates[candidate.0].timing == ExecutionTiming::IngestionTime - && fixture - .physical - .physical_node(candidate) - .operator - .kind_name() - == "Scan" - }) - .collect(); - assert_eq!(ingestion_scans.len(), 1, "{:?}", checked.dag); -} - -/// Solves `queries` (ingesting) with two ingestion-time sketch states -/// selected, every query-time scan forbidden and every ingestion-time state -/// whose refresh window is not one of `widths_ms` (phase 0) forbidden; -/// returns the checked solve (objective = `plan_cost`) and its -/// ingestion-time scan runs, each with the refresh windows of the states -/// reading it. -fn ingestion_scans_under_refresh_windows( - queries: &[Query<'_>], - widths_ms: &[u64], -) -> (Checked, Vec>) { - use asap_types::ir::materialized_view::RefreshWindow; - let fixture = through_stages_with(queries, true, true); - let checked = solve_and_check(&fixture, &cost_inputs(), |milp| { - let states = candidates_where(milp, |candidate, _| { - candidate.timing == ExecutionTiming::IngestionTime - && candidate.materialization != Materialization::NotMaterialized - }); - let window_of = |state: PhysicalCandidateId| { - milp.inputs - .ingestion_context_of_shareable - .get(&state) - .map(|&context| milp.inputs.contexts[context].kind) - }; - let (allowed, other): (Vec<_>, Vec<_>) = states.iter().partition(|&&state| { - widths_ms.iter().any(|&width_ms| { - window_of(state) - == Some(ContextKind::Ingestion(IngestionPipeline::Window( - RefreshWindow { - width_ms, - phase_ms: 0, - }, - ))) - }) - }); - forbid(milp, &other); - let sketches = candidates_where(milp, |_, node| { - matches!( - node.asap(), - Some(asap_types::ir::ASAPOp::SummaryTumblingWindowChain { .. }) - ) - }); - let sum: Expression = allowed - .iter() - .filter(|state| sketches.contains(state)) - .filter_map(|state| milp.selected.get(state).copied()) - .map(Expression::from) - .sum(); - milp.builder.row("test", constraint!(sum >= 2.0)); - let scans = candidates_where(milp, |candidate, node| { - candidate.timing == ExecutionTiming::QueryTime && node.operator.kind_name() == "Scan" - }); - forbid(milp, &scans); - }) - .unwrap(); - // Per ingestion-time scan node of the decoded selection: the pane widths - // of the chains above it. - let dag = &checked.dag; - let nodes = dag.candidate_of_node.len(); - let mut readers: Vec> = vec![Vec::new(); nodes]; - for edge in &dag.edges { - readers[edge.producer.0].push(edge.consumer.0); - } - let width_above = |scan: usize| { - let mut widths = BTreeSet::new(); - let mut stack = vec![scan]; - let mut seen = BTreeSet::new(); - while let Some(node) = stack.pop() { - if !seen.insert(node) { - continue; - } - let candidate = dag.candidate_of_node[node]; - match fixture.physical.physical_node(candidate).asap() { - Some(asap_types::ir::ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms, - .. - }) => { - widths.insert(*tumbling_window_width_ms); - } - _ => stack.extend(readers[node].iter().copied()), - } - } - widths - }; - let scans = (0..nodes) - .filter(|&node| { - let candidate = dag.candidate_of_node[node]; - fixture.physical.candidates[candidate.0].timing == ExecutionTiming::IngestionTime - && fixture - .physical - .physical_node(candidate) - .operator - .kind_name() - == "Scan" - }) - .map(width_above) - .collect(); - (checked, scans) -} - -// Two ingestion-time pane chains of one refresh window (10 s panes) over -// one source run in one ingestion pipeline: the scan below them runs and -// is charged once, and the objective equals `plan_cost`. -#[test] -fn one_ingestion_scan_under_chains_of_one_refresh_window_is_charged_once() { - let (_, scans) = ingestion_scans_under_refresh_windows( - &[ - approximate("quantile_over_time(0.5, m[5m])", 60_000, 0.01), - approximate("quantile_over_time(0.9, m[5m])", 60_000, 0.05), - ], - &[10_000], - ); - assert_eq!(scans, vec![BTreeSet::from([10_000])]); -} - -// Chains of 10 s and 60 s panes over one source run in two ingestion -// pipelines (#1151): the scan runs and is charged once in each, and the -// objective equals `plan_cost`, which charges it once per refresh window. -#[test] -fn ingestion_scans_under_chains_of_two_refresh_windows_are_charged_twice() { - let (_, scans) = ingestion_scans_under_refresh_windows( - &[ - approximate("quantile_over_time(0.5, m[10s])", 10_000, 0.01), - approximate("quantile_over_time(0.9, m[5m])", 60_000, 0.05), - ], - &[10_000, 60_000], - ); - let mut scans = scans; - scans.sort(); - assert_eq!( - scans, - vec![BTreeSet::from([10_000]), BTreeSet::from([60_000])] - ); -} - -// Every producer candidate of every input reads one view window on real -// Stage 2 output (ingesting, and not), so a refresh window computed from -// first producers is the realized plan's. -#[test] -fn producers_of_an_input_read_one_view_window_through_stages() { - let fixtures = [ - through_stages_with( - &[ - approximate("quantile_over_time(0.5, m[10s])", 10_000, 0.01), - approximate("quantile_over_time(0.9, m[5m])", 60_000, 0.05), - ], - true, - true, - ), - through_stages_with( - &[ - query("sum by (job) (rate(m[1m]))", 30_000), - query("sum(increase(m[5m] offset 1m))", 60_000), - approximate( - "quantile_over_time(0.5, m[5m]) / quantile_over_time(0.9, m[5m])", - 60_000, - 0.05, - ), - ], - true, - true, - ), - through_stages( - &[ - query("avg_over_time(m[5m])", 60_000), - query("max_over_time(m[1m])", 60_000), - ], - true, - ), - ]; - for fixture in &fixtures { - assert_eq!( - crate::groups::producer_window_disagreements_of_all(&fixture.physical), - Vec::new() - ); - } -} - -// A stateless subtree read by two state nodes of one query evaluation -// (`avg` reads a sum and a count accumulator over one `q50/q50 + q90` -// subtree) is computed once: no candidate runs twice in the plan. -#[test] -fn a_stateless_subtree_two_state_nodes_read_runs_once() { - let fixture = through_stages( - &[query( - "avg by (job) (quantile_over_time(0.5, m[5m]) / quantile_over_time(0.5, m[5m]) \ - + quantile_over_time(0.9, m[5m]))", - 60_000, - )], - true, - ); - let checked = solve_and_check(&fixture, &cost_inputs(), |_| {}).unwrap(); - let mut runs: BTreeMap = BTreeMap::new(); - for &candidate in &checked.dag.candidate_of_node { - *runs.entry(candidate).or_default() += 1; - } - assert!(runs.values().all(|&n| n == 1), "{runs:?}"); -} - -// --------------------------------------------------------------------------- -// Structure -// --------------------------------------------------------------------------- - -// "Used only if read": a query context may not select a shareable -// candidate no used reader there reads, so a phantom read cannot satisfy -// `MinimumQueryReaders`. With every reader of a two-reader `MatAtQuery` -// result in the second query's context forbidden, selecting it is -// infeasible. -#[test] -fn a_phantom_reader_does_not_satisfy_minimum_query_readers() { - let fixture = shared_sum(); - let costs = cost_inputs(); - let inputs = prepare_milp_inputs(&fixture.physical, &fixture.entries, &fixture.data, &costs); - let options = SelectionMilpOptions::default(); - let mut milp = build_selection_milp( - &fixture.physical, - &inputs, - &fixture.entries, - &costs, - options, - ) - .unwrap(); - let constrained: Vec = fixture - .physical - .reader_constraints - .iter() - .filter_map(|constraint| match constraint { - ReaderConstraint::MinimumQueryReaders { - candidate, minimum, .. - } if *minimum >= 2 => Some(*candidate), - _ => None, - }) - .filter(|&m| milp.selected.contains_key(&m) && milp.choice[1].contains_key(&m)) - .collect(); - let m = *constrained.first().expect("a two-reader shareable result"); - let selected = milp.selected[&m]; - milp.builder.row("test", constraint!(selected >= 1.0)); - let readers: Vec = milp.readers[1][&m] - .iter() - .filter_map(|reader| match reader { - Reader::Candidate(reader, _) => Some(*reader), - _ => None, - }) - .collect(); - assert!( - !milp.readers[1][&m] - .iter() - .any(|reader| matches!(reader, Reader::Output(_))), - "the result is not an output of the second query" - ); - for reader in readers { - let choice = milp.choice[1][&reader]; - milp.builder.row("test", constraint!(choice <= 0.0)); - } - assert_eq!( - solve_selection_milp(milp, 0).err(), - Some(SelectionMilpError::Infeasible) - ); -} - -// A `NotMaterialized` query-time candidate selected in two contexts is two -// nodes; a shareable candidate two queries read is one node. -#[test] -fn runs_decode_one_node_per_context_and_one_per_shareable_candidate() { - let fixture = through_stages( - &[query("sum(m)", 60_000), query("sum(m) * 2", 60_000)], - true, - ); - let unshared = solve_and_check(&fixture, &cost_inputs(), |milp| { - let shared = candidates_where(milp, |candidate, _| { - candidate.timing == ExecutionTiming::QueryTime - && candidate.materialization != Materialization::NotMaterialized - }); - forbid(milp, &shared); - let ingestion = candidates_where(milp, |candidate, _| { - candidate.timing == ExecutionTiming::IngestionTime - }); - forbid(milp, &ingestion); - }) - .unwrap(); - let mut runs: BTreeMap = BTreeMap::new(); - for &candidate in &unshared.dag.candidate_of_node { - *runs.entry(candidate).or_default() += 1; - } - assert!(runs.values().any(|&n| n == 2), "{runs:?}"); - let mut shared_result = Vec::new(); - let shared = solve_and_check(&fixture, &cost_inputs(), |milp| { - shared_result = candidates_where(milp, |candidate, _| { - candidate.materialization == Materialization::MatAtQuery(Medium::Memory) - }) - .into_iter() - .filter(|&m| milp.queries_reaching(m).len() == 2) - .collect(); - require_one_of(milp, &shared_result); - }) - .unwrap(); - let m = *shared - .solution - .selected - .iter() - .find(|m| shared_result.contains(m)) - .expect("selected"); - assert_eq!(runs_of(&shared.dag, m), 1); -} - -// A query over its latency bound: with the bound below the fastest plan's -// latency there is no plan; at the optimum's latency, the optimum stays. -#[test] -fn a_plan_over_a_latency_bound_is_infeasible() { - let fixture = shared_sum(); - let optimum = solve_and_check(&fixture, &cost_inputs(), |_| {}).unwrap(); - let latency = optimum.plan.usage.latency_per_query[1].total_ms(); - let mut bounded = shared_sum(); - bounded.entries[1].requirements.max_latency_ms = Some(latency * (1.0 + 1e-9)); - let at_bound = solve_and_check(&bounded, &cost_inputs(), |_| {}).unwrap(); - assert!(at_bound.plan.usage.latency_per_query[1].total_ms() <= latency * (1.0 + 1e-9)); - let mut tight = shared_sum(); - tight.entries[1].requirements.max_latency_ms = Some(1e-9); - assert!(matches!( - solve_and_check(&tight, &cost_inputs(), |_| {}).err(), - Some(SelectionMilpError::Infeasible) - )); -} - -// Cost models 2 and 3 are not encoded. -#[test] -fn cost_models_2_and_3_are_unsupported() { - use asap_cost::{ClusterLimits, ClusterObjective, CostModel}; - let fixture = shared_sum(); - for model in [ - CostModel::ProvisionedCapacity(ResourceUnitCost::illustrative()), - CostModel::ClusterCapacity(ClusterLimits { - memory_bytes: u64::MAX, - disk_bytes: u64::MAX, - cpu_cores: None, - objective: ClusterObjective::LowestLatency, - }), - ] { - let costs = CostInputs { - cost_model: model, - ..cost_inputs() - }; - let inputs = - prepare_milp_inputs(&fixture.physical, &fixture.entries, &fixture.data, &costs); - let options = SelectionMilpOptions::default(); - assert!(matches!( - build_selection_milp( - &fixture.physical, - &inputs, - &fixture.entries, - &costs, - options - ) - .err(), - Some(SelectionMilpError::UnsupportedCostModel(_)) - )); - } -} - -// Accuracy rows on `q0.5 / q0.9` over metric `m` declared non-negative, -// weights from the composition: a candidate unusable for the query (a -// DDSketch denominator sized for the whole target) is never depended on, -// and two at `parts = 2` are depended on together unless their pair is -// excluded. (Pass 1 already prunes KLL, whose rank error composes no -// guarantee through the quotient.) -#[test] -fn accuracy_rows_follow_the_composition_on_a_ratio() { - let fixture = through_stages_non_negative(&[approximate( - "quantile_over_time(0.5, m[5m]) / quantile_over_time(0.9, m[5m])", - 60_000, - 0.01, - )]); - let costs = cost_inputs(); - let inputs = prepare_milp_inputs(&fixture.physical, &fixture.entries, &fixture.data, &costs); - let of = |wanted: &dyn Fn(&AccuracyWeight, usize) -> bool| -> Vec { - inputs.accuracy_weights[0] - .iter() - .filter(|(class, weight)| { - wanted(weight, inputs.candidate_data[class.0].accuracy_budget_parts) - }) - .map(|(class, _)| *class) - .collect() - }; - let unusable = of(&|weight, _| matches!(weight, AccuracyWeight::Unusable(_))); - let usable_at_two = of(&|weight, parts| *weight == AccuracyWeight::One && parts == 2); - assert!(!unusable.is_empty() && usable_at_two.len() >= 2); - let solve = |inputs: &MilpInputs, restrict: &dyn Fn(&mut SelectionMilp<'_>)| { - let mut milp = build_selection_milp( - &fixture.physical, - inputs, - &fixture.entries, - &costs, - SelectionMilpOptions::default(), - ) - .unwrap(); - restrict(&mut milp); - solve_selection_milp(milp, 0).map(|_| ()) - }; - // Each candidate's runs: `x[m]`, or `p` in a context computing it. - let runs_of = |milp: &SelectionMilp<'_>, candidates: &[PhysicalCandidateId]| -> Expression { - let mut sum = Expression::from(0.0); - for &candidate in candidates { - if let Some(&selected) = milp.selected.get(&candidate) { - sum += selected; - } - for context in 0..milp.choice.len() { - if milp.computes(context, candidate) { - if let Some(&choice) = milp.choice[context].get(&candidate) { - sum += choice; - } - } - } - } - sum - }; - assert_eq!( - solve(&inputs, &|milp| { - let sum = runs_of(milp, &unusable); - milp.builder.row("test", constraint!(sum >= 1.0)); - }) - .err(), - Some(SelectionMilpError::Infeasible) - ); - let two_of = |milp: &mut SelectionMilp<'_>| { - let sum = runs_of(milp, &usable_at_two); - milp.builder.row("test", constraint!(sum >= 2.0)); - }; - assert!(solve(&inputs, &two_of).is_ok()); - let mut excluded = inputs.clone(); - for (index, &first) in usable_at_two.iter().enumerate() { - for &second in &usable_at_two[index + 1..] { - excluded.accuracy_excluded_pairs[0].push((first, second)); - } - } - assert_eq!( - solve(&excluded, &two_of).err(), - Some(SelectionMilpError::Infeasible) - ); -} - -// The warm start is a feasible plan whose completion's objective equals -// what `plan_cost` says of it: fixing every binary to it, the solve -// returns the warm start's cost. -#[test] -fn the_warm_start_is_a_feasible_plan_costed_exactly() { - for fixture in [shared_sum(), two_windows()] { - let mut start_cost = None; - let checked = solve_and_check(&fixture, &cost_inputs(), |milp| { - let completion = crate::milp::completion::Completion::new(milp); - let start = - crate::milp::warm_start::warm_start(milp, &completion).expect("a warm start"); - start_cost = Some(start.cost_per_second); - let values: HashMap = start.values.into_iter().collect(); - let mut fixed = Vec::new(); - for choices in &milp.choice { - fixed.extend(choices.values().copied()); - } - fixed.extend(milp.selected.values().copied()); - for variable in fixed { - let value = values.get(&variable).copied().unwrap_or(0.0); - milp.builder.row("test", constraint!(variable == value)); - } - }) - .unwrap(); - let start_cost = start_cost.expect("costed"); - let tolerance = 1e-6 * start_cost.abs().max(1e-12); - assert!( - (checked.solution.cost_per_second - start_cost).abs() <= tolerance, - "warm start {start_cost:e} != plan_cost {:e}", - checked.solution.cost_per_second - ); - } -} - -// RangeHeavy templates 7 and 4: the widened scan puts a `Project` in the -// memo group of the scan it reads. The group's roots are split by -// self-read depth, so the plan is acyclic, valid, accepted by the realizer -// and costed exactly. -#[test] -fn a_widened_scan_reading_its_own_memo_group_plans_exactly() { - let fixture = through_stages( - &[ - approximate("sum by (job) (sum_over_time(m[1h]))", 60_000, 0.05), - approximate("sum_over_time(m[1h])", 60_000, 0.05), - ], - true, - ); - let reads_own_group = (0..fixture.physical.candidates.len()).any(|candidate| { - let group = fixture - .physical - .memo_group_of(PhysicalCandidateId(candidate)); - fixture.physical.candidates[candidate] - .producer_candidates_per_input - .iter() - .any(|producers| { - producers - .iter() - .any(|p| p.0 != candidate && fixture.physical.memo_group_of(*p) == group) - }) - }); - assert!( - reads_own_group, - "the fixture has a root reading its own group" - ); - for costs in [cost_inputs(), aws_fargate_cost_inputs()] { - solve_and_check(&fixture, &costs, |_| {}).unwrap(); - } -} - -// #1095 §5 (3a): queries over two metrics share no candidate, run of an `I_r` or -// source, so the model splits into one part per metric; solving the parts -// apart gives the plan and cost of one solve of the whole model. -#[test] -fn independent_query_sets_solve_apart_to_the_same_plan() { - let queries = [ - query("sum_over_time(m[5m])", 60_000), - query("sum_over_time(m[1h])", 60_000), - query("sum(n) + max(n)", 30_000), - ]; - let fixture = through_stages_over(&queries, &["m", "n"], true, false, false); - let costs = cost_inputs(); - let inputs = prepare_milp_inputs(&fixture.physical, &fixture.entries, &fixture.data, &costs); - let options = SelectionMilpOptions::default(); - let milp = build_selection_milp( - &fixture.physical, - &inputs, - &fixture.entries, - &costs, - options, - ) - .unwrap(); - let parts = independent_parts(&milp); - let queries_per_part: Vec> = parts.iter().map(|part| part.queries.clone()).collect(); - assert_eq!(queries_per_part, vec![vec![0, 1], vec![2]]); - let columns: usize = parts.iter().map(|part| part.columns.len()).sum(); - let rows: usize = parts.iter().map(|part| part.rows.len()).sum(); - assert_eq!(columns, milp.builder.variables.len()); - assert_eq!(rows, milp.builder.rows.len()); - - let apart = solve_and_check_by(&fixture, &costs, |_| {}, true).unwrap(); - let whole = solve_and_check_by(&fixture, &costs, |_| {}, false).unwrap(); - assert_eq!(apart.solution.parts.len(), 2); - assert_eq!(whole.solution.parts.len(), 1); - let tolerance = 1e-9 * whole.solution.cost_per_second.abs(); - assert!( - (apart.solution.cost_per_second - whole.solution.cost_per_second).abs() <= tolerance, - "apart {:e} != whole {:e}", - apart.solution.cost_per_second, - whole.solution.cost_per_second - ); - assert_eq!(apart.dag.candidate_of_node, whole.dag.candidate_of_node); -} - -thread_local! { - /// Set by a test to build the big-M bounds as before #1095 step 1 - /// ([`super::tests_loose_big_m`]). - pub(crate) static LOOSE_BIG_M: std::cell::Cell = const { std::cell::Cell::new(false) }; -} - -/// Per row of family `family` in `fixture`'s model (built with the loose -/// big-M bounds when `loose`), the largest coefficient on an `x` variable. -fn big_m_per_row(fixture: &Fixture, family: &str, loose: bool) -> Vec { - LOOSE_BIG_M.with(|cell| cell.set(loose)); - let costs = aws_fargate_cost_inputs(); - let inputs = prepare_milp_inputs(&fixture.physical, &fixture.entries, &fixture.data, &costs); - let milp = build_selection_milp( - &fixture.physical, - &inputs, - &fixture.entries, - &costs, - SelectionMilpOptions::default(), - ) - .unwrap(); - LOOSE_BIG_M.with(|cell| cell.set(false)); - let selected: std::collections::HashSet = milp.selected.values().copied().collect(); - milp.builder - .rows - .iter() - .zip(&milp.builder.row_families) - .filter(|(_, &f)| f == family) - .map(|(row, _)| { - row.expression() - .linear_coefficients() - .filter(|(variable, _)| selected.contains(variable)) - .map(|(_, coefficient)| coefficient.abs()) - .fold(0.0, f64::max) - }) - .collect() -} - -// #1095 step 1: `Ē[m]` in `V ≥ E − Ē (x[m] − z)` is the sum over the -// context's slots of its costliest run. With a rollup read every 20 s and -// every 30 s, a shared context has slots with several runs, so the bound is -// below the old sum over every run; the plan and its cost are unchanged. -#[test] -fn a_tighter_context_execution_bound_keeps_the_plan() { - let fixture = through_stages( - &[ - query("sum by (job) (sum_over_time(m[10m]))", 20_000), - query("sum(sum_over_time(m[10m]))", 30_000), - ], - true, - ); - let family = "V (context execution at a rate component)"; - let tight = big_m_per_row(&fixture, family, false); - let loose = big_m_per_row(&fixture, family, true); - assert_eq!(tight.len(), loose.len()); - assert!(tight - .iter() - .zip(&loose) - .all(|(t, l)| *t <= l * (1.0 + 1e-12))); - assert!(tight.iter().zip(&loose).any(|(t, l)| *t < l * (1.0 - 1e-9))); - - let tightened = solve_and_check(&fixture, &aws_fargate_cost_inputs(), |_| {}).unwrap(); - LOOSE_BIG_M.with(|cell| cell.set(true)); - let looser = solve_and_check(&fixture, &aws_fargate_cost_inputs(), |_| {}); - LOOSE_BIG_M.with(|cell| cell.set(false)); - let looser = looser.unwrap(); - let tolerance = 1e-9 * looser.solution.cost_per_second.abs(); - assert!( - (tightened.solution.cost_per_second - looser.solution.cost_per_second).abs() <= tolerance - ); - assert_eq!( - tightened.dag.candidate_of_node, - looser.dag.candidate_of_node - ); -} diff --git a/crates/plan-selection/src/milp/warm_start.rs b/crates/plan-selection/src/milp/warm_start.rs deleted file mode 100644 index b7de6ae37..000000000 --- a/crates/plan-selection/src/milp/warm_start.rs +++ /dev/null @@ -1,780 +0,0 @@ -//! A feasible starting plan for the group selection MILP (#1095 §2.0, -//! "Solving strategy" step 1), so HiGHS starts with an incumbent. -//! -//! **Each query's cheapest plan.** Queries are planned one after another -//! into one plan. A query's plan is derived from its output references -//! down: in each context, each slot an input needs takes the candidate -//! with the cheapest estimated subtree (its usage at the context's rate -//! plus its inputs' cheapest subtrees), unless the context already chose a -//! candidate of that slot, which every reader must then accept (one -//! candidate per slot per context); a dead end backtracks to the next -//! candidate. A shareable candidate's context is derived once, when it is -//! first selected; a candidate of an `I_r` is chosen in the `I_r` of the -//! reading context's refresh window. -//! -//! **Repairs per query.** If the query's plan depends on more approximate -//! values than one of them is sized for, on a pair excluded together, or -//! on a candidate unusable for it, the query is planned again with exact -//! candidates only; if its estimated latency is over its bound, it is -//! planned again by latency alone. -//! -//! **Greedy sharing.** A second plan prices what the queries planned -//! before already selected (a shareable candidate, a candidate of an `I_r`) at -//! zero, so later queries read it. Both plans, in workload order and -//! reversed, are costed exactly by the MILP's own rows and objective -//! ([`Completion`]); the cheapest feasible one is the start. -use std::collections::{BTreeMap, BTreeSet, HashMap}; - -use asap_cost::execution_demand::evaluation_rate; -use asap_types::stages::{output_references, PhysicalCandidateId, ReaderConstraint}; -use good_lp::Variable; - -use crate::groups::{AccuracyWeight, ContextKind, ResultSlot, RunKind}; - -use super::completion::Completion; -use super::cost_terms::NodeNumbers; -use super::model::{latency_bound_ms, SelectionMilp}; - -/// The most derivation steps one query's plan takes before giving up. -const MOST_STEPS_PER_QUERY: usize = 200_000; - -/// The starting plan. -pub(crate) struct StartingPlan { - /// Every variable's value. - pub(crate) values: Vec<(Variable, f64)>, - /// Its objective (cost per second without the request fees). - pub(crate) cost_per_second: f64, -} - -/// What a derivation minimizes. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum Objective { - Cost, - Latency, -} - -/// The candidates a query's derivation may use. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum Approximate { - /// Approximate candidates usable for the query. - Usable, - /// Those sized for at least this many shares. - SizedFor(usize), - /// Exact candidates only. - None, -} - -/// A plan under construction: per context, the candidate of each slot, and -/// the selected shareable candidates. -#[derive(Debug, Clone, Default)] -struct Plan { - chosen: Vec>, - selected: BTreeSet, - /// Per multi-slot input `(context, reader, input)`, the slot it reads. - slot_reads: BTreeMap<(usize, PhysicalCandidateId, usize), ResultSlot>, -} - -/// One change to a [`Plan`], to undo. -enum Undo { - Chose(usize, ResultSlot), - Selected(PhysicalCandidateId), - SlotRead(usize, PhysicalCandidateId, usize), -} - -/// A feasible plan (module documentation), or `None` when none was found. -/// It tries a fixed number of query orders ([`MORE_ORDERS`]), so every run -/// starts from the same plan. -pub(crate) fn warm_start( - milp: &SelectionMilp<'_>, - completion: &Completion, -) -> Option { - let estimates = Estimates::new(milp); - let mut best: Option = None; - let count = milp.query_count(); - let mut orders: Vec> = vec![(0..count).collect(), (0..count).rev().collect()]; - // More orders: a fixed pseudo-random sequence of permutations, so runs - // are reproducible. - let mut state: u64 = 0x9e37_79b9_7f4a_7c15; - for _ in 0..MORE_ORDERS { - let mut order: Vec = (0..count).collect(); - for at in (1..count).rev() { - state ^= state << 13; - state ^= state >> 7; - state ^= state << 17; - order.swap(at, (state % (at as u64 + 1)) as usize); - } - orders.push(order); - } - // (share, amortized): alone; reading what earlier queries selected; and - // that with each shareable candidate's own cost divided among the - // queries that may reach it. - let strategies = [(false, false), (true, false), (true, true)]; - let mut tried = 0; - for order in &orders { - for &(share, amortized) in &strategies { - tried += 1; - let Some(plan) = - sequential_plan(milp, &estimates, order, share, amortized, &BTreeSet::new()) - else { - debug(|| format!("warm start: no plan (share {share}, amortized {amortized})")); - continue; - }; - let binaries = binaries_of(milp, &plan); - let completed = completion.complete(&binaries); - if let Err(reason) = &completed { - debug(|| format!("warm start: completion failed (share {share}): {reason}")); - } - if let Ok(completed) = completed { - debug(|| { - format!( - "warm start: plan (share {share}, amortized {amortized}) costs {:e}, {} shareable selected", - completed.cost_per_second, - plan.selected.len() - ) - }); - if best - .as_ref() - .is_none_or(|known| completed.cost_per_second < known.cost_per_second) - { - best = Some(StartingPlan { - values: completion.values(&completed), - cost_per_second: completed.cost_per_second, - }); - } - } - } - } - // Greedy sharing: prefer one more shareable candidate that several - // queries may read (most readers first), kept when the exact cost - // falls. - let mut sharable: Vec<(usize, PhysicalCandidateId)> = milp - .inputs - .context_of_shareable - .keys() - .map(|&m| (estimates.readers[m.0], m)) - .filter(|&(readers, _)| readers >= 2) - .collect(); - sharable.sort_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(&b.1))); - let mut preferred: BTreeSet = BTreeSet::new(); - for (_, m) in sharable { - if best.is_none() { - break; - } - let mut trial = preferred.clone(); - trial.insert(m); - tried += 1; - let Some(plan) = sequential_plan(milp, &estimates, &orders[0], true, false, &trial) else { - continue; - }; - if let Ok(completed) = completion.complete(&binaries_of(milp, &plan)) { - if best - .as_ref() - .is_some_and(|known| completed.cost_per_second < known.cost_per_second) - { - preferred = trial; - best = Some(StartingPlan { - values: completion.values(&completed), - cost_per_second: completed.cost_per_second, - }); - } - } - } - debug(|| { - format!( - "warm start: {tried} plans tried, best {:?}", - best.as_ref().map(|best| best.cost_per_second) - ) - }); - best -} - -/// The pseudo-random orders tried after workload order and its reverse. -const MORE_ORDERS: usize = 30; - -/// The plan's binaries: `p[ctx, c]` and `x[m]` set to 1. -fn binaries_of(milp: &SelectionMilp<'_>, plan: &Plan) -> Vec<(Variable, f64)> { - let mut binaries = Vec::new(); - for (context, chosen) in plan.chosen.iter().enumerate() { - for candidate in chosen.values() { - if let Some(&variable) = milp.choice[context].get(candidate) { - binaries.push((variable, 1.0)); - } - } - } - for shareable in &plan.selected { - if let Some(&variable) = milp.selected.get(shareable) { - binaries.push((variable, 1.0)); - } - } - for (key, slot) in &plan.slot_reads { - if let Some(links) = milp.slot_links.get(key) { - for &(at, link) in links { - if at == *slot { - binaries.push((link, 1.0)); - } - } - } - } - binaries -} - -/// Every query planned in `order` into one plan; with `share`, what is -/// already selected costs nothing to read. -fn sequential_plan( - milp: &SelectionMilp<'_>, - estimates: &Estimates, - order: &[usize], - share: bool, - amortized: bool, - preferred: &BTreeSet, -) -> Option { - let mut plan = Plan { - chosen: vec![BTreeMap::new(); milp.choice.len()], - selected: BTreeSet::new(), - slot_reads: BTreeMap::new(), - }; - for &query in order { - let attempts = [ - (Objective::Cost, Approximate::Usable), - (Objective::Cost, Approximate::SizedFor(2)), - (Objective::Cost, Approximate::SizedFor(4)), - (Objective::Cost, Approximate::None), - (Objective::Latency, Approximate::Usable), - (Objective::Latency, Approximate::SizedFor(2)), - (Objective::Latency, Approximate::SizedFor(4)), - (Objective::Latency, Approximate::None), - ]; - // The first attempt meeting accuracy and the latency estimate; else - // the last meeting accuracy (the latency estimate is from each - // candidate's first costed usage, so it may be pessimistic: the - // completion checks the exact latency). - let mut planned = false; - let mut fallback = None; - for (attempt, (objective, approximate)) in attempts.into_iter().enumerate() { - let mut deriving = Derivation { - milp, - estimates, - query, - objective, - approximate, - share, - amortized, - preferred, - undo: Vec::new(), - steps: 0, - references: output_references(&milp.physical.query_roots) - .into_iter() - .enumerate() - .filter(|(_, (q, _))| *q == query) - .map(|(reference, _)| reference) - .collect(), - }; - let derived = deriving.derive_query(&mut plan).is_ok(); - let accurate = derived && meets_accuracy(milp, &plan, query); - let fast = accurate && meets_latency(milp, &plan, query); - debug(|| { - format!( - "warm start: q{query} {objective:?}/{approximate:?}: derived {derived}, \ - accuracy {accurate}, latency {fast}, steps {}", - deriving.steps - ) - }); - if fast { - planned = true; - break; - } - if accurate { - fallback = Some(attempt); - } - let undo = std::mem::take(&mut deriving.undo); - undo_all(&mut plan, undo); - } - if let (false, Some(attempt)) = (planned, fallback) { - let (objective, approximate) = attempts[attempt]; - let mut deriving = Derivation { - milp, - estimates, - query, - objective, - approximate, - share, - amortized, - preferred, - undo: Vec::new(), - steps: 0, - references: output_references(&milp.physical.query_roots) - .into_iter() - .enumerate() - .filter(|(_, (q, _))| *q == query) - .map(|(reference, _)| reference) - .collect(), - }; - planned = deriving.derive_query(&mut plan).is_ok(); - } - if !planned { - return None; - } - } - Some(plan) -} - -fn undo_all(plan: &mut Plan, undo: Vec) { - for change in undo.into_iter().rev() { - match change { - Undo::Chose(context, slot) => { - plan.chosen[context].remove(&slot); - } - Undo::Selected(shareable) => { - plan.selected.remove(&shareable); - } - Undo::SlotRead(context, reader, input) => { - plan.slot_reads.remove(&(context, reader, input)); - } - } - } -} - -/// Per context and candidate, the estimated cost and latency of the -/// cheapest subtree, per query (the rate of a query context; shared -/// contexts at the slowest rate of the queries that may reach them). -struct Estimates { - /// Per context, its rate (executions per second). - rate: Vec, - /// Per candidate, its numbers (first costed context). - numbers: Vec>, - /// Per candidate, the queries that may depend on it (shareable ones). - readers: Vec, -} - -impl Estimates { - fn new(milp: &SelectionMilp<'_>) -> Self { - let horizon = milp.cost_inputs.non_repeating_query_horizon_seconds; - let query_rate: Vec = milp - .entries - .iter() - .map(|entry| { - evaluation_rate([(&entry.recurrence, entry.range_evaluation)], horizon) - .unwrap_or(1.0) - }) - .collect(); - let rate = milp - .inputs - .contexts - .iter() - .map(|context| match context.kind { - ContextKind::Query(query) => query_rate[query], - ContextKind::Ingestion(_) => 1.0, - ContextKind::Shared(m) if context.at_query_time => { - let readers = milp.queries_reaching(m); - readers - .iter() - .map(|&query| query_rate[query]) - .fold(f64::INFINITY, f64::min) - .min(1e9) - } - ContextKind::Shared(_) => 1.0, - }) - .collect(); - let numbers = milp - .inputs - .candidate_data - .iter() - .map(|data| data.usage.as_ref().ok().map(NodeNumbers::of)) - .collect(); - let mut readers = vec![0; milp.inputs.candidate_data.len()]; - for &m in milp.inputs.context_of_shareable.keys() { - readers[m.0] = milp.queries_reaching(m).len(); - } - Self { - rate, - numbers, - readers, - } - } -} - -/// One query's derivation into a plan. -struct Derivation<'m, 'a> { - milp: &'m SelectionMilp<'a>, - estimates: &'m Estimates, - query: usize, - objective: Objective, - approximate: Approximate, - share: bool, - /// Divide a shareable candidate's own cost among the queries that may - /// reach it. - amortized: bool, - /// Shareable candidates estimated free to read (greedy sharing). - preferred: &'m BTreeSet, - undo: Vec, - steps: usize, - /// The query's output references. - references: Vec, -} - -impl Derivation<'_, '_> { - /// Whether the query may use `candidate`. - fn allowed(&self, candidate: PhysicalCandidateId) -> bool { - let inputs = self.milp.inputs; - if inputs.candidate_data[candidate.0].approximate { - if self.approximate == Approximate::None - || self.milp.entries[self.query] - .requirements - .accuracy - .is_exact() - { - return false; - } - if let Some(AccuracyWeight::Unusable(_)) = - inputs.accuracy_weights[self.query].get(&candidate) - { - return false; - } - if let Approximate::SizedFor(shares) = self.approximate { - if inputs.candidate_data[candidate.0].accuracy_budget_parts < shares { - return false; - } - } - } - // A candidate needing two or more reading queries is not planned - // alone; one some of the query's outputs may not reach is not used. - let references = &self.references; - !self - .milp - .physical - .reader_constraints - .iter() - .any(|constraint| match constraint { - ReaderConstraint::MinimumQueryReaders { - candidate: c, - minimum, - .. - } => *c == candidate && *minimum > 1, - ReaderConstraint::AllowedOutputs { - candidate: c, - outputs, - } => *c == candidate && references.iter().any(|r| !outputs.contains(r)), - ReaderConstraint::IncompatibleOutputs { - candidate: c, - left, - right, - } => *c == candidate && references.contains(left) && references.contains(right), - ReaderConstraint::HighestSharedQueryMaterialization { .. } => false, - }) - } - - /// The estimated cost (or latency) of `candidate`'s own run read in - /// `context`, plus the cheapest subtrees of its inputs. - fn estimate( - &self, - plan: &Plan, - memo: &mut HashMap<(usize, PhysicalCandidateId), f64>, - context: usize, - candidate: PhysicalCandidateId, - ) -> f64 { - if let Some(&known) = memo.get(&(context, candidate)) { - return known; - } - memo.insert((context, candidate), f64::INFINITY); - let milp = self.milp; - let inputs = milp.inputs; - let value = (|| { - if !self.allowed(candidate) { - return f64::INFINITY; - } - let Some(numbers) = self.estimates.numbers[candidate.0] else { - return f64::INFINITY; - }; - let kind = inputs.kind[candidate.0]; - // Where its own run is and where it reads its inputs. - let (run_context, already) = if milp.computes(context, candidate) { - (context, false) - } else { - match kind { - RunKind::Shareable => match milp.context_of(candidate) { - Some(own) => (own, plan.selected.contains(&candidate)), - None => return f64::INFINITY, - }, - RunKind::Ingestion => { - let Some(ingestion) = milp - .ingestion_context_read_by(context) - .filter(|&at| milp.choice[at].contains_key(&candidate)) - else { - return f64::INFINITY; - }; - let chosen = plan.chosen[ingestion].get(&inputs.slot[candidate.0]); - (ingestion, chosen == Some(&candidate)) - } - RunKind::PerContext => return f64::INFINITY, - } - }; - if (self.share && already) || self.preferred.contains(&candidate) { - return 0.0; - } - let mut own = match self.objective { - Objective::Cost => { - numbers.execution_cost * self.estimates.rate[run_context] - + numbers.standing_cost - } - Objective::Latency => numbers.latency_ms, - }; - if self.amortized && kind == RunKind::Shareable && self.objective == Objective::Cost { - own /= self.estimates.readers[candidate.0].max(1) as f64; - } - let mut total = own; - for producers in &inputs.producer_candidates_per_input[candidate.0] { - let best = producers - .iter() - .filter(|producer| milp.choice[run_context].contains_key(producer)) - .filter(|producer| { - kind == RunKind::Shareable - || inputs.slot[producer.0] != inputs.slot[candidate.0] - }) - .map(|&producer| self.estimate(plan, memo, run_context, producer)) - .fold(f64::INFINITY, f64::min); - total += best; - } - total - })(); - memo.insert((context, candidate), value); - value - } - - fn derive_query(&mut self, plan: &mut Plan) -> Result<(), ()> { - let milp = self.milp; - let mut memo = HashMap::new(); - for (reference, (query, _)) in output_references(&milp.physical.query_roots) - .into_iter() - .enumerate() - { - if query != self.query { - continue; - } - let allowed: Vec = milp.inputs.output_candidates[reference] - .iter() - .copied() - .filter(|candidate| milp.choice[query].contains_key(candidate)) - .collect(); - self.read(plan, &mut memo, query, &allowed)?; - } - Ok(()) - } - - /// Makes `context` give one of `allowed` (one input's producers, or an - /// output's candidates). - fn read( - &mut self, - plan: &mut Plan, - memo: &mut HashMap<(usize, PhysicalCandidateId), f64>, - context: usize, - allowed: &[PhysicalCandidateId], - ) -> Result { - self.steps += 1; - if self.steps > MOST_STEPS_PER_QUERY { - return Err(()); - } - let milp = self.milp; - let inputs = milp.inputs; - // A slot already chosen here with an allowed candidate. - for &candidate in allowed { - if plan.chosen[context].get(&inputs.slot[candidate.0]) == Some(&candidate) { - return Ok(candidate); - } - } - let mut options: Vec<(f64, PhysicalCandidateId)> = allowed - .iter() - .copied() - .filter(|candidate| milp.choice[context].contains_key(candidate)) - .filter(|candidate| !plan.chosen[context].contains_key(&inputs.slot[candidate.0])) - .map(|candidate| (self.estimate(plan, memo, context, candidate), candidate)) - .collect(); - options.retain(|(estimate, _)| estimate.is_finite()); - options.sort_by(|a, b| a.0.total_cmp(&b.0).then(a.1.cmp(&b.1))); - for (_, candidate) in options { - let mark = self.undo.len(); - if self.choose(plan, memo, context, candidate).is_ok() { - return Ok(candidate); - } - let undo = self.undo.split_off(mark); - undo_all(plan, undo); - } - Err(()) - } - - /// Chooses `candidate` for its slot in `context` and derives what it - /// needs. - fn choose( - &mut self, - plan: &mut Plan, - memo: &mut HashMap<(usize, PhysicalCandidateId), f64>, - context: usize, - candidate: PhysicalCandidateId, - ) -> Result<(), ()> { - let milp = self.milp; - let inputs = milp.inputs; - let slot = inputs.slot[candidate.0]; - plan.chosen[context].insert(slot, candidate); - self.undo.push(Undo::Chose(context, slot)); - if milp.computes(context, candidate) { - for (input, producers) in inputs.producer_candidates_per_input[candidate.0] - .iter() - .enumerate() - { - let producers: Vec = producers - .iter() - .copied() - .filter(|producer| inputs.slot[producer.0] != slot) - .collect(); - let read = self.read(plan, memo, context, &producers)?; - if milp.slot_links.contains_key(&(context, candidate, input)) { - plan.slot_reads - .insert((context, candidate, input), inputs.slot[read.0]); - self.undo.push(Undo::SlotRead(context, candidate, input)); - } - } - return Ok(()); - } - match inputs.kind[candidate.0] { - RunKind::Shareable => { - if plan.selected.contains(&candidate) { - return Ok(()); - } - plan.selected.insert(candidate); - self.undo.push(Undo::Selected(candidate)); - let own = milp.context_of(candidate).ok_or(())?; - for producers in &inputs.producer_candidates_per_input[candidate.0] { - self.read(plan, memo, own, producers)?; - } - Ok(()) - } - RunKind::Ingestion => { - let ingestion = milp.ingestion_context_read_by(context).ok_or(())?; - if context == ingestion { - return Err(()); - } - self.read(plan, memo, ingestion, &[candidate]).map(|_| ()) - } - RunKind::PerContext => Err(()), - } - } -} - -/// The runs query `query` depends on in `plan`: `(context, candidate)` for -/// candidates computed in its context and in the contexts of the -/// shareable candidates it reaches, and each such shareable candidate -/// `(its context, itself)`. -fn runs_reached( - milp: &SelectionMilp<'_>, - plan: &Plan, - query: usize, -) -> BTreeSet<(usize, PhysicalCandidateId)> { - let inputs = milp.inputs; - let mut reached = BTreeSet::new(); - let mut stack: Vec<(usize, PhysicalCandidateId)> = Vec::new(); - for (reference, (q, _)) in output_references(&milp.physical.query_roots) - .into_iter() - .enumerate() - { - if q != query { - continue; - } - for &candidate in &inputs.output_candidates[reference] { - if plan.chosen[query].get(&inputs.slot[candidate.0]) == Some(&candidate) { - stack.push((query, candidate)); - } - } - } - while let Some((context, candidate)) = stack.pop() { - let (run_context, own) = if milp.computes(context, candidate) { - (context, false) - } else { - match inputs.kind[candidate.0] { - RunKind::Shareable => match milp.context_of(candidate) { - Some(own) => (own, true), - None => continue, - }, - RunKind::Ingestion => match milp.ingestion_context_read_by(context) { - Some(ingestion) => (ingestion, false), - None => continue, - }, - RunKind::PerContext => continue, - } - }; - if !reached.insert((run_context, candidate)) { - continue; - } - let slot = inputs.slot[candidate.0]; - for producers in &inputs.producer_candidates_per_input[candidate.0] { - for &producer in producers { - if !own && inputs.slot[producer.0] == slot { - continue; - } - if plan.chosen[run_context].get(&inputs.slot[producer.0]) == Some(&producer) { - stack.push((run_context, producer)); - break; - } - } - } - } - reached -} - -/// Whether query `query`'s approximate runs in `plan` meet the A1 rows. -fn meets_accuracy(milp: &SelectionMilp<'_>, plan: &Plan, query: usize) -> bool { - let inputs = milp.inputs; - let runs: Vec = runs_reached(milp, plan, query) - .into_iter() - .map(|(_, candidate)| candidate) - .filter(|candidate| inputs.candidate_data[candidate.0].approximate) - .filter(|candidate| inputs.accuracy_weights[query].contains_key(candidate)) - .collect(); - if runs.is_empty() { - return true; - } - if milp.entries[query].requirements.accuracy.is_exact() { - return false; - } - let count = runs.len(); - for &candidate in &runs { - if matches!( - inputs.accuracy_weights[query].get(&candidate), - Some(AccuracyWeight::Unusable(_)) - ) { - return false; - } - if inputs.candidate_data[candidate.0].accuracy_budget_parts < count { - return false; - } - } - !inputs.accuracy_excluded_pairs[query] - .iter() - .any(|(first, second)| runs.contains(first) && runs.contains(second)) -} - -/// Whether query `query`'s estimated latency in `plan` is within its bound. -fn meets_latency(milp: &SelectionMilp<'_>, plan: &Plan, query: usize) -> bool { - let Some(bound) = latency_bound_ms(&milp.entries[query]) else { - return true; - }; - let request_ms = milp.cost_inputs.measured_operation_costs.ns_per_evaluation * 1e-6; - let latency: f64 = runs_reached(milp, plan, query) - .into_iter() - .filter(|&(context, _)| milp.runs_at_query_time(context)) - .filter_map(|(_, candidate)| milp.inputs.candidate_data[candidate.0].usage.as_ref().ok()) - .map(|usage| usage.own_latency_ms) - .sum(); - debug(|| { - format!( - "warm start: q{query} estimated latency {} ms, bound {bound} ms", - latency + request_ms - ) - }); - latency + request_ms <= bound -} - -/// Prints `message` to stderr when `ASAP_STAGE3_DEBUG` is set. -pub(crate) fn debug(message: impl FnOnce() -> String) { - if std::env::var_os("ASAP_STAGE3_DEBUG").is_some() { - eprintln!("{}", message()); - } -} diff --git a/crates/plan-selection/src/models.rs b/crates/plan-selection/src/models.rs index bbc0d88f0..763476654 100644 --- a/crates/plan-selection/src/models.rs +++ b/crates/plan-selection/src/models.rs @@ -1,9 +1,6 @@ -//! The models Stage 3 checks and compares with: [`AccuracyCheckInputs`] -//! (validity) and [`CostInputs`] (the objective), and the engine's view of -//! both, [`SelectionModels`]. +//! The accuracy models Stage 3 checks with: [`AccuracyCheckInputs`]. use asap_accuracy::{AccuracyEvidenceProvider, AccuracyModel}; -use asap_cost::CostInputs; /// Inputs of the composed-accuracy check. `raw_data_retained` is a Stage 2 /// input only; to ignore accuracy guarantees the facade passes an @@ -18,56 +15,3 @@ pub struct AccuracyCheckInputs<'a> { /// guarantees). pub evidence: &'a dyn AccuracyEvidenceProvider, } - -/// Stage 3's accuracy model and evidence with its cost inputs, as the -/// engine reads them. -#[derive(Clone, Copy)] -pub(crate) struct SelectionModels<'a> { - pub accuracy: &'a dyn AccuracyModel, - pub evidence: &'a dyn AccuracyEvidenceProvider, - pub cost_inputs: &'a CostInputs, -} - -impl<'a> SelectionModels<'a> { - pub(crate) fn new( - accuracy_checks: &AccuracyCheckInputs<'a>, - cost_inputs: &'a CostInputs, - ) -> Self { - Self { - accuracy: accuracy_checks.model, - evidence: accuracy_checks.evidence, - cost_inputs, - } - } - - /// The built-in accuracy model and no evidence, under default cost - /// inputs. - #[cfg(test)] - pub(crate) fn builtin() -> SelectionModels<'static> { - static DEFAULT_COST_INPUTS: std::sync::LazyLock = - std::sync::LazyLock::new(CostInputs::default); - SelectionModels { - accuracy: &asap_accuracy::BuiltinAccuracyModel, - evidence: &asap_accuracy::NoAccuracyEvidence, - cost_inputs: &DEFAULT_COST_INPUTS, - } - } - - #[cfg(test)] - pub(crate) fn with_accuracy(mut self, accuracy: &'a dyn AccuracyModel) -> Self { - self.accuracy = accuracy; - self - } - - #[cfg(test)] - pub(crate) fn with_evidence(mut self, evidence: &'a dyn AccuracyEvidenceProvider) -> Self { - self.evidence = evidence; - self - } - - #[cfg(test)] - pub(crate) fn with_cost_inputs(mut self, cost_inputs: &'a CostInputs) -> Self { - self.cost_inputs = cost_inputs; - self - } -} diff --git a/crates/plan-selection/src/report.rs b/crates/plan-selection/src/report.rs index e2617b580..7cac6899e 100644 --- a/crates/plan-selection/src/report.rs +++ b/crates/plan-selection/src/report.rs @@ -7,8 +7,6 @@ use asap_cost::analytical::AnalyticalCostError; use asap_cost::{CostModel, PlanCost}; use asap_types::ir::physical_export::PhysicalASAPNodeId; use asap_types::ir::properties::AccuracyMetric; -#[cfg(any(test, feature = "reference-pricer"))] -use asap_types::ir::QueryRoot; use asap_types::stages::{ DagOfSelectedCandidates, PhysicalCandidateId, PrunedCandidate, SelectedNodeId, }; @@ -91,7 +89,7 @@ pub enum RejectionCause { /// order. TiedLostOnEnumerationOrder, /// No plan meets every query's accuracy target, latency bound and the - /// catalog's reader constraints together: the selection MILP is + /// catalog's consumer constraints together: the selection MILP is /// infeasible. The reason names a query when it has no plan even alone. Infeasible, } @@ -183,32 +181,6 @@ impl std::fmt::Display for AcceptedAccuracy { } } -/// The whole-plan selector's result (`PlanSelector`): -/// the selected candidate among whole physical plans and the plan to run, -/// with what it kept of the other candidates. The reference pricer (feature -/// `reference-pricer`) selects among enumerated plans with it. Keeping -/// every candidate, `costs` and `accepted` list every valid candidate and -/// `rejected` every other one; otherwise only the selected candidate is -/// kept, and `rejected` holds the first rejection of each cause. -#[cfg(any(test, feature = "reference-pricer"))] -#[derive(Debug, Clone, PartialEq)] -pub struct WholePlanSelection { - pub selected: String, - /// The selected candidate's query roots with every weight requirement - /// Stage 3 confirmed made a proof: the plan to run. The Stage 2 - /// candidate keeps the open requirement. - pub plan: Vec, - pub costs: BTreeMap, - /// For each valid candidate, per query, the accuracy Stage 3 accepted. - pub accepted: BTreeMap>, - pub rejected: Vec, - /// The selected plan's tumbling-window retention, by node id in its physical DAG - /// ([`RealizedPlan::tumbling_window_retention_ms`](asap_types::stages::RealizedPlan::tumbling_window_retention_ms)). - pub tumbling_window_retention_ms: BTreeMap, - /// The physical candidates checked, valid or not. - pub candidates_checked: u64, -} - /// Stage 3's result ([`select_plan`](crate::select_plan)): the selected DAG /// over the [`PhysicalASAPMemoCandidates`](asap_types::stages::PhysicalASAPMemoCandidates) /// it was given, its cost and accepted accuracy, and the search report. diff --git a/crates/plan-selection/src/select.rs b/crates/plan-selection/src/select.rs index ed5546432..f0468ae33 100644 --- a/crates/plan-selection/src/select.rs +++ b/crates/plan-selection/src/select.rs @@ -1,63 +1,21 @@ -//! Stage 3, plan selection: [`select_plan`] runs the group selection MILP -//! (#1095). Inputs it does not price yet (cost models 2 and 3, or a measured -//! operation cost table without a frozen calibration factor) are an -//! `UnsupportedCostModel` error (#888). -//! -//! One MILP solve and one realization, no loop: -//! 1. the MILP's inputs (candidate dominance, execution contexts, accuracy -//! weights) and the model, solved once with HiGHS from a warm start to -//! proven optimality (no time limit); -//! 2. the solution decoded into a [`DagOfSelectedCandidates`] -//! ([`crate::milp::decode`]) and validated against the catalog; -//! 3. the plan realized once through the injected realizer: a rejection is -//! a Stage 2 catalog bug ([`ErrorSelectingPlan::RealizerRejected`]); -//! 4. exact checks: `asap_cost::plan_cost` of the realized plan minus the -//! request fees equals the objective within a relative 10⁻⁶ (otherwise -//! an error, or, when the model encoded some term inexactly, -//! [`SearchReport::inexact_objective`]); every query's accuracy -//! (`satisfies`) and latency hold, or it is an error. -//! -//! With `ASAP_STAGE3_REPORT=`, one JSON line per run is appended -//! there: the model's size per family and per scope (the shareable -//! candidates' contexts apart), the timings, the warm start's cost, the -//! objective, `plan_cost` and the dual bound (measurement only). - -use std::collections::BTreeMap; -use std::time::Instant; - -use asap_cost::{kept_tumbling_window_chains, node_map, plan_cost, CostInputs}; -use asap_types::ir::{ASAPOp, Operator}; -use asap_types::stages::{ - AssignmentRealizer, DagOfSelectedCandidates, PhysicalASAPMemoCandidates, - PhysicalCandidateError, PhysicalCandidateId, PruneReason, PrunedCandidate, RealizedPlan, - SelectedNodeId, -}; -use asap_types::workload::bounds::confirmed_weights; +//! Stage 3, plan selection: [`select_plan`] builds the selection MILP over +//! Stage 2's physical candidates and admissible edges +//! ([`crate::milp::model`]), solves it, decodes $\{x = 1\}$ and $\{y = 1\}$ +//! into a [`DagOfSelectedCandidates`](asap_types::stages::DagOfSelectedCandidates), +//! and realizes it once through the injected realizer. + +use asap_cost::CostInputs; +use asap_types::stages::{AssignmentRealizer, PhysicalASAPMemoCandidates}; use asap_types::workload::{DataWorkload, QueryWorkloadEntry}; -use crate::checks::{accuracy_check, accuracy_targets, latency_violation}; -use crate::groups::{prepare_milp_inputs_with, MilpInputs}; -use crate::milp::decode::{decode, ChosenCandidates}; -use crate::milp::model::{ - build_selection_milp, solve_selection_milp, SelectionMilpError, SelectionMilpOptions, -}; -use crate::models::{AccuracyCheckInputs, SelectionModels}; -use crate::report::{ - ErrorSelectingPlan, InexactObjective, Rejection, RejectionCause, SearchReport, SearchStatus, - Selection, -}; - -/// The largest relative difference between the objective and `plan_cost` -/// that counts as equal. -const COST_EQUALITY_TOLERANCE: f64 = 1e-6; +use crate::models::AccuracyCheckInputs; +use crate::report::{ErrorSelectingPlan, Selection}; /// Stage 3, plan selection: select one plan among `physical`'s candidates -/// with the group selection MILP (module documentation). The selection is -/// realized through `realizer`, checked against every query's accuracy -/// target (`accuracy_checks`) and latency bound, and priced under -/// `cost_inputs`. `entries[q]` is query `q`'s workload entry (accuracy -/// target, latency bound, recurrence); `data` the data workload with its -/// bounds. The solve has no time limit: it runs to proven optimality. +/// (module documentation). The selection is realized through `realizer`, +/// checked against every query's accuracy target (`accuracy_checks`) and +/// priced under `cost_inputs`. `entries[q]` is query `q`'s workload entry; +/// `data` the data workload with its bounds. /// /// Err: invalid or unsupported cost inputs, an infeasible catalog, a /// solver failure, or the realizer rejected the selected plan @@ -73,301 +31,6 @@ pub fn select_plan( cost_inputs .validate() .map_err(ErrorSelectingPlan::InvalidSettings)?; - // The MILP's coefficients and `plan_cost` must use one calibration: a - // measured cost table needs a frozen factor. - if cost_inputs.frozen_calibration_factor.is_none() - && !cost_inputs.measured_operation_costs.operators.is_empty() - { - return Err(ErrorSelectingPlan::UnsupportedCostModel( - "a measured operation cost table without a frozen calibration factor".into(), - )); - } - let formula_scale = cost_inputs.frozen_calibration_factor.unwrap_or(1.0); - let models = SelectionModels::new(accuracy_checks, cost_inputs); - // The A1 rows decide accuracy only when the built-in rule does: a model - // or evidence that may accept an output otherwise is asked after the - // solve. - let enforce_accuracy = !(accuracy_checks.model.may_accept_regardless_of_guarantee() - || accuracy_checks.evidence.may_provide_evidence()); - - let construction_started = Instant::now(); - let inputs = prepare_milp_inputs_with(physical, entries, data, &models); - let pruned_candidates = pruned_before_the_milp(physical, &inputs); - let milp = build_selection_milp( - physical, - &inputs, - entries, - cost_inputs, - SelectionMilpOptions { - enforce_accuracy, - ..SelectionMilpOptions::default() - }, - ) - .map_err(selection_error)?; - let build_ms = construction_started.elapsed().as_millis() as u64; - let solution = solve_selection_milp(milp, build_ms).map_err(|error| { - with_uncostable_reasons(selection_error(error), &inputs, &pruned_candidates) - })?; - - let validation_started = Instant::now(); - let chosen = ChosenCandidates { - chosen: solution.chosen.clone(), - selected: solution.selected.clone(), - slot_reads: solution.slot_reads.clone(), - }; - let selected_dag = decode(physical, &inputs, &chosen).map_err(|reason| { - ErrorSelectingPlan::Solver(format!("the selection does not decode: {reason}")) - })?; - selected_dag.validate(physical).map_err(|reason| { - ErrorSelectingPlan::Solver(format!("the decoded selection is invalid: {reason}")) - })?; - let realized = realize(physical, &selected_dag, realizer)?; - let plan = &realized.plan; - let recurrence: Vec<_> = entries.iter().map(|e| e.recurrence.clone()).collect(); - let ranges: Vec<_> = entries.iter().map(|e| e.range_evaluation).collect(); - let (cost, _) = plan_cost( - &plan.dag, - &node_map(&plan.dag), - &kept_tumbling_window_chains(&plan.dag), - &recurrence, - &ranges, - data, - cost_inputs, - formula_scale, - ) - .map_err(|(node, error)| ErrorSelectingPlan::Cost { - candidate: plan.id.clone(), - node, - error, - })?; - let plan_cost_per_second = cost.cost_per_second.ok_or_else(|| { - ErrorSelectingPlan::UnsupportedCostModel("a cost model without a cost per second".into()) - })? - solution.request_cost_per_second; - let objective_per_second = solution.cost_per_second; - let difference = (objective_per_second - plan_cost_per_second).abs(); - let tolerance = COST_EQUALITY_TOLERANCE * plan_cost_per_second.abs().max(f64::MIN_POSITIVE); - let within = difference <= tolerance; - report_run(&solution, objective_per_second, plan_cost_per_second); - let inexact_objective = if within { - None - } else if solution.size.inexact.is_empty() { - return Err(ErrorSelectingPlan::ObjectiveDiffersFromPlanCost { - objective_per_second, - plan_cost_per_second, - }); - } else { - Some(InexactObjective { - objective_per_second, - plan_cost_per_second, - inexact_terms: solution - .size - .inexact - .iter() - .map(|(reason, count)| (reason.to_string(), *count)) - .collect(), - }) - }; - let accepted_accuracy_per_query = - accuracy_check(plan, &accuracy_targets(entries), data, &models).map_err(|failure| { - ErrorSelectingPlan::SelectedPlanFailedCheck { - cause: failure.cause, - reason: failure.reason, - } - })?; - if let Some((cause, reason)) = latency_violation(&cost, entries) { - return Err(ErrorSelectingPlan::SelectedPlanFailedCheck { cause, reason }); - } - let nodes_with_proven_non_negative_weights = proven_non_negative_weights(&realized, data); - let validation_ms = validation_started.elapsed().as_millis() as u64; - Ok(Selection { - selected_dag, - nodes_with_proven_non_negative_weights, - cost, - accepted_accuracy_per_query, - pruned_candidates, - search_report: SearchReport { - status: SearchStatus::Optimal, - // The dual bound is on the objective, which leaves out the - // request fees every plan pays; the selection's cost has them. - cost_lower_bound_per_second: solution - .cost_lower_bound_per_second - .is_finite() - .then_some(solution.cost_lower_bound_per_second + solution.request_cost_per_second), - milp_variables: solution.size.variables, - milp_constraints: solution.size.rows, - model_construction_ms: solution.build_ms + solution.warm_start_ms, - solver_ms: solution.solve_ms, - validation_ms, - inexact_objective, - }, - }) -} - -/// The selection MILP's error as Stage 3's. -fn selection_error(error: SelectionMilpError) -> ErrorSelectingPlan { - let infeasible = |id: String, cause: RejectionCause, reason: String| { - ErrorSelectingPlan::NoValidCandidate(vec![Rejection { id, cause, reason }]) - }; - match error { - SelectionMilpError::UnsupportedCostModel(model) => { - ErrorSelectingPlan::UnsupportedCostModel(model.to_string()) - } - SelectionMilpError::EntryCount { roots, entries } => ErrorSelectingPlan::EntryCount { - candidate: "the physical memo".into(), - roots, - entries, - }, - SelectionMilpError::Infeasible => infeasible( - "workload".into(), - RejectionCause::Infeasible, - "no plan meets every query's accuracy target, latency bound and the reader \ - constraints together (the selection MILP is infeasible)" - .into(), - ), - SelectionMilpError::Cost(reason) | SelectionMilpError::Solver(reason) => { - ErrorSelectingPlan::Solver(reason) - } - } -} - -/// An infeasible selection with the reasons `asap-cost` could not cost -/// the candidates the model left out (e.g. a source missing the fact its -/// pricing needs), each once in candidate order: those candidates may be -/// why no plan is left. -fn with_uncostable_reasons( - error: ErrorSelectingPlan, - inputs: &MilpInputs, - pruned_candidates: &[PrunedCandidate], -) -> ErrorSelectingPlan { - let ErrorSelectingPlan::NoValidCandidate(mut rejections) = error else { - return error; - }; - let mut reasons: Vec<&str> = Vec::new(); - for pruned in pruned_candidates { - if pruned.reason != PruneReason::UsageNotCostable { - continue; - } - if let Err(reason) = &inputs.candidate_data[pruned.candidate.0].usage { - if !reasons.contains(&reason.as_str()) { - reasons.push(reason); - } - } - } - if let Some(rejection) = rejections.first_mut() { - if !reasons.is_empty() { - rejection.reason = format!( - "{}; candidates left out because they could not be costed: {}", - rejection.reason, - reasons.join("; ") - ); - } - } - ErrorSelectingPlan::NoValidCandidate(rejections) -} - -/// What Stage 3 removes before the MILP: dominated candidates, and those -/// `asap-cost` could not cost (the model leaves them out), in candidate -/// order. -fn pruned_before_the_milp( - physical: &PhysicalASAPMemoCandidates, - inputs: &MilpInputs, -) -> Vec> { - let mut reasons: BTreeMap = inputs - .pruned_candidates - .iter() - .map(|pruned| (pruned.class, PruneReason::DominatedBy(pruned.dominated_by))) - .collect(); - for (id, data) in inputs.candidate_data.iter().enumerate() { - if data.usage.is_err() { - reasons - .entry(PhysicalCandidateId(id)) - .or_insert(PruneReason::UsageNotCostable); - } - } - reasons - .into_iter() - .map(|(candidate, reason)| PrunedCandidate { - group: physical.memo_group_of(candidate), - candidate, - reason, - }) - .collect() -} - -/// `selected` realized through `realizer`; a rejection is a Stage 2 -/// catalog bug. -fn realize( - physical: &PhysicalASAPMemoCandidates, - selected: &DagOfSelectedCandidates, - realizer: &dyn AssignmentRealizer, -) -> Result { - realizer - .realize(physical, selected) - .map_err(|error| match error { - PhysicalCandidateError::RejectedAssignment(reason) => { - ErrorSelectingPlan::RealizerRejected(reason) - } - error => ErrorSelectingPlan::RealizerRejected(error.to_string()), - }) -} - -/// The selected nodes whose summary (`SummaryAgg` or `SummaryTumblingWindowChain`) reads -/// update weights `data` proves non-negative although the candidate leaves -/// the requirement open. -fn proven_non_negative_weights( - realized: &RealizedPlan, - data: &DataWorkload, -) -> Vec { - realized - .operator_node_of_selected_node - .iter() - .enumerate() - .filter(|(_, node)| match &node.operator { - Operator::ASAP( - ASAPOp::SummaryAgg { input, .. } | ASAPOp::SummaryTumblingWindowChain { input, .. }, - ) => confirmed_weights(&input.weight_domain, data) != input.weight_domain, - _ => false, - }) - .map(|(node, _)| SelectedNodeId(node)) - .collect() -} - -#[cfg(test)] -mod tests; - -/// Appends one JSON line describing this run to `ASAP_STAGE3_REPORT`, if -/// set (measurement only; failures to write are ignored). -fn report_run( - solution: &crate::milp::model::SelectionMilpSolution, - objective_per_second: f64, - plan_cost_per_second: f64, -) { - let Ok(path) = std::env::var("ASAP_STAGE3_REPORT") else { - return; - }; - let line = serde_json::json!({ - "variables": solution.size.variables, - "binaries": solution.size.binaries, - "rows": solution.size.rows, - "nonzeros": solution.size.nonzeros, - "scopes": solution.size.scopes.iter().map(|(scope, size)| (format!("{scope:?}"), size)).collect::>(), - "families": solution.size.families, - "inexact": solution.size.inexact, - "build_ms": solution.build_ms, - "warm_start_ms": solution.warm_start_ms, - "solve_ms": solution.solve_ms, - "warm_start_cost_per_second": solution.warm_start_cost_per_second, - "objective_per_second": objective_per_second, - "plan_cost_per_second": plan_cost_per_second, - "dual_bound_per_second": solution.cost_lower_bound_per_second, - "parts": solution.parts, - }); - use std::io::Write; - if let Ok(mut file) = std::fs::OpenOptions::new() - .create(true) - .append(true) - .open(path) - { - let _ = writeln!(file, "{line}"); - } + let _ = (physical, entries, data, accuracy_checks, realizer); + todo!("#1207 PR 6: the baseline MILP over x, y, z, t, its decode and realization") } diff --git a/crates/plan-selection/src/select/tests.rs b/crates/plan-selection/src/select/tests.rs deleted file mode 100644 index 68ce48681..000000000 --- a/crates/plan-selection/src/select/tests.rs +++ /dev/null @@ -1,173 +0,0 @@ -use super::*; -use crate::milp::model::tests::{approximate, cost_inputs, shared_sum, through_stages, Fixture}; -use asap_accuracy::{AccuracyModel, BuiltinAccuracyModel, NoAccuracyEvidence}; -use asap_types::ir::properties::{AccuracyGuarantee, AccuracyTarget}; -use asap_types::stages::RealizedPlan; - -fn realizer(fixture: &Fixture) -> impl AssignmentRealizer + '_ { - asap_physical_asap_optimization::Stage2AssignmentRealizer { - original_roots: &fixture.roots, - entries: &fixture.entries, - data: &fixture.data, - query_time_results_retained: fixture.query_time_results_retained, - } -} - -fn select( - fixture: &Fixture, - accuracy: &dyn AccuracyModel, - realizer: &dyn AssignmentRealizer, -) -> Result { - select_plan( - &fixture.physical, - &fixture.entries, - &fixture.data, - &AccuracyCheckInputs { - model: accuracy, - evidence: &NoAccuracyEvidence, - }, - &cost_inputs(), - realizer, - ) -} - -/// `fixture` with metric `m` declared non-negative. -fn non_negative(mut fixture: Fixture) -> Fixture { - let source = fixture.data.sources.get_mut("m").expect("m"); - source.values_non_negative = true; - fixture -} - -// The group selection MILP's selection: an optimal, valid plan whose cost is -// `plan_cost`'s, the MILP's size and every query's accepted accuracy. The -// ratio of quantiles is a regression of #1127: the A1 rows admitted two -// KLL sketches at parts = 2, whose rank errors compose no guarantee through -// the quotient, so the search returned `SelectedPlanFailedCheck`. -#[test] -fn the_milp_search_selects_a_checked_plan() { - let ratio = || { - through_stages( - &[approximate( - "quantile_over_time(0.5, m[5m]) / quantile_over_time(0.9, m[5m])", - 60_000, - 0.01, - )], - true, - ) - }; - for fixture in [ - shared_sum(), - through_stages( - &[approximate("quantile_over_time(0.9, m[5m])", 60_000, 0.01)], - true, - ), - ratio(), - non_negative(ratio()), - ] { - let stage2 = realizer(&fixture); - let selection = select(&fixture, &BuiltinAccuracyModel, &stage2).unwrap(); - selection.selected_dag.validate(&fixture.physical).unwrap(); - let report = &selection.search_report; - assert_eq!(report.status, SearchStatus::Optimal); - assert_eq!(report.inexact_objective, None); - assert!(report.milp_variables > 0 && report.milp_constraints > 0); - let cost = selection.cost.cost_per_second.unwrap(); - assert!(report.relative_gap(cost).unwrap() <= 1e-6, "{report:?}"); - assert_eq!( - selection.accepted_accuracy_per_query.len(), - fixture.entries.len() - ); - for pruned in &selection.pruned_candidates { - assert!(matches!( - pruned.reason, - PruneReason::DominatedBy(_) | PruneReason::UsageNotCostable - )); - assert_eq!( - pruned.group, - fixture.physical.memo_group_of(pruned.candidate) - ); - } - } -} - -/// A realizer that rejects every selection. -struct Rejecting; - -impl AssignmentRealizer for Rejecting { - fn realize( - &self, - _physical: &PhysicalASAPMemoCandidates, - _selected: &DagOfSelectedCandidates, - ) -> Result { - Err(PhysicalCandidateError::RejectedAssignment( - "a combination the catalog allows".into(), - )) - } -} - -// A realizer rejection is a Stage 2 catalog bug: an error, no re-solve. -#[test] -fn a_realizer_rejection_is_realizer_rejected() { - let fixture = shared_sum(); - match select(&fixture, &BuiltinAccuracyModel, &Rejecting) { - Err(ErrorSelectingPlan::RealizerRejected(reason)) => { - assert!(reason.contains("the catalog allows"), "{reason}") - } - other => panic!("{other:?}"), - } -} - -/// An accuracy model that accepts no guarantee, exact included, and never -/// accepts an output otherwise: the A1 rows still run, so the MILP's plan -/// misses `satisfies`, as it would with a wrong weight table. -struct AcceptsNothing; - -impl AccuracyModel for AcceptsNothing { - fn satisfies(&self, _guarantee: &AccuracyGuarantee, _target: &AccuracyTarget) -> bool { - false - } - - fn may_accept_regardless_of_guarantee(&self) -> bool { - false - } -} - -// A selected plan that misses `satisfies` is an error, never returned. -#[test] -fn an_accuracy_miss_is_an_error() { - let fixture = shared_sum(); - let stage2 = realizer(&fixture); - match select(&fixture, &AcceptsNothing, &stage2) { - Err(ErrorSelectingPlan::SelectedPlanFailedCheck { cause, .. }) => { - assert!(cause.failed_a_check(), "{cause:?}") - } - other => panic!("{other:?}"), - } -} - -// Cost models 2 and 3 are not supported by the group selection MILP (#888). -#[test] -fn cost_model_2_is_unsupported() { - let fixture = shared_sum(); - let costs = CostInputs { - cost_model: asap_cost::CostModel::ProvisionedCapacity( - asap_cost::ResourceUnitCost::illustrative(), - ), - ..cost_inputs() - }; - let result = select_plan( - &fixture.physical, - &fixture.entries, - &fixture.data, - &AccuracyCheckInputs { - model: &BuiltinAccuracyModel, - evidence: &NoAccuracyEvidence, - }, - &costs, - &Rejecting, - ); - assert!(matches!( - result, - Err(ErrorSelectingPlan::UnsupportedCostModel(_)) - )); -} diff --git a/crates/plan-selection/src/selector.rs b/crates/plan-selection/src/selector.rs deleted file mode 100644 index 5270385cd..000000000 --- a/crates/plan-selection/src/selector.rs +++ /dev/null @@ -1,471 +0,0 @@ -//! Choosing among whole checked plans: [`PlanSelector`] and -//! [`select_among_plans`]. The reference pricer (feature `reference-pricer`) -//! selects among enumerated plans with it. - -use std::collections::BTreeMap; - -use asap_cost::{ - worst_latency_ms, CalibrationFit, ClusterLimits, CostModel, PlanCost, PlanRanking, - ResourceUsage, COST_UNIT, -}; -use asap_types::ir::properties::AccuracyTarget; -use asap_types::physical::PhysicalWorkloadPlan; -use asap_types::workload::{DataWorkload, QueryRecurrence, QueryWorkloadEntry, RangeEvaluation}; - -use crate::checks::{accuracy_check, accuracy_targets, latency_violation}; -use crate::report::{ - AcceptedAccuracy, ErrorSelectingPlan, Rejection, RejectionCause, WholePlanSelection, -}; -use crate::SelectionModels; -use asap_cost::analytical::AnalyticalCostError; -use asap_cost::{ - kept_tumbling_window_chains, node_map, plan_cost, query_reach, tumbling_window_retention_ms, -}; - -type Ranking = PlanRanking; -use asap_types::workload::bounds::with_confirmed_weights; - -/// A cost per second for a rejection reason: three decimals, or three -/// significant digits in scientific notation when it is nonzero and below -/// 0.001 (e.g. "2.361e-8" under [`ResourceUnitCost::aws_fargate_us_east_1`]'s -/// dollars), so small costs do not all read "0.000". -fn display_cost(cost_per_second: f64) -> String { - if cost_per_second != 0.0 && cost_per_second.abs() < 1e-3 { - format!("{cost_per_second:.3e}") - } else { - format!("{cost_per_second:.3}") - } -} - -/// Select among whole plans: reject candidates that the executor cannot -/// run or that miss a query's accuracy target or latency bound (or, under -/// cost model 3, the cluster's capacity), measure the rest's resource -/// usage per second and choose one under the deployment's cost model -/// (`models.cost_inputs`): the cheapest under cost models 1 and 2, the -/// best by the cluster objective under cost model 3 (the first on ties). `entries[i]` is the workload -/// entry of `candidate.roots[i]`: its accuracy target, its latency bound and -/// how often it is evaluated. Every candidate is kept in the selection -/// ([`PlanSelector`] with `keep_every_candidate`). -#[cfg(any(test, feature = "reference-pricer"))] -pub(crate) fn select_among_plans( - cands: &[PhysicalWorkloadPlan], - entries: &[QueryWorkloadEntry], - data: &DataWorkload, - models: SelectionModels<'_>, -) -> Result { - let mut selector = PlanSelector::new(entries, data, models, true)?; - if selector.needs_calibration() { - let mut fit = CalibrationFit::default(); - for candidate in cands { - match selector.calibration_fit(candidate) { - // Rejected below when offered. - Err(ErrorSelectingPlan::Cost { - error: AnalyticalCostError::Overflow, - .. - }) => {} - fit_or_error => fit.add(fit_or_error?), - } - } - selector.set_calibration(fit); - } - // An unpriceable candidate is rejected, as the search rejects it - // (`cost_not_representable`), rather than failing the selection. - for candidate in cands { - match selector.offer(candidate) { - Err( - error @ ErrorSelectingPlan::Cost { - error: AnalyticalCostError::Overflow, - .. - }, - ) => selector.reject( - &candidate.id, - RejectionCause::CostNotRepresentable, - error.to_string(), - ), - result => result?, - } - } - selector.finish().map(|(selection, _)| selection) -} - -/// Whole plans offered one at a time, in candidate order -/// ([`select_among_plans`] over all of them at once). Each candidate is -/// checked and priced as it is offered; only the best so far is kept, -/// unless `keep_every_candidate`. Either way it selects what -/// [`select_among_plans`] selects over the same candidates. -pub struct PlanSelector<'a> { - entries: &'a [QueryWorkloadEntry], - data: &'a DataWorkload, - models: SelectionModels<'a>, - targets: Vec, - recurrence: Vec, - range_evaluations: Vec>, - ranking: Ranking, - keep_every_candidate: bool, - candidates_checked: u64, - costs: BTreeMap, - accepted: BTreeMap>, - rejected: Vec, - /// The valid candidates kept, in candidate order: every one, the best - /// so far, or the Pareto front so far. - kept: Vec, - /// The factor that calibrates the formulas' CPU time to the measured - /// table's ([`Self::set_calibration`]); 1 without a measured table. - pub(crate) formula_scale: f64, -} - -impl<'a> PlanSelector<'a> { - /// A selector for the workload `entries` over `data` under `models`. - /// Bad settings are an error here, before any candidate. - pub fn new( - entries: &'a [QueryWorkloadEntry], - data: &'a DataWorkload, - models: SelectionModels<'a>, - keep_every_candidate: bool, - ) -> Result { - models - .cost_inputs - .validate() - .map_err(ErrorSelectingPlan::InvalidSettings)?; - Ok(Self { - entries, - data, - models, - targets: accuracy_targets(entries), - recurrence: entries.iter().map(|e| e.recurrence.clone()).collect(), - range_evaluations: entries.iter().map(|e| e.range_evaluation).collect(), - ranking: Ranking::of(&models.cost_inputs.cost_model), - keep_every_candidate, - candidates_checked: 0, - costs: BTreeMap::new(), - accepted: BTreeMap::new(), - rejected: Vec::new(), - kept: Vec::new(), - formula_scale: models.cost_inputs.frozen_calibration_factor.unwrap_or(1.0), - }) - } - - /// Whether the deployment's measured cost table prices some operator, - /// so the formulas' CPU time is calibrated to it: one factor for every - /// candidate, fitted over the nodes the table prices in every candidate - /// that passes the checks before pricing. The caller then gives every - /// candidate to [`Self::calibration_fit`], adds up the fits and passes - /// the sum to [`Self::set_calibration`] before the first - /// [`Self::offer`]. - pub fn needs_calibration(&self) -> bool { - self.models.cost_inputs.frozen_calibration_factor.is_none() - && !self - .models - .cost_inputs - .measured_operation_costs - .operators - .is_empty() - } - - /// `candidate`'s part of the calibration fit: its measured nodes' CPU - /// time, as measured and as the formulas price it, when it passes the - /// checks before pricing; nothing otherwise. - pub fn calibration_fit( - &self, - candidate: &PhysicalWorkloadPlan, - ) -> Result { - if candidate.roots.len() != self.entries.len() { - return Err(ErrorSelectingPlan::EntryCount { - candidate: candidate.id.clone(), - roots: candidate.roots.len(), - entries: self.entries.len(), - }); - } - if accuracy_check(candidate, &self.targets, self.data, &self.models).is_err() { - return Ok(CalibrationFit::default()); - } - let nodes = node_map(&candidate.dag); - let kept_chains = kept_tumbling_window_chains(&candidate.dag); - let (_, fit) = plan_cost( - &candidate.dag, - &nodes, - &kept_chains, - &self.recurrence, - &self.range_evaluations, - self.data, - self.models.cost_inputs, - 1.0, - ) - .map_err(|(node, error)| ErrorSelectingPlan::Cost { - candidate: candidate.id.clone(), - node, - error, - })?; - Ok(fit) - } - - /// Calibrate the formulas with `fit`, the sum of every candidate's - /// [`Self::calibration_fit`]. - pub fn set_calibration(&mut self, fit: CalibrationFit) { - self.formula_scale = self - .models - .cost_inputs - .frozen_calibration_factor - .unwrap_or_else(|| fit.factor()); - } - - /// Check and price `candidate`, the next in candidate order, and keep - /// it if it can still be selected. - #[cfg(any(test, feature = "reference-pricer"))] - pub fn offer(&mut self, candidate: &PhysicalWorkloadPlan) -> Result<(), ErrorSelectingPlan> { - if candidate.roots.len() != self.entries.len() { - return Err(ErrorSelectingPlan::EntryCount { - candidate: candidate.id.clone(), - roots: candidate.roots.len(), - entries: self.entries.len(), - }); - } - self.candidates_checked += 1; - let accepted_accuracy = - match accuracy_check(candidate, &self.targets, self.data, &self.models) { - Ok(accepted) => accepted, - Err(failed) => { - self.reject(&candidate.id, failed.cause, failed.reason); - return Ok(()); - } - }; - let nodes = node_map(&candidate.dag); - let kept_chains = kept_tumbling_window_chains(&candidate.dag); - let settings = self.models.cost_inputs; - let (cost, _) = plan_cost( - &candidate.dag, - &nodes, - &kept_chains, - &self.recurrence, - &self.range_evaluations, - self.data, - settings, - self.formula_scale, - ) - .map_err(|(node, error)| ErrorSelectingPlan::Cost { - candidate: candidate.id.clone(), - node, - error, - })?; - let violation = - latency_violation(&cost, self.entries).or_else(|| match &settings.cost_model { - CostModel::ClusterCapacity(limits) => capacity_violation(&cost.usage, limits) - .map(|reason| (RejectionCause::OverClusterCapacity, reason)), - _ => None, - }); - if let Some((cause, reason)) = violation { - self.reject(&candidate.id, cause, reason); - return Ok(()); - } - if !self.keep_every_candidate { - // Ties keep the earlier candidate. - let selectable = self - .kept - .first() - .is_none_or(|best| self.ranking.better(&cost, &self.costs[&best.id])); - if !selectable { - return Ok(()); - } - } - if !self.keep_every_candidate { - for kept in self.kept.drain(..) { - self.costs.remove(&kept.id); - self.accepted.remove(&kept.id); - } - } - self.costs.insert(candidate.id.clone(), cost); - self.accepted - .insert(candidate.id.clone(), accepted_accuracy); - self.kept.push(candidate.clone()); - Ok(()) - } - - /// The selection over every candidate offered, and the selected - /// candidate as Stage 2 built it (the selection's plan is its roots - /// with the weight requirements Stage 3 confirmed made proofs). - pub fn finish( - mut self, - ) -> Result<(WholePlanSelection, PhysicalWorkloadPlan), ErrorSelectingPlan> { - let ranking = self.ranking; - let valid_costs: Vec<&PlanCost> = - self.kept.iter().map(|kept| &self.costs[&kept.id]).collect(); - let Some(best) = choose(ranking, &valid_costs) else { - return Err(ErrorSelectingPlan::NoValidCandidate(self.rejected)); - }; - let selected = self.kept[best].id.clone(); - for kept in &self.kept { - let id = &kept.id; - if *id != selected { - let (cause, reason) = lost_reason(ranking, id, &selected, &self.costs); - self.rejected.push(Rejection { - id: id.clone(), - cause, - reason, - }); - } - } - let built = self.kept.swap_remove(best); - let plan = with_confirmed_weights(&built.roots, self.data); - let tumbling_window_retention_ms = tumbling_window_retention_ms( - &built.dag, - &query_reach(&built.dag, &node_map(&built.dag)), - &self.range_evaluations, - ); - let selection = WholePlanSelection { - selected, - plan, - costs: self.costs, - accepted: self.accepted, - rejected: self.rejected, - tumbling_window_retention_ms, - candidates_checked: self.candidates_checked, - }; - Ok((selection, built)) - } - - /// Record that candidate `id` failed a check: every rejection when - /// keeping every candidate, otherwise the first of each cause. - fn reject(&mut self, id: &str, cause: RejectionCause, reason: String) { - if self.keep_every_candidate || !self.rejected.iter().any(|r| r.cause == cause) { - self.rejected.push(Rejection { - id: id.to_string(), - cause, - reason, - }); - } - } -} - -/// The index of the candidate `ranking` selects among `costs` (the first on -/// ties); `None` without candidates. -pub(crate) fn choose(ranking: Ranking, costs: &[&PlanCost]) -> Option { - (0..costs.len()).reduce( - |best, next| match ranking.better(costs[next], costs[best]) { - true => next, - false => best, - }, - ) -} - -/// Under cost model 3, the first limit of the cluster `usage` is over, as a -/// rejection reason. -fn capacity_violation(usage: &ResourceUsage, limits: &ClusterLimits) -> Option { - if usage.peak_memory_bytes > limits.memory_bytes { - return Some(format!( - "peak memory {} bytes is over the cluster's {} bytes", - usage.peak_memory_bytes, limits.memory_bytes - )); - } - if usage.disk_bytes > limits.disk_bytes { - return Some(format!( - "disk {} bytes is over the cluster's {} bytes", - usage.disk_bytes, limits.disk_bytes - )); - } - match limits.cpu_cores { - Some(cores) if usage.peak_cpu_cores > cores => Some(format!( - "{} CPU cores at peak is over the cluster's {cores}", - usage.peak_cpu_cores - )), - _ => None, - } -} - -/// Why valid candidate `id` was not `selected` under `ranking`, and the -/// reason with its numbers. -fn lost_reason( - ranking: Ranking, - id: &str, - selected: &str, - costs: &BTreeMap, -) -> (RejectionCause, String) { - let (cost, best) = (&costs[id], &costs[selected]); - if !ranking.better(best, cost) { - let tied = match ranking { - Ranking::Price => format!( - "{} {COST_UNIT}", - display_cost(cost.cost_per_second.unwrap_or_default()) - ), - _ => describe_usage(&cost.usage), - }; - return ( - RejectionCause::TiedLostOnEnumerationOrder, - format!( - "tied with the selected candidate {selected} at {tied}; lost on enumeration order" - ), - ); - } - match ranking { - Ranking::Price => ( - RejectionCause::Costlier, - format!( - "costlier: {} vs {} {COST_UNIT}", - display_cost(cost.cost_per_second.unwrap_or_default()), - display_cost(best.cost_per_second.unwrap_or_default()) - ), - ), - Ranking::Latency => ( - RejectionCause::Slower, - format!( - "slower: {} vs {}", - describe_usage(&cost.usage), - describe_usage(&best.usage) - ), - ), - Ranking::Memory => ( - RejectionCause::MoreMemory, - format!( - "more memory: {} vs {}", - describe_usage(&cost.usage), - describe_usage(&best.usage) - ), - ), - } -} - -/// E.g. `latency 126.0 ms, peak memory 9648000000 bytes, disk 480000000 bytes`. -fn describe_usage(usage: &ResourceUsage) -> String { - format!( - "latency {:.1} ms, peak memory {} bytes, disk {} bytes", - worst_latency_ms(usage), - usage.peak_memory_bytes, - usage.disk_bytes - ) -} - -#[cfg(test)] -mod tests { - use super::*; - use asap_cost::{QueryLatency, COST_SOURCE}; - - /// A candidate with the given latency, peak memory and disk under cost - /// model 3. - fn cluster_cost(latency_ms: f64, peak_memory_bytes: u64, disk_bytes: u64) -> PlanCost { - PlanCost { - usage: ResourceUsage { - peak_memory_bytes, - disk_bytes, - latency_per_query: vec![QueryLatency { - compaction_ms: 0.0, - query_ms: latency_ms, - }], - ..ResourceUsage::default() - }, - cost_per_second: None, - cost_unit: COST_UNIT, - source: COST_SOURCE, - per_node: BTreeMap::new(), - } - } - - /// Peak memory and disk are compared exactly: 100 bytes less at 124 GB - /// is less memory, not a tie. - #[test] - fn memory_differing_by_a_few_bytes_is_compared_exactly() { - let larger = cluster_cost(7.0, 124_000_000_100, 0); - let smaller = cluster_cost(7.0, 124_000_000_000, 0); - let costs = [&larger, &smaller]; - assert_eq!(choose(Ranking::Memory, &costs).unwrap(), 1); - assert_eq!(choose(Ranking::Latency, &costs).unwrap(), 1); - } -} diff --git a/crates/planner/Cargo.toml b/crates/planner/Cargo.toml index 80589f63a..63802bcf6 100644 --- a/crates/planner/Cargo.toml +++ b/crates/planner/Cargo.toml @@ -31,17 +31,9 @@ serde = { version = "1", features = ["derive"] } serde_json = "1" thiserror = "2" -[features] -# `test_support::query_time_reference`, for a runtime's correctness tests. -# It reads `asap-types`' test-only DAG helpers. -test-support = ["asap-types/test-support"] - [dev-dependencies] -# The reference pricer over fixed candidate plans. -asap-plan-selection = { path = "../plan-selection", features = ["reference-pricer"] } -# The tests compare installed plans with `test_support::query_time_reference`. -asap-planner = { path = ".", features = ["test-support"] } -asap-physical-asap-optimization = { path = "../physical-asap-optimization", features = ["test-support"] } +asap-plan-selection = { path = "../plan-selection" } +asap-physical-asap-optimization = { path = "../physical-asap-optimization" } # Pass 1's target list (`per_computation_candidates`) for the tests. asap-logical-asap-optimization = { path = "../logical-asap-optimization", features = ["test-support"] } # The deployment accuracy model tests implement `AccuracyModel`. diff --git a/crates/planner/src/installable.rs b/crates/planner/src/installable.rs index ea836d674..a2dcf5f48 100644 --- a/crates/planner/src/installable.rs +++ b/crates/planner/src/installable.rs @@ -10,7 +10,7 @@ //! Everything else a runtime needs is derived from the DAG //! (`asap_types::ir::materialized_view`): the views to maintain, with what //! each is refreshed from and when ([`InstallablePlan::materialized_views`]), -//! and how far back each raw-data source is kept for query-time scans +//! and how far back each raw-data source is retained for query-time scans //! ([`InstallablePlan::raw_data_retention`]). A runtime reads the file with //! [`InstallablePlan::from_json`]. Raw data is read only at the DAG's //! `Scan`s, by their `Source`, and a view by its `MaterializedView`: the @@ -23,8 +23,7 @@ //! costing and display) is not the runtime's concern: the installable DAG //! has every chain expanded into the tumbling windows the executor runs. -use std::collections::{BTreeMap, HashMap, HashSet}; -use std::rc::Rc; +use std::collections::{HashMap, HashSet}; use asap_cost::{NodePriceSource, OperatorKind}; use asap_logical_asap_optimization::DeploymentCapabilities; @@ -32,17 +31,15 @@ use asap_plan_selection::{SearchReport, Selection}; use asap_types::ir::materialized_view::{ IngestionRefreshGroup, InstallableDagError, MaterializedView, RawDataRetention, }; -use asap_types::ir::operator::tumbling_window_chain::expand_tumbling_window_chains_in_workload; + use asap_types::ir::physical_export::{ PhysicalASAPDAG, PhysicalASAPDAGEdge, PhysicalASAPDAGNode, PhysicalASAPDAGValidationError, - PhysicalASAPNodeId, ReferencePlanError, + PhysicalASAPNodeId, }; -use asap_types::ir::properties::{ExecutionDataStateError, ExecutionTiming}; +use asap_types::ir::properties::ExecutionTiming; use asap_types::ir::SchemaDerivationError; -use asap_types::ir::{ - apply_materialization_timings_to_roots, ASAPOp, MaterializationAssignment, OperatorNode, - QueryRoot, -}; + +use asap_types::stages::RealizedPlan; use asap_types::workload::QueryWorkloadEntry; use serde::{Deserialize, Serialize}; @@ -50,7 +47,7 @@ use crate::{e2e_plan, PlanError, UserInput}; /// The version of the plan-file format [`InstallablePlan`] serializes to: /// the DAG, its nodes' fields and the query texts. -pub const INSTALLABLE_PLAN_WIRE_VERSION: u32 = 11; +pub const INSTALLABLE_PLAN_WIRE_VERSION: u32 = 13; /// The selected plan, ready to install and execute: what /// [`plan_installable`] returns and a runtime writes as its plan file. @@ -104,7 +101,7 @@ pub struct PredictedUsage { pub average_memory_bytes: f64, /// The most memory live at once. pub peak_memory_bytes: u64, - /// Bytes kept on disk, raw data query-time scans read included. + /// Bytes retained on disk, raw data query-time scans read included. pub disk_bytes: u64, /// CPU cores to provision for the peak. pub peak_cpu_cores: u32, @@ -156,11 +153,6 @@ pub struct PredictedQueryUsage { #[derive(Debug, thiserror::Error)] #[non_exhaustive] pub enum InstallablePlanError { - /// Internal bug: the selected plan's timings do not hold, or it does - /// not export; or, for the query-time reference, bad input: the plan - /// cannot run at query time. - #[error("timing: {0}")] - Timing(#[from] ExecutionDataStateError), /// Internal bug: a tumbling-window chain of the selected plan does not expand. #[error("tumbling-window chain expansion: {0}")] TumblingWindowChainExpansion(#[from] SchemaDerivationError), @@ -216,15 +208,6 @@ impl From for InstallablePlanError { } } -impl From for InstallablePlanError { - fn from(error: ReferencePlanError) -> Self { - match error { - ReferencePlanError::InvalidDag(e) => Self::InvalidDag(e), - ReferencePlanError::Timing(e) => Self::Timing(e), - } - } -} - /// Plan `input` for a deployment runtime: the query-language frontends and /// Stages 1-3 as [`e2e_plan`] runs them, returning the single selected DAG /// ready to install, and beside it Stage 3's report of the selection @@ -250,27 +233,20 @@ pub async fn plan_installable( pass: input.pass, }; let output = e2e_plan(input).await?; - installable_plan_of( - &output.roots(), - output.selection.as_ref(), - &output.tumbling_window_retention_ms, - &entries, - ) - .map_err(PlanError::Installable) + let _ = (output, entries); + todo!("#1207 PR 6: install the plan the realizer built from the selected DAG") } -/// The installable plan of `roots`, the selected roots of a workload whose -/// entries are `entries` (in `QueryWorkload::entries()` order), with -/// Stage 3's report of its `selection` of them when there is one, and the -/// tumbling-window retention of the selected plan's kept chains -/// (`RealizedPlan::tumbling_window_retention_ms`, by node id in the DAG `roots` -/// export): what [`plan_installable`] returns after planning. A caller that -/// runs the planning stages itself (e.g. a developer tool) uses it to see -/// what a runtime would install. +/// The installable plan of `realized`, the plan the realizer +/// (`AssignmentRealizer::realize`) built from the selected DAG of a +/// workload whose entries are `entries` (in `QueryWorkload::entries()` +/// order), with Stage 3's report of its `selection` when there is one: what +/// [`plan_installable`] returns after planning. A caller that runs the +/// planning stages itself (e.g. a developer tool) uses it to see what a +/// runtime would install. pub fn installable_plan_of( - roots: &[QueryRoot], + realized: &RealizedPlan, selection: Option<&Selection>, - tumbling_window_retention_ms: &BTreeMap, entries: &[QueryWorkloadEntry], ) -> Result<(InstallablePlan, Option), InstallablePlanError> { // Each query's longest range-evaluation span, which the views it @@ -280,13 +256,7 @@ pub fn installable_plan_of( .map(|entry| entry.range_evaluation.map_or(0, |range| range.span_ms)) .collect(); let query_texts: Vec = entries.iter().map(|entry| entry.query.0.clone()).collect(); - // Every node keeps the timing Stage 3 gave it. - let timed = - apply_materialization_timings_to_roots(roots, &MaterializationAssignment::default())?; - let expanded = tumbling_window_retention_of_expanded_tumbling_windows( - &timed, - tumbling_window_retention_ms, - )?; + let expanded = tumbling_window_retention_of_expanded_tumbling_windows(realized)?; let report = selection.map(|selection| report_of(selection, &expanded.installed_node_ids)); let reader_span_ms_by_node = reader_span_ms_by_node(&expanded.dag, &range_spans_ms); let plan = InstallablePlan::of_dag( @@ -315,7 +285,7 @@ impl InstallablePlan { Ok(self.dag.ingestion_refresh_groups()?) } - /// How far back each raw-data source is kept for query-time scans + /// How far back each raw-data source is retained for query-time scans /// ([`PhysicalASAPDAG::raw_data_retention`]): the raw data a runtime /// keeps, the raw-data counterpart of each view's retention. pub fn raw_data_retention(&self) -> Result, InstallablePlanError> { @@ -640,98 +610,20 @@ fn executor_refreshes( struct ExpandedWithRetention { dag: PhysicalASAPDAG, retention_ms_by_node: HashMap, - /// Each node id of the selected candidate's physical DAG (`timed` + /// Each node id of the selected candidate's physical DAG (its roots /// exported, Stage 3's ids) with the ids in `dag` that run it /// ([`PredictedNodeUsage::installed_node_ids`]). installed_node_ids: HashMap>, } -/// Expand the tumbling-window chains of `timed` and export it; each tumbling window takes the -/// retention Stage 2 gives its chain (`RealizedPlan::tumbling_window_retention_ms`, by -/// node id in the selected candidate's physical DAG, which is `timed` -/// exported). +/// The realized plan's DAG with its tumbling-window chains expanded; each +/// tumbling window takes the retention Stage 2 gives its chain +/// (`RealizedPlan::tumbling_window_retention_ms`). fn tumbling_window_retention_of_expanded_tumbling_windows( - timed: &[QueryRoot], - tumbling_window_retention_ms: &BTreeMap, + realized: &RealizedPlan, ) -> Result { - use asap_types::ir::physical_export::compile_physical_asap_workload_with_node_ids; - let selected = compile_physical_asap_workload_with_node_ids(timed)?; - // Reject unsupported raw-source shapes before expanding possibly thousands - // of tumbling windows. The complete expanded-plan validation remains authoritative. - selected.dag.check_raw_data_windows()?; - let mut retention_by_chain: Vec<(Rc, u64)> = Vec::new(); - for (&node_id, &retention_ms) in tumbling_window_retention_ms { - let chain = selected - .node_ids - .operator_node(node_id) - .filter(|node| matches!(node.asap(), Some(ASAPOp::SummaryTumblingWindowChain { .. }))) - .ok_or(InstallablePlanError::TumblingWindowRetentionOfUnknownChain { node_id })?; - retention_by_chain.push((Rc::clone(chain), retention_ms)); - } - let expansion = expand_tumbling_window_chains_in_workload(timed) - .map_err(InstallablePlanError::TumblingWindowChainExpansion)?; - let compiled = compile_physical_asap_workload_with_node_ids(&expansion.roots)?; - // Index physical identity once. Looking up every expanded tumbling window through - // the identity map's linear search makes this phase quadratic in tumbling windows. - let mut installed_ids_by_pointer = HashMap::with_capacity(compiled.dag.nodes.len()); - for node in &compiled.dag.nodes { - if let Some(operator) = compiled.node_ids.operator_node(node.id) { - installed_ids_by_pointer.insert(Rc::as_ptr(operator), node.id); - } - } - let installed_id = - |node: &Rc| installed_ids_by_pointer.get(&Rc::as_ptr(node)).copied(); - let mut retention_ms_by_node = HashMap::new(); - for expanded in &expansion.chains { - let Some((_, retention_ms)) = retention_by_chain - .iter() - .find(|(chain, _)| Rc::ptr_eq(chain, &expanded.chain)) - else { - continue; - }; - for tumbling_window in &expanded.tumbling_windows { - if let Some(node_id) = installed_id(tumbling_window) { - retention_ms_by_node.insert(node_id, *retention_ms); - } - } - } - // A chain runs as its tumbling windows; every other node as the node replacing it - // (itself when nothing below it changed). - let mut installed_node_ids = HashMap::new(); - for node in &selected.dag.nodes { - let Some(input) = selected.node_ids.operator_node(node.id) else { - continue; - }; - let ids: Vec = match expansion - .chains - .iter() - .find(|expanded| Rc::ptr_eq(&expanded.chain, input)) - { - Some(expanded) => { - let mut ids = Vec::with_capacity(expanded.tumbling_windows.len()); - for tumbling_window in &expanded.tumbling_windows { - if let Some(id) = installed_id(tumbling_window) { - ids.push(id); - } - } - ids - } - None => { - let replacement = expansion - .expanded_nodes - .iter() - .find(|(original, _)| Rc::ptr_eq(original, input)) - .map_or(input, |(_, replacement)| replacement); - installed_id(replacement).into_iter().collect() - } - }; - installed_node_ids.insert(node.id, ids); - } - Ok(ExpandedWithRetention { - dag: compiled.dag, - retention_ms_by_node, - installed_node_ids, - }) + let _ = realized; + todo!("#1207 PR 6: expand the realized DAG's tumbling-window chains") } /// Stage 3's record of its selected plan. @@ -792,8 +684,6 @@ fn report_of( mod tests { use super::*; use asap_types::ir::schema::{SketchAlgorithm, StatisticKind, SupportedSummary}; - use asap_types::ir::{NonASAPOp, Operator}; - use asap_types::physical::Materialization; thread_local! { /// The plan nodes handed to the executor's compiler by the views' @@ -802,7 +692,7 @@ mod tests { pub(super) static COMPILED_NODES: std::cell::Cell = const { std::cell::Cell::new(0) }; } - /// A p99 panel over a 300 s window every second, with no raw data kept: + /// A p99 panel over a 300 s window every second, with no raw data retained: /// one DDSketch per series in each of 300 one-second tumbling windows built at /// ingestion, about 300 materialized views in a DAG of about 900 nodes. /// Each refresh (here one refresh group of all the tumbling windows) is compiled @@ -859,200 +749,6 @@ mod tests { ); } - /// `TimeShift(10 min) <- TimeRange(5 min) <- Scan`, all at query time: - /// the shift above the range moves its window, which a window of a - /// fixed offset below the range cannot say, so it is refused; without - /// the shift the window is listed. - #[test] - fn a_time_shift_above_the_range_is_refused() { - use asap_types::ir::operator::operator_properties::TimeShift; - use asap_types::ir::operator::Source; - use asap_types::ir::physical_export::compile_physical_asap_dag; - use asap_types::ir::schema::{DataType, Field, Schema}; - use asap_types::ir::{ - apply_materialization_timings, NodeToTimedNodeCache, OperatorNode, TimeRangeKind, - }; - use std::time::Duration; - let scan = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Scan { - source: Source::TimeSeries { - metric: "latency_ms".into(), - }, - predicates: vec![], - schema: Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("value", DataType::Float64, false), - ], - 0, - vec![], - ), - })) - .unwrap(); - let range = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeRange { - range: Duration::from_secs(300), - kind: TimeRangeKind::Range, - child: scan, - })) - .unwrap(); - let shifted = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeShift { - shift: TimeShift { - offset_ms: 600_000, - at: None, - }, - child: std::rc::Rc::clone(&range), - })) - .unwrap(); - let export = |root| { - let timed = apply_materialization_timings( - &root, - &MaterializationAssignment::default(), - &mut NodeToTimedNodeCache::new(), - ) - .unwrap(); - compile_physical_asap_dag(&timed).unwrap() - }; - let install = |dag| { - InstallablePlan::of_dag(dag, vec![String::new()], &HashMap::new(), &HashMap::new()) - }; - match install(export(shifted)) { - Err(InstallablePlanError::Uninstallable( - InstallableDagError::UnsupportedRawDataWindow { reason, .. }, - )) => { - assert_eq!(reason, "a subquery or a TimeShift above the range") - } - other => panic!("{other:?}"), - } - let plan = install(export(range)).unwrap(); - assert!(plan.materialized_views().unwrap().is_empty()); - assert_eq!( - plan.raw_data_retention().unwrap(), - [RawDataRetention::new( - Source::TimeSeries { - metric: "latency_ms".into() - }, - Some(300_000) - )] - ); - } - - // A summary stored at ingestion time over a window ending at the - // evaluation and one ending a minute before it cannot be built in one - // ingestion run, so the plan is refused, not built wrong. - #[test] - fn an_ingestion_build_of_windows_at_different_offsets_is_refused() { - use asap_types::ir::operator::operator_properties::TimeShift; - use asap_types::ir::operator::{Reduction, Source}; - use asap_types::ir::physical_export::compile_physical_asap_dag; - use asap_types::ir::scalar::ColumnRef; - use asap_types::ir::schema::{ - DataType, ExactKind, ExactParams, Field, FieldDataType, Schema, SummaryUpdate, - }; - use asap_types::ir::{ - apply_materialization_timings, NodeToTimedNodeCache, OperatorNode, TimeRangeKind, - }; - use asap_types::physical::Medium; - use std::time::Duration; - let scan = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Scan { - source: Source::TimeSeries { - metric: "latency_ms".into(), - }, - predicates: vec![], - schema: Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("value", DataType::Float64, false), - ], - 0, - vec![], - ), - })) - .unwrap(); - // A one-minute window ending `offset_ms` before the evaluation. - let window = |offset_ms: i64| { - let shifted = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeShift { - shift: TimeShift { - offset_ms, - at: None, - }, - child: Rc::clone(&scan), - })) - .unwrap(); - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeRange { - range: Duration::from_secs(60), - kind: TimeRangeKind::Range, - child: shifted, - })) - .unwrap() - }; - // One sum over both windows' rows. - let both = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Concat { - children: vec![window(0), window(60_000)], - })) - .unwrap(); - let merged = OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryAgg { - child: both, - family: FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), - input: SummaryUpdate::column(ColumnRef::Named("value".into())), - reduction: Reduction::by(vec![]), - grouping: Default::default(), - filter: None, - })) - .unwrap(); - let mut assignment = MaterializationAssignment::default(); - assignment.set(&merged, Materialization::MatAtIngestion(Medium::Memory)); - let timed = - apply_materialization_timings(&merged, &assignment, &mut NodeToTimedNodeCache::new()) - .unwrap(); - let dag = compile_physical_asap_dag(&timed).unwrap(); - match InstallablePlan::of_dag(dag, vec![String::new()], &HashMap::new(), &HashMap::new()) { - Err(InstallablePlanError::Uninstallable( - InstallableDagError::IngestionBuildReadsWindowsOfDifferentOffsets { - end_offsets_ms, - .. - }, - )) => assert_eq!(end_offsets_ms, [0, 60_000]), - other => panic!("{other:?}"), - } - // A merge stored at ingestion time over two tumbling windows stored at - // ingestion time, one a minute older: an ingestion run would read - // both tumbling windows for the window ending at its end, so it is refused too. - let tumbling_window = |offset_ms: i64| { - OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryAgg { - child: window(offset_ms), - family: FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), - input: SummaryUpdate::column(ColumnRef::Named("value".into())), - reduction: Reduction::by(vec![]), - grouping: Default::default(), - filter: None, - })) - .unwrap() - }; - let tumbling_windows = [tumbling_window(0), tumbling_window(60_000)]; - let merge = OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryMerge { - children: tumbling_windows.to_vec(), - newest_tumbling_windows: None, - group_by: None, - })) - .unwrap(); - let mut assignment = MaterializationAssignment::default(); - for node in tumbling_windows.iter().chain([&merge]) { - assignment.set(node, Materialization::MatAtIngestion(Medium::Memory)); - } - let timed = - apply_materialization_timings(&merge, &assignment, &mut NodeToTimedNodeCache::new()) - .unwrap(); - let dag = compile_physical_asap_dag(&timed).unwrap(); - match InstallablePlan::of_dag(dag, vec![String::new()], &HashMap::new(), &HashMap::new()) { - Err(InstallablePlanError::Uninstallable( - InstallableDagError::IngestionBuildReadsWindowsOfDifferentOffsets { - end_offsets_ms, - .. - }, - )) => assert_eq!(end_offsets_ms, [0, 60_000]), - other => panic!("{other:?}"), - } - } - /// Unrestricted summaries become the executor's families; a caller's /// own list and its retention flags are kept. #[test] diff --git a/crates/planner/src/lib.rs b/crates/planner/src/lib.rs index ffefd3f12..3aa06b879 100644 --- a/crates/planner/src/lib.rs +++ b/crates/planner/src/lib.rs @@ -32,8 +32,8 @@ pub use pass::{ PassNameConflict, PassRegistry, PlanOutput, PlanningModels, QueryPlan, }; pub use run::{ - run_planning, selected_chosen_candidates, PlanSearchSettings, PlanningReport, PlanningRun, - SelectionError, Stage1ToStage3Pass, + run_planning, PlanSearchSettings, PlanningReport, PlanningRun, SelectionError, + Stage1ToStage3Pass, }; mod installable; @@ -43,32 +43,6 @@ pub use installable::{ INSTALLABLE_PLAN_WIRE_VERSION, }; -/// Helpers for a runtime's correctness tests (feature `test-support`). -#[cfg(any(test, feature = "test-support"))] -pub mod test_support { - use std::collections::HashMap; - - use crate::{InstallablePlan, InstallablePlanError}; - - /// For correctness tests only, never for installing: `plan` with - /// nothing materialized, the same operators, summaries and parameters - /// with every node at query time, and the same query texts. A runtime's - /// test compares its installed plan's answers with the executor running - /// this one over raw data samples. Validates `plan` first. - pub fn query_time_reference( - plan: &InstallablePlan, - ) -> Result { - plan.validate()?; - let dag = plan.dag.with_every_node_at_query_time()?; - InstallablePlan::of_dag( - dag, - plan.query_texts.clone(), - &HashMap::new(), - &HashMap::new(), - ) - } -} - // What a deployment runtime needs to build a `UserInput` and read an // `InstallablePlan`, so it depends on this crate and `asap-executor` alone. pub use asap_cost::{ diff --git a/crates/planner/src/pass.rs b/crates/planner/src/pass.rs index 9d699cfc9..539ac72e8 100644 --- a/crates/planner/src/pass.rs +++ b/crates/planner/src/pass.rs @@ -16,11 +16,6 @@ use std::collections::BTreeMap; use std::rc::Rc; -use asap_types::ir::physical_export::compile_physical_asap_workload; -use asap_types::ir::properties::timing::{ - apply_materialization_timings_to_roots, MaterializationAssignment, -}; -use asap_types::ir::properties::ExecutionDataStateError; use asap_types::ir::OperatorNode; use asap_types::workload::parsed_workload::ParsedWorkload; use asap_types::workload::WorkloadError; @@ -217,9 +212,9 @@ pub struct PlanOutput { /// How plan selection chose the plan, when the pass selects among priced /// candidates. pub selection: Option, - /// For each tumbling-window chain of the selected plan kept beyond one + /// For each tumbling-window chain of the selected plan materialized beyond one /// evaluation, by node id in its physical DAG, how long its completed - /// tumbling windows are kept ([`RealizedPlan::tumbling_window_retention_ms`](asap_types::stages::RealizedPlan::tumbling_window_retention_ms)); + /// tumbling windows are retained ([`RealizedPlan::tumbling_window_retention_ms`](asap_types::stages::RealizedPlan::tumbling_window_retention_ms)); /// empty for a pass that does not set it. pub tumbling_window_retention_ms: BTreeMap, } @@ -298,32 +293,6 @@ impl PlanOutput { nodes } - /// The workload as one physical ASAP DAG, with shared sub-DAGs exported - /// once. A node keeps the timing and materialization its pass gave it: - /// for [`Stage1ToStage3Pass`] that is Stage 3's selected plan as - /// selected (e.g. ingestion-time tumbling windows), the document a deployment - /// runtime executes. Only a node the pass left untimed takes the default - /// [`MaterializationAssignment`] (not materialized, computed when its - /// consumer runs). Operator queries become its `roots` and standalone scalar - /// queries its `scalar_roots`, keyed by position in the batch. That - /// position is the entry index, because `check_contract` requires dense - /// entry indices. - /// - /// An empty workload is an error - /// ([`PhysicalASAPDAGValidationError::NoRoots`](asap_types::ir::physical_export::PhysicalASAPDAGValidationError::NoRoots)). - /// A workload of only literal scalars yields a DAG with no nodes: just - /// its `scalar_roots`. - pub fn physical_asap_dag( - &self, - ) -> Result { - // A node several roots share by `Rc` is timed and exported once. - let timed = apply_materialization_timings_to_roots( - &self.roots(), - &MaterializationAssignment::default(), - )?; - compile_physical_asap_workload(&timed) - } - pub fn len(&self) -> usize { self.plans.len() + self.scalar_roots.len() } @@ -629,28 +598,6 @@ mod tests { assert!(check_contract(&output, "stub", 0).is_ok()); } - /// An empty workload has no DAG: the export's `NoRoots`, wrapped in - /// `InvalidPhysicalDAG`. A workload of only literal scalars exports a DAG - /// with no nodes, just its `scalar_roots`. - #[test] - fn physical_asap_dag_of_an_empty_and_a_literal_only_workload() { - use asap_types::ir::physical_export::PhysicalASAPDAGValidationError; - use asap_types::ir::ScalarExpr; - assert_eq!( - PlanOutput::new(Vec::new()).physical_asap_dag().err(), - Some(ExecutionDataStateError::InvalidPhysicalDAG( - PhysicalASAPDAGValidationError::NoRoots - )) - ); - let mut literal = PlanOutput::new(Vec::new()); - literal.scalar_roots = vec![(0, ScalarExpr::literal_f64(1.0))]; - let dag = literal.physical_asap_dag().unwrap(); - assert!(dag.nodes.is_empty() && dag.roots.is_empty()); - let [(0, ScalarExpr::Literal(_))] = dag.scalar_roots.as_slice() else { - panic!("one literal scalar root: {:?}", dag.scalar_roots) - }; - } - /// Registering a name twice fails instead of overwriting, so a comparison /// run cannot silently measure one pass twice. #[test] diff --git a/crates/planner/src/run.rs b/crates/planner/src/run.rs index 5069ca1de..269ba9405 100644 --- a/crates/planner/src/run.rs +++ b/crates/planner/src/run.rs @@ -3,19 +3,15 @@ //! finally the selected plan built through Stage 2's realizer. No stage //! calls another, and none has a deadline: each runs to completion. -use asap_logical_asap_optimization::generate_dag_of_logical_memo_groups; -use asap_types::stages::PerComputationLogicalMemoCandidates; +use asap_logical_asap_optimization::memo_of_queries; + use std::collections::HashSet; use std::rc::Rc; use std::time::Instant; use asap_logical_asap_optimization::generate_logical_asap_memo_candidates; -use asap_logical_asap_optimization::ChosenCandidatePerMemoGroup; use asap_logical_asap_optimization::DeploymentCapabilitiesError; -use asap_logical_asap_optimization::{ - generate_per_computation_logical_memo_candidates, PerComputationCandidateError, - PerComputationLogicalMemoCandidatesError, -}; +use asap_logical_asap_optimization::{pass1_memo, Pass1Error, PerComputationCandidateError}; use asap_physical_asap_optimization::{ generate_physical_asap_memo_candidates, Stage2AssignmentRealizer, }; @@ -23,8 +19,7 @@ use asap_plan_selection::{select_plan, ErrorSelectingPlan, SearchReport, Selecti use asap_types::ir::schema::schema_support::with_promql_series_identity; use asap_types::ir::{OperatorNode, QueryRoot}; use asap_types::stages::{ - AssignmentRealizer, CandidateForMemoGroupId, DagOfSelectedCandidates, - LogicalASAPMemoCandidates, Memo, MemoGroupId, PhysicalASAPMemoCandidates, + AssignmentRealizer, LogicalASAPMemoCandidates, Memo, PhysicalASAPMemoCandidates, PhysicalCandidateError, RealizedPlan, }; use asap_types::workload::bounds::with_confirmed_weights_at; @@ -38,7 +33,7 @@ use crate::pass::{ #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] pub struct PlanSearchSettings { /// Keep Pass 1's output for inspection (`asap_planner`, the DAG - /// viewer): [`PlanningRun::per_computation_candidates`]. Default off: + /// viewer): [`PlanningRun::pass1_memo`]. Default off: /// production planning copies no memo. pub inspect: bool, } @@ -76,13 +71,8 @@ pub struct PlanningReport { /// Pass 2's output: memo groups and their candidates. pub memo_groups: usize, pub logical_candidates: usize, - /// Stage 2's output: operator nodes, physical candidates and pruned - /// candidates. - pub operator_nodes: usize, + /// Stage 2's output: physical candidates and pruned candidates. pub physical_candidates: usize, - /// Stage 2's producer links: over every physical candidate and input, - /// the candidates that may produce it. - pub physical_producer_links: usize, pub stage2_pruned_candidates: usize, pub pass2_ms: f64, pub stage2_ms: f64, @@ -97,7 +87,7 @@ pub struct PlanningRun { /// Stage 1, Pass 1's output: the memo groups with their per-computation /// candidates and the pruned candidates. /// `None` when Pass 1 failed or without [`PlanSearchSettings::inspect`]. - pub per_computation_candidates: Option, + pub pass1_memo: Option, /// Pass 2's output, when it ran. pub logical_asap_candidates: Option, /// Stage 2's output, when it ran: the catalog `selection`'s candidate @@ -128,7 +118,7 @@ pub fn run_planning( root.validate_structure()?; } // Stage 0: the memo of logical groups. - let initial = generate_dag_of_logical_memo_groups(&roots); + let initial = memo_of_queries(&roots)?; // Pass 1: the per-computation candidates the deployment can run and // some query can use. let accuracy_targets: Vec<_> = entries @@ -136,7 +126,7 @@ pub fn run_planning( .map(|entry| entry.requirements.accuracy.clone()) .collect(); let accuracy = models.accuracy_check_inputs(); - let pass1 = generate_per_computation_logical_memo_candidates( + let pass1 = pass1_memo( initial, &accuracy_targets, accuracy.model, @@ -149,14 +139,14 @@ pub fn run_planning( ); // A query Pass 1 cannot plan is an input error. let pass1 = match pass1 { - Err(PerComputationLogicalMemoCandidatesError::PerComputation(error)) => return Err(error), + Err(Pass1Error::PerComputation(error)) => return Err(error), pass1 => pass1, }; let mut report = PlanningReport { pass1_ms: elapsed_ms(started), ..PlanningReport::default() }; - let per_computation_candidates = pass1.as_ref().ok().filter(|_| settings.inspect).cloned(); + let pass1_memo = pass1.as_ref().ok().filter(|_| settings.inspect).cloned(); let realizer = Stage2AssignmentRealizer { original_roots: &roots, entries, @@ -182,8 +172,13 @@ pub fn run_planning( let logical = logical_asap_candidates.insert( logical.map_err(|error| SelectionError::LogicalASAPCandidates(error.to_string()))?, ); - report.memo_groups = logical.memo.groups.len(); - report.logical_candidates = logical.memo.candidates.iter().map(Vec::len).sum(); + report.memo_groups = logical.memo.groups().len(); + report.logical_candidates = logical + .memo + .groups() + .iter() + .map(|group| group.candidates.len()) + .sum(); // Stage 2. let started = Instant::now(); let physical = generate_physical_asap_memo_candidates( @@ -197,14 +192,7 @@ pub fn run_planning( let physical = physical_asap_candidates.insert( physical.map_err(|error| SelectionError::PhysicalASAPCandidates(error.to_string()))?, ); - report.operator_nodes = physical.operator_nodes_of_candidates_for_memo_groups.len(); - report.physical_candidates = physical.candidates.len(); - report.physical_producer_links = physical - .candidates - .iter() - .flat_map(|candidate| &candidate.producer_candidates_per_input) - .map(|producers| producers.len()) - .sum(); + report.physical_candidates = physical.physical_candidates.len(); report.stage2_pruned_candidates = physical.pruned_candidates.len(); // Stage 3. let started = Instant::now(); @@ -238,7 +226,7 @@ pub fn run_planning( Ok(selection) })(); Ok(PlanningRun { - per_computation_candidates, + pass1_memo, logical_asap_candidates, physical_asap_candidates, selection, @@ -252,82 +240,6 @@ fn elapsed_ms(started: Instant) -> f64 { started.elapsed().as_secs_f64() * 1000.0 } -/// The selected plan's memo candidate for each group it uses: the memo -/// candidate whose root each selected node's candidate is. Factoring may give several -/// memo candidates of a group one root (identical terminal readers of -/// different inputs); the one reading the selected producers' groups is taken. -/// `memo` and `physical` are the run's -/// ([`PlanningRun::logical_asap_candidates`]'s memo and -/// [`PlanningRun::physical_asap_candidates`]); `selected` is the selection's -/// `selected_dag`. An error says why the plan is not one candidate per group. -pub fn selected_chosen_candidates( - memo: &Memo, - physical: &PhysicalASAPMemoCandidates, - selected: &DagOfSelectedCandidates, -) -> Result { - use std::collections::HashMap; - let mut producer_groups = HashMap::<_, Vec>::new(); - for edge in &selected.edges { - producer_groups - .entry(edge.consumer) - .or_default() - .push(physical.memo_group_of(selected.candidate_of_node[edge.producer.0])); - } - let mut chosen_candidates = ChosenCandidatePerMemoGroup::new(); - for (node_id, &id) in selected.candidate_of_node.iter().enumerate() { - let node_id = asap_types::stages::SelectedNodeId(node_id); - let node = &physical.operator_nodes_of_candidates_for_memo_groups - [physical.candidates[id.0].operator_node.0]; - if !node.is_implementation_root { - continue; // a candidate's private node - } - let options = &node.candidates_for_memo_group; - let reads = |&&CandidateForMemoGroupId { group, index }: &&CandidateForMemoGroupId| { - producer_groups.get(&node_id).is_none_or(|read| { - read.iter().all(|&child| { - child == group - || memo.candidates[group.0 as usize][index] - .input_groups - .contains(&child) - }) - }) - }; - let &CandidateForMemoGroupId { group, index } = - options.iter().find(reads).ok_or_else(|| { - format!( - "selected candidate {} reads groups none of its memo candidates read", - id.0 - ) - })?; - if let Some(previous) = chosen_candidates.insert(group, index) { - if previous != index { - return Err(format!( - "the selected plan implements group {} with two candidates, {previous} and {index}", - group.0 - )); - } - } - } - // Every group the chosen candidates need, from the query roots, is chosen. - let mut pending: Vec<_> = asap_types::stages::root_groups(&memo.query_roots) - .into_iter() - .collect(); - let mut seen = HashSet::new(); - while let Some(group) = pending.pop() { - if !seen.insert(group) { - continue; - } - let index = chosen_candidates.get(&group).ok_or_else(|| { - format!( - "the selected plan uses group {} but no candidate of it is selected", - group.0 - ) - })?; - pending.extend(&memo.candidates[group.0 as usize][*index].input_groups); - } - Ok(chosen_candidates) -} - /// The planner pipeline behind the optimization-pass interface: the /// workload's roots in entry order, PromQL roots with each series' full /// identity as a column (the row representation per-series state needs at @@ -402,10 +314,9 @@ impl OptimizationPass for Stage1ToStage3Pass { } } } - output.tumbling_window_retention_ms = run - .selected_physical_plan - .map(|realized| realized.tumbling_window_retention_ms) - .unwrap_or_default(); + if let Some(realized) = run.selected_physical_plan { + output.tumbling_window_retention_ms = realized.tumbling_window_retention_ms; + } output.selection = Some(selection); Ok(output) } diff --git a/crates/planner/tests/deployment_accuracy_model.rs b/crates/planner/tests/deployment_accuracy_model.rs deleted file mode 100644 index 91cda2b04..000000000 --- a/crates/planner/tests/deployment_accuracy_model.rs +++ /dev/null @@ -1,374 +0,0 @@ -//! The deployment's accuracy model plugs into plan selection (#509 Stage 3) -//! only. Stage 1 composes guarantees with the built-in analytical rules, so -//! a different model leaves the candidates unchanged and changes only which -//! of them Stage 3 accepts. - -use std::collections::BTreeSet; -use std::rc::Rc; - -use asap_accuracy::{AccuracyModel, BuiltinAccuracyModel}; -use asap_frontend_promql::{lower_promql_query_workload, promql_metric_names}; -use asap_plan_selection::reference_pricer::{select_among_plans, WholePlanSelection}; -use asap_plan_selection::{ErrorSelectingPlan, RejectionCause}; -use asap_planner::PlanningModels; - -/// The reference pricer's selection among every given candidate, and the -/// selected plan's operator roots. -struct PlanOutput { - plans: Vec, - selection: Option, -} - -struct QueryPlan { - root: Rc, -} -use asap_types::ir::properties::{AccuracyGuarantee, AccuracyMetric, AccuracyTarget}; -use asap_types::ir::schema::FieldDataType; -use asap_types::ir::{ASAPOp, OperatorNode, QueryRoot}; -use asap_types::workload::{ - DataArrival, DataWorkload, DurationMs, PlanningWorkload, Predictability, Query, QueryLanguage, - QueryRequirements, QueryWorkload, RepeatedDemand, RepeatingEntry, RepetitionInterval, -}; - -/// A deployment that trusts no approximate estimate: only an exact -/// guarantee meets a target. -struct ExactOnlyAccuracyModel; - -impl AccuracyModel for ExactOnlyAccuracyModel { - fn satisfies(&self, guarantee: &AccuracyGuarantee, _target: &AccuracyTarget) -> bool { - guarantee.is_exact() - } -} - -/// `query` with a 1% rank error, repeated every ten minutes over metrics -/// sampled every 15 s. -fn rank_target_workload(query: &str) -> PlanningWorkload { - PlanningWorkload { - query_workload: QueryWorkload { - language: QueryLanguage::PromQL, - query_batch: None, - repeating_queries: Some(vec![RepeatingEntry { - query: Query(query.into()), - demand: RepeatedDemand::FixedInterval(RepetitionInterval(600_000)), - requirements: QueryRequirements { - accuracy: AccuracyTarget::max_error(0.01).accuracy_metric(AccuracyMetric::Rank), - max_latency_ms: None, - }, - predictability: Predictability::Unknown, - range_evaluation: None, - }]), - }, - data_workload: Some(DataWorkload { - arrival: DataArrival::Ingesting { - rows_per_second: 1.0, - }, - ..DataWorkload::sampled_every(promql_metric_names(query).unwrap(), DurationMs(15_000)) - }), - } -} - -/// Whether `root` reads a sketch state. -fn reads_sketch(root: &Rc) -> bool { - OperatorNode::reachable(root).into_iter().any(|node| { - matches!( - node.asap(), - Some(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(..), - .. - }) - ) - }) -} - -/// A small, fixed physical inventory, independent of deployment accuracy models. -/// Exercise each local implementation with the remaining computations exact. -/// These fixtures test evaluator hooks, not a timed search's archive prefix. -fn physical_candidates( - workload: &PlanningWorkload, -) -> Vec { - use asap_logical_asap_optimization::test_support::{ - compose_logical_candidate, generate_per_computation_candidates, - }; - use asap_physical_asap_optimization::test_support::optimize_physical_asap_chosen; - use asap_types::physical::Materialization; - - let roots = lower_promql_query_workload(workload).expect("lowers"); - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&roots), - ) - .unwrap(); - let mut choices = BTreeSet::from([vec![0; inventory.targets.len()]]); - for (index, target) in inventory.targets.iter().enumerate() { - for implementation in 1..target.choice_count() { - let mut choice = vec![0; inventory.targets.len()]; - choice[index] = implementation; - choices.insert(choice); - } - } - choices - .into_iter() - .enumerate() - .map(|(index, choice)| { - let roots: Vec<_> = compose_logical_candidate(&inventory, &choice).unwrap(); - let mut candidate = optimize_physical_asap_chosen( - &format!("fixture-{index}"), - &roots, - &workload.query_workload.entries().collect::>(), - workload.data_workload.as_ref().unwrap(), - false, - &mut |_, _, _| Some(Materialization::NotMaterialized), - ) - .unwrap() - .expect("query-time fixture is executable"); - candidate.id = format!("fixture-{index}"); - candidate - }) - .collect() -} - -fn plan_every_candidate( - workload: &PlanningWorkload, - candidates: &[asap_types::physical::PhysicalWorkloadPlan], - models: PlanningModels<'_>, -) -> Result { - let selection = select_among_plans( - candidates, - &workload.query_workload.entries().collect::>(), - workload.data_workload.as_ref().unwrap(), - &models.accuracy_check_inputs(), - models.cost_inputs, - )?; - let plans = selection - .plan - .iter() - .map(|root| match root { - QueryRoot::Operator(node) => QueryPlan { - root: Rc::clone(node), - }, - QueryRoot::Scalar(_) => panic!("operator queries only"), - }) - .collect(); - Ok(PlanOutput { - plans, - selection: Some(selection), - }) -} - -/// Every candidate Stage 3 was given: the valid ones it priced and the -/// ones it rejected. -fn stage3_candidate_ids(output: &PlanOutput) -> BTreeSet { - let selection = output.selection.as_ref().expect("plan selection ran"); - selection - .costs - .keys() - .cloned() - .chain(selection.rejected.iter().map(|r| r.id.clone())) - .collect() -} - -/// The candidates for which Stage 3 accepted an approximate output for some -/// query. -fn approximate_accepted(output: &PlanOutput) -> BTreeSet { - let selection = output.selection.as_ref().expect("plan selection ran"); - selection - .accepted - .iter() - .filter(|(_, queries)| queries.iter().any(|q| q.accepted_metric.is_some())) - .map(|(id, _)| id.clone()) - .collect() -} - -#[tokio::test] -async fn the_deployment_accuracy_model_decides_only_in_plan_selection() { - // The user's workload: one p99 query with a 1% rank-error target. - let workload = rank_target_workload("quantile_over_time(0.99, lat[5m])"); - - let candidates = physical_candidates(&workload); - - // The built-in models: Stage 3 checks guarantees with BuiltinAccuracyModel. - let builtin = plan_every_candidate(&workload, &candidates, PlanningModels::builtin()) - .expect("plans with the built-in models"); - // The candidates whose approximate output the built-in model accepts: - // the sketch candidates, sized in Stage 1 for the 1% target. - let builtin_approximate = approximate_accepted(&builtin); - assert!(!builtin_approximate.is_empty()); - - // The same workload with a deployment model that accepts only exact - // outputs, plugged into Stage 3 through `PlanningModels::with_accuracy`. - let exact_only_model = ExactOnlyAccuracyModel; - let exact_only = plan_every_candidate( - &workload, - &candidates, - PlanningModels::builtin().with_accuracy(&exact_only_model), - ) - .expect("plans with the exact-only model"); - - // Stages 1 and 2 do not read the model: Stage 3 got the same - // candidates, with the same guarantees and summary parameters. - assert_eq!( - stage3_candidate_ids(&builtin), - stage3_candidate_ids(&exact_only) - ); - - // Stage 3 now rejects each of those candidates for missing its target, - // accepts no approximate output and selects an exact plan. - let exact_only_selection = exact_only.selection.as_ref().unwrap(); - for id in &builtin_approximate { - assert!(exact_only_selection - .rejected - .iter() - .any(|r| &r.id == id && r.cause == RejectionCause::AccuracyTargetMissed)); - } - assert!(approximate_accepted(&exact_only).is_empty()); - assert!(!reads_sketch(&exact_only.plans[0].root)); -} - -/// A deployment that has measured every output itself and accepts each one -/// for any target, whatever its guarantee says. -struct AcceptsEveryOutput; - -impl AccuracyModel for AcceptsEveryOutput { - fn satisfies(&self, guarantee: &AccuracyGuarantee, target: &AccuracyTarget) -> bool { - BuiltinAccuracyModel.satisfies(guarantee, target) - } - - fn accepts_output_regardless_of_guarantee( - &self, - _output: &QueryRoot, - _target: &AccuracyTarget, - ) -> bool { - true - } -} - -/// The candidates for which Stage 3 accepted some query's output because -/// the deployment's model accepted it regardless of its guarantee. -fn accepted_regardless_of_guarantee(output: &PlanOutput) -> BTreeSet { - let selection = output.selection.as_ref().expect("plan selection ran"); - selection - .accepted - .iter() - .filter(|(_, queries)| queries.iter().any(|q| q.accepted_regardless_of_guarantee)) - .map(|(id, _)| id.clone()) - .collect() -} - -/// The ids of the candidates `output`'s selection rejected with `cause`. -fn rejected_with(output: &PlanOutput, cause: RejectionCause) -> BTreeSet { - output - .selection - .as_ref() - .expect("plan selection ran") - .rejected - .iter() - .filter(|r| r.cause == cause) - .map(|r| r.id.clone()) - .collect() -} - -/// A deployment model that accepts an output directly also accepts one with -/// no guarantee: the built-in model rejects the summary candidates of a -/// difference of two quantile estimates as having none, and the accepting -/// model makes them valid, and Stage 3 selects one. -#[tokio::test] -async fn the_deployment_accuracy_model_accepts_outputs_without_a_guarantee() { - // The user's workload: a difference of two quantile estimates, 1% rank - // error. Stage 1 has no analytical rule for a difference of rank - // estimates, so it keeps the summary candidates with no guarantee. - let workload = rank_target_workload("quantile(0.99, rate(x[5m])) - quantile(0.5, rate(x[5m]))"); - - let candidates = physical_candidates(&workload); - - // The built-in model (default hook: accepts nothing directly) rejects - // every summary candidate as having no guarantee and selects exact. - let builtin = plan_every_candidate(&workload, &candidates, PlanningModels::builtin()) - .expect("plans with the built-in models"); - // Candidates rejected for having no guarantee. - let unguaranteed = rejected_with(&builtin, RejectionCause::NoAccuracyGuarantee); - assert!(!unguaranteed.is_empty()); - assert!(accepted_regardless_of_guarantee(&builtin).is_empty()); - assert!(!reads_sketch(&builtin.plans[0].root)); - - // The deployment's model accepts every output directly. - let accepting_model = AcceptsEveryOutput; - let accepting = plan_every_candidate( - &workload, - &candidates, - PlanningModels::builtin().with_accuracy(&accepting_model), - ) - .expect("plans with the accepting model"); - let accepting_selection = accepting.selection.as_ref().unwrap(); - assert_eq!( - stage3_candidate_ids(&builtin), - stage3_candidate_ids(&accepting) - ); - - // Every candidate the built-in model rejected for having no guarantee - // is now valid and priced, accepted regardless of its guarantee, and no - // candidate fails the accuracy check. - assert!(accepted_regardless_of_guarantee(&accepting).is_superset(&unguaranteed)); - for id in &unguaranteed { - assert!(accepting_selection.costs.contains_key(id), "{id}"); - } - assert!(accepting_selection.rejected.iter().all(|r| !matches!( - r.cause, - RejectionCause::NoAccuracyGuarantee | RejectionCause::AccuracyTargetMissed - ))); - // A singleton previously rejected sketch is now selectable; the cheapest - // plan in the complete fixture need not be that sketch. - let sketch = candidates - .iter() - .find(|c| unguaranteed.contains(&c.id)) - .unwrap(); - let singleton = plan_every_candidate( - &workload, - std::slice::from_ref(sketch), - PlanningModels::builtin().with_accuracy(&accepting_model), - ) - .unwrap(); - assert_eq!(singleton.selection.as_ref().unwrap().selected, sketch.id); - assert!(reads_sketch(&singleton.plans[0].root)); -} - -/// The hook has priority over a guarantee that misses: a candidate whose -/// guarantee misses its target under the built-in models (a relative bound -/// for a rank target) is accepted under the accepting model. -#[tokio::test] -async fn the_deployment_accuracy_model_accepts_outputs_whose_guarantee_misses() { - // The user's workload: one p99 query with a 1% rank-error target. - let workload = rank_target_workload("quantile_over_time(0.99, lat[5m])"); - let candidates = physical_candidates(&workload); - // Under the built-in models: the candidates whose guarantee misses the - // target (a relative bound for a rank target). - let builtin = plan_every_candidate(&workload, &candidates, PlanningModels::builtin()) - .expect("plans with the built-in models"); - let missed = rejected_with(&builtin, RejectionCause::AccuracyTargetMissed); - assert!(!missed.is_empty()); - // The same workload with the model that accepts every output directly. - let accepting_model = AcceptsEveryOutput; - let accepting = plan_every_candidate( - &workload, - &candidates, - PlanningModels::builtin().with_accuracy(&accepting_model), - ) - .expect("plans with the accepting model"); - // Each of them is now valid, priced and accepted regardless of guarantee. - let accepting_selection = accepting.selection.as_ref().unwrap(); - for id in &missed { - assert!(accepting_selection.costs.contains_key(id), "{id}"); - assert!( - accepting_selection.accepted[id][0].accepted_regardless_of_guarantee, - "{id}" - ); - } - // The hook also overrides guarantees that meet the target: every - // accepted output, including exact and KLL candidates, is accepted - // regardless of its guarantee. - assert_eq!(accepting_selection.accepted.len(), candidates.len()); - for (id, queries) in &accepting_selection.accepted { - assert!( - queries.iter().all(|q| q.accepted_regardless_of_guarantee), - "{id}" - ); - } -} diff --git a/crates/planner/tests/main.rs b/crates/planner/tests/main.rs index bb91f08fe..9d06c8c46 100644 --- a/crates/planner/tests/main.rs +++ b/crates/planner/tests/main.rs @@ -5,10 +5,7 @@ mod time_series; mod columnar_batches; -mod deployment_accuracy_model; mod e2e_plan; -mod per_sample_updater; mod runtime_install; -mod sql_per_group_topk; mod sql_table_scans; mod summary_sharing; diff --git a/crates/planner/tests/per_sample_updater.rs b/crates/planner/tests/per_sample_updater.rs deleted file mode 100644 index b8f83f1e5..000000000 --- a/crates/planner/tests/per_sample_updater.rs +++ /dev/null @@ -1,405 +0,0 @@ -//! A runtime that ingests sample by sample refreshes a materialized tumbling window's states -//! with `StateColumnUpdater`, the code the plan's `SummaryAgg` runs on a -//! batch (issue #762): the same states as the executor's tumbling-window build. - -use std::collections::BTreeMap; -use std::sync::Arc; - -use asap_executor::physical_planner::compile; -use asap_executor::planner::ir::physical_export::PhysicalASAPOperatorPayload as Payload; -use asap_executor::planner::ir::ASAPOp; -use asap_executor::runtime::{Limits, RunContext, Scope}; -use asap_executor::summary_kernels::exact::ExactAccumulator; -use asap_executor::StateColumnUpdater; -use asap_planner::{ - plan_installable, test_support::query_time_reference, DeploymentCapabilities, FrontendInput, - PlanningModels, PlanningWorkload, UserInput, -}; -use futures::StreamExt; - -use crate::time_series; -use time_series::{series_identity, series_labels, time_series_tables, RawDataSample}; - -/// The evaluation time: the end of an hour. -const EVALUATION_MS: i64 = 3_600_000; - -#[tokio::test] -async fn per_sample_updates_build_the_materialized_tumbling_window_the_plan_refreshes() { - // What the user writes: the 5-minute rate of a counter, every minute, - // exactly, over 1,000 series sampled every 15 s. - let workload: PlanningWorkload = serde_json::from_str( - r#"{"query_workload": {"language": "promql", "repeating_queries": [{ - "query": "rate(http_requests_total[5m])", - "demand": {"fixed_interval": 60000}, - "requirements": {}, - "predictability": {"predictable": {"known_at": null}}}]}, - "data_workload": {"arrival": {"ingesting": {"rows_per_second": 66.7}}, - "sources": {"http_requests_total": {"series_count": 1000, "sample_interval": 15000}}}}"#, - ) - .unwrap(); - // No raw data samples retained: the plan stores rate tumbling windows built at ingestion. - let capabilities = DeploymentCapabilities { - raw_data_retained: false, - ..DeploymentCapabilities::default() - }; - // The installable plan: the selected DAG and its materialized views. - let plan = plan_installable(UserInput::new( - &workload, - FrontendInput::Promql, - PlanningModels::builtin().with_deployment_capabilities(&capabilities), - )) - .await - .unwrap() - .0; - let dag = &plan.dag; - // The newest tumbling window view and its window at this evaluation. - let views = plan.materialized_views().unwrap(); - let output = views - .iter() - .max_by_key(|output| { - output.inputs.raw_data_windows()[0] - .bounds_ms(EVALUATION_MS) - .1 - }) - .unwrap(); - let (start_ms, end_ms) = output.inputs.raw_data_windows()[0].bounds_ms(EVALUATION_MS); - // Five one-minute tumbling windows, the newest (59 min, 60 min]. - assert_eq!(views.len(), 5); - assert_eq!((start_ms, end_ms), (3_540_000, 3_600_000)); - // The view node's summary family, as the plan declares it: the - // `SummaryAgg` at or below the view node. - let family = { - let mut id = output.node_id; - loop { - let node = dag.nodes.iter().find(|n| n.id == id).unwrap(); - if let Payload::ASAP(ASAPOp::SummaryAgg { family, .. }) = &node.payload { - break family.clone(); - } - id = dag - .edges - .iter() - .find(|e| e.consumer == id) - .unwrap() - .producer; - } - }; - // What the runtime received in the tumbling window: 3 series, one counter sample - // every 15 s, one series with a reset. - assert_eq!( - family, - asap_executor::planner::ir::schema::FieldDataType::ExactAggregate( - asap_executor::planner::ir::schema::ExactKind::Rate, - asap_executor::planner::ir::schema::ExactParams::Rate, - ) - ); - let samples: Vec = ["a", "b", "c"] - .iter() - .flat_map(|instance| { - let mut value = 0.0; - (1..=(end_ms - start_ms) / 15_000).map(move |step| { - value = if *instance == "c" && step == 3 { - 2.0 - } else { - value + 10.0 - }; - RawDataSample { - labels: BTreeMap::from([ - ("__name__".to_string(), "http_requests_total".to_string()), - ("instance".to_string(), instance.to_string()), - ]), - timestamp_ms: start_ms + step * 15_000, - value, - } - }) - }) - .collect(); - - // The executor's tumbling-window build: the samples bound at the plan's `Scan`, - // its `TimeRange` keeping the time range the tumbling window covers. - let compiled = compile(dag, &[], &[output.node_id as u64]).unwrap(); - let build = compiled - .instantiate(Arc::new(time_series_tables(dag, &samples))) - .unwrap(); - let context = RunContext::new( - Scope::Ingestion { - window_start_ms: start_ms, - window_end_ms: end_ms, - }, - Limits::default(), - ) - .unwrap(); - let tumbling_window = build - .execute(compiled.roots(), context) - .unwrap() - .remove(0) - .next() - .await - .unwrap() - .unwrap(); - // Its states, by series labels. - let state_column = tumbling_window - .schema() - .fields - .iter() - .position(|field| !field.is_plain()) - .unwrap(); - let from_dag: BTreeMap, String> = tumbling_window - .to_rows() - .iter() - .map(|row| { - let asap_executor::values::Value::Summary { state, .. } = &row[state_column] else { - panic!("a state") - }; - let exact = state.as_any().downcast_ref::().unwrap(); - ( - series_labels(tumbling_window.schema(), row), - serde_json::to_string(exact).unwrap(), - ) - }) - .collect(); - - // The per-sample build: one updater for the tumbling window, a group per series. - let mut updater = StateColumnUpdater::new(&family).unwrap(); - let mut groups: BTreeMap, u32> = BTreeMap::new(); - for sample in &samples { - let group = match groups.get(&sample.labels) { - Some(&group) => group, - None => { - let group = updater.add_group().unwrap(); - groups.insert(sample.labels.clone(), group); - group - } - }; - updater - .update(group, sample.value, sample.timestamp_ms) - .unwrap(); - } - // Sealed at tumbling window close: one state per group. - let states = updater.seal(); - let per_sample: BTreeMap, String> = groups - .into_iter() - .map(|(mut labels, group)| { - let state = states.state(group as usize); - let exact = state.as_any().downcast_ref::().unwrap(); - // A rate's series drops its metric name, as the DAG's does. - if !from_dag.keys().any(|l| l.contains_key("__name__")) { - labels.remove("__name__"); - } - (labels, serde_json::to_string(exact).unwrap()) - }) - .collect(); - assert_eq!(from_dag.len(), 3); - assert_eq!(per_sample, from_dag); -} - -/// Run `ids` of `dag` with the view states `inputs` bound at their nodes -/// and the raw `samples` at its `Scan`s, in `scope`: each id's answer rows -/// as (series labels, value), sorted. -async fn answers( - dag: &asap_planner::PhysicalASAPDAG, - inputs: BTreeMap, - samples: &[RawDataSample], - ids: &[u64], - scope: Scope, -) -> Vec, f64)>> { - // The plan's views whose states are bound; each one's schema is its DAG - // node's. - let views: Vec<_> = dag - .materialized_views() - .unwrap() - .into_iter() - .filter(|view| inputs.contains_key(&(view.node_id as u64))) - .collect(); - assert_eq!(views.len(), inputs.len()); - // The executor's plan for `ids`, reading the bound views. - let compiled = compile(dag, &views, ids).unwrap(); - // Each input as a view's states in an in-memory store. - let mut store = time_series_tables(dag, samples); - for (id, batch) in inputs { - store = store.with_materialized_view(id as _, vec![batch]).unwrap(); - } - let instance = compiled.instantiate(Arc::new(store)).unwrap(); - let context = RunContext::new(scope, Limits::default()).unwrap(); - let mut results = Vec::new(); - for mut output in instance.execute(compiled.roots(), context).unwrap() { - let mut rows = Vec::new(); - while let Some(batch) = output.next().await { - let batch = batch.unwrap(); - for row in &batch.to_rows() { - let labels = series_labels(batch.schema(), row); - let value = row - .iter() - .find_map(|v| match v { - asap_executor::values::Value::Float64(v) => Some(*v), - _ => None, - }) - .unwrap(); - rows.push((labels, value)); - } - } - rows.sort_by(|a, b| a.0.cmp(&b.0)); - results.push(rows); - } - results -} - -/// A runtime builds every materialized tumbling window sample by sample, installs each as a -/// batch with `Batch::try_from_columns` (no DAG run), and answers the query -/// from them: the same answers as the plan's query-time reference over the -/// raw data samples. -#[tokio::test] -async fn tumbling_windows_installed_from_per_sample_states_answer_the_query() { - use asap_executor::arrow::array::{DictionaryArray, TimestampMillisecondArray, UInt32Array}; - use asap_executor::arrow::datatypes::UInt32Type; - use asap_executor::values::{Batch, BatchColumn}; - use std::sync::Arc; - // What the user writes: the 5-minute rate of a counter, every minute, - // exactly, over 1,000 series sampled every 15 s. - let workload: PlanningWorkload = serde_json::from_str( - r#"{"query_workload": {"language": "promql", "repeating_queries": [{ - "query": "rate(http_requests_total[5m])", - "demand": {"fixed_interval": 60000}, - "requirements": {}, - "predictability": {"predictable": {"known_at": null}}}]}, - "data_workload": {"arrival": {"ingesting": {"rows_per_second": 66.7}}, - "sources": {"http_requests_total": {"series_count": 1000, "sample_interval": 15000}}}}"#, - ) - .unwrap(); - // No raw data samples retained: five one-minute rate tumbling windows built at ingestion. - let capabilities = DeploymentCapabilities { - raw_data_retained: false, - ..DeploymentCapabilities::default() - }; - let plan = plan_installable(UserInput::new( - &workload, - FrontendInput::Promql, - PlanningModels::builtin().with_deployment_capabilities(&capabilities), - )) - .await - .unwrap() - .0; - let dag = &plan.dag; - // What the runtime received: 3 series, a counter sample every 15 s over - // the hour (one series resets), the last at the evaluation time. - let samples: Vec = ["a", "b", "c"] - .iter() - .flat_map(|instance| { - let mut value = 0.0; - (1..=EVALUATION_MS / 15_000).map(move |step| { - value = if *instance == "c" && step == 230 { - 3.0 - } else { - value + 7.0 - }; - RawDataSample { - labels: BTreeMap::from([ - ("__name__".to_string(), "http_requests_total".to_string()), - ("instance".to_string(), instance.to_string()), - ]), - timestamp_ms: step * 15_000, - value, - } - }) - }) - .collect(); - // Every materialized tumbling window, built sample by sample and installed as a batch. - let mut tumbling_windows = BTreeMap::new(); - for output in &plan.materialized_views().unwrap() { - // The time range the tumbling window covers, `(start_ms, end_ms]`, at this evaluation. - let (start_ms, end_ms) = output.inputs.raw_data_windows()[0].bounds_ms(EVALUATION_MS); - // The view node's family and output schema, from the plan. - let node = dag.nodes.iter().find(|n| n.id == output.node_id).unwrap(); - let mut id = output.node_id; - let family = loop { - let node = dag.nodes.iter().find(|n| n.id == id).unwrap(); - if let Payload::ASAP(ASAPOp::SummaryAgg { family, .. }) = &node.payload { - break family.clone(); - } - id = dag - .edges - .iter() - .find(|e| e.consumer == id) - .unwrap() - .producer; - }; - // The runtime's per-sample build: a group per series ... - let mut updater = StateColumnUpdater::new(&family).unwrap(); - let mut identities: Vec = Vec::new(); - let mut group_of_series: BTreeMap, u32> = BTreeMap::new(); - for sample in samples - .iter() - .filter(|s| start_ms < s.timestamp_ms && s.timestamp_ms <= end_ms) - { - let group = match group_of_series.get(&sample.labels) { - Some(&group) => group, - None => { - let group = updater.add_group().unwrap(); - group_of_series.insert(sample.labels.clone(), group); - identities.push(series_identity(&sample.labels)); - group - } - }; - updater - .update(group, sample.value, sample.timestamp_ms) - .unwrap(); - } - // ... sealed at tumbling window close: one rate record per series. - let states = Arc::new(updater.seal()); - // The tumbling window's columns, in the view node's schema: the tumbling window end as - // its time, each series' state, and its identity. - let schema = Arc::new(node.output_schema.clone()); - let groups = identities.len(); - let columns = schema - .fields - .iter() - .enumerate() - .map(|(index, field)| { - if Some(index) == schema.time_index { - BatchColumn::Plain(Arc::new(TimestampMillisecondArray::from(vec![ - end_ms; - groups - ]))) - } else if !field.is_plain() { - BatchColumn::States { - indices: UInt32Array::from_iter_values(0..groups as u32), - states: states.clone(), - } - } else { - BatchColumn::Plain(Arc::new( - identities - .iter() - .map(String::as_str) - .collect::>(), - )) - } - }) - .collect(); - // Installed without running the DAG. - tumbling_windows.insert( - output.node_id as u64, - Batch::try_from_columns(schema, columns).unwrap(), - ); - } - assert_eq!(tumbling_windows.len(), 5); - let query = Scope::Query { - evaluation_time_ms: EVALUATION_MS, - }; - let roots: Vec = plan.dag.roots.iter().map(|&r| r as u64).collect(); - // The query's answers from the installed tumbling windows. - let from_tumbling_windows = - answers(dag, tumbling_windows, &samples, &roots, query.clone()).await; - // The plan's query-time reference over the raw data samples. - let reference = query_time_reference(&plan).unwrap(); - let reference_roots: Vec = reference.dag.roots.iter().map(|&r| r as u64).collect(); - let expected = answers( - &reference.dag, - BTreeMap::new(), - &samples, - &reference_roots, - query, - ) - .await; - // One rate per series, equal to the reference. - assert_eq!(from_tumbling_windows[0].len(), 3); - assert_eq!(from_tumbling_windows, expected); -} diff --git a/crates/planner/tests/runtime_install.rs b/crates/planner/tests/runtime_install.rs index 7b952acb2..b8087aab7 100644 --- a/crates/planner/tests/runtime_install.rs +++ b/crates/planner/tests/runtime_install.rs @@ -16,28 +16,16 @@ use asap_executor::sources::{ }; use asap_executor::values::{Batch, Value}; use asap_executor::Error as ExecutorError; -use asap_physical_asap_optimization::test_support::split_shared_by_timing; use asap_planner::{ - plan_installable, run_planning, CostInputs, DeploymentCapabilities, FrontendInput, - InstallablePlan, InstallablePlanError, MaterializedView, MaterializedViewRefresh, OperatorKind, - OptimizationInput, OptimizationPass, OptimizeError, PhysicalASAPDAG, PlanOutput, - PlanSearchSettings, PlanningModels, PlanningWorkload, RefreshWindow, SelectedPlanReport, - SelectionBox, Stage1ToStage3Pass, UserInput, + plan_installable, CostInputs, DeploymentCapabilities, FrontendInput, InstallablePlan, + MaterializedView, OperatorKind, PhysicalASAPDAG, PlanningModels, PlanningWorkload, + SelectedPlanReport, SelectionBox, UserInput, }; use asap_types::ir::operator::Source; -use asap_types::ir::schema::schema_support::with_promql_series_identity; -use asap_types::ir::schema::{FieldDataType, SketchAlgorithm, SketchParams}; -use asap_types::ir::{ - apply_materialization_timings_to_roots, ASAPOp, MaterializationAssignment, OperatorNode, - QueryRoot, -}; -use asap_types::physical::{Materialization, Medium}; -use asap_types::stages::HowToComputeAggregateMeasure; -use asap_types::stages::PruneReason; + use futures::{executor::block_on, StreamExt}; -use std::collections::HashMap; use std::ops::Bound; -use std::rc::Rc; + use time_series::{series_labels, time_series_tables, RawDataSample}; /// Example 4b's evaluation time: the end of the 60-minute window. @@ -80,31 +68,6 @@ fn sample(metric: &str, instance: &str, timestamp_ms: i64, value: f64) -> RawDat } } -/// Raw `latency_ms` samples of series `a` and `b`, one every 10 s from 70 -/// minutes before `EVALUATION_MS` to a minute after it (for a second -/// evaluation), off the tumbling-window boundaries, and one -/// sample of another metric the plan never reads. A sample's value says -/// which 10-minute tumbling window before `EVALUATION_MS` it falls in: `1000 × tumbling_window + -/// base + 0..=100`, with base 100 for `a` and 300 for `b` (tumbling window 0 is the -/// newest), so an answer read from the wrong tumbling windows is visibly off. -fn remote_write_samples() -> Vec { - let mut samples = Vec::new(); - for (instance, base) in [("a", 100.0), ("b", 300.0)] { - for step in 0..426 { - let timestamp_ms = EVALUATION_MS - 4_195_000 + step * 10_000; - let tumbling_window = (EVALUATION_MS - timestamp_ms) / 600_000; - samples.push(sample( - "latency_ms", - instance, - timestamp_ms, - (1000 * tumbling_window) as f64 + base + (step * 37 % 101) as f64, - )); - } - } - samples.push(sample("cpu_seconds", "a", 5_000, 1e9)); - samples -} - thread_local! { /// How many `read_raw_data` calls the runs on this thread made. static RAW_DATA_READS: Cell = const { Cell::new(0) }; @@ -206,23 +169,6 @@ fn answers(batches: &[Batch]) -> Vec { answers } -/// Each answer is a p99 read from tumbling window `tumbling_window` of its series -/// ([`remote_write_samples`]): within `1000 × tumbling_window + base + 0..=100`. -fn assert_answers_from_tumbling_window(answers: &[Answer], tumbling_window: i64) { - for (labels, value) in answers { - let base = if labels["instance"] == "a" { - 100.0 - } else { - 300.0 - }; - let low = (1000 * tumbling_window) as f64 + base; - assert!( - (low..=low + 100.0).contains(value), - "{labels:?}: {value} is not from tumbling window {tumbling_window}" - ); - } -} - /// The control plane: plan `queries` for a deployment that keeps no raw /// samples, with its cost model read from JSON, and write the plan file. async fn plan_json(queries: &[&str]) -> String { @@ -280,453 +226,11 @@ async fn plan_with_report_for( .unwrap() } -/// What one data-plane evaluation of a plan file returned. -struct DataPlaneRun { - /// Per query, in workload order, the installed plan's answers. - answers_per_query: Vec>, - /// Per query, the query-time reference's answers. - reference_answers_per_query: Vec>, - /// How many materialized views this evaluation refreshed (ingestion or - /// query runs); the others it read came from the store. - outputs_built: usize, - /// How many `read_raw_data` calls those refreshes made. - raw_data_reads_refreshing: usize, -} - -/// One summary state this test stores: its refreshed state, and how long -/// to keep it. -struct StoredState { - /// The refreshed state, bound at every materialized view with its key. - state: Batch, - /// When the data it summarizes starts, in Unix milliseconds. - start_ms: i64, - /// How far back from the newest evaluation it is kept: the largest - /// `retention_ms` among the views it served. - retention_ms: u64, -} - -/// This stand-in runtime's summary store, by [`state_key`]. -type SummaryStore = BTreeMap; - -/// The key of `view`'s state at evaluation `evaluation_ms`, as a summary -/// store keys instances: its definition id and its selection made -/// concrete (relative time made absolute). A view whose selection -/// names no time (its definition reads a window relative to the -/// evaluation, e.g. an instant selector) is keyed by the evaluation too. -fn state_key(view: &MaterializedView, evaluation_ms: i64) -> String { - use std::ops::Bound; - let absolute = |bound: &Bound| match bound { - Bound::Included(at) => Bound::Included(evaluation_ms + at), - Bound::Excluded(at) => Bound::Excluded(evaluation_ms + at), - Bound::Unbounded => Bound::Unbounded, - }; - let boxes = view.coverage().selection(); - let timed = boxes.iter().any(|b| b.relative_time().is_some()); - let selection: Vec = boxes - .iter() - .map(|selection_box| { - SelectionBox::new( - selection_box.columns().clone(), - selection_box - .relative_time() - .map(|(lower, upper)| (absolute(&lower), absolute(&upper))), - ) - }) - .collect(); - serde_json::json!({ - "definition_id": view.coverage().definition_id(), - "selection": selection, - "evaluation_ms": (!timed).then_some(evaluation_ms), - }) - .to_string() -} - /// The node that answers each query of `plan`, in workload order. fn query_roots(plan: &InstallablePlan) -> Vec { plan.dag.roots.iter().map(|&root| root as u64).collect() } -/// The data plane at evaluation time `evaluation_ms`: from `plan_json`, -/// `samples` and the states earlier evaluations left in `store`, refresh each -/// materialized view the store lacks once, from its raw-data windows and the -/// views it reads: the views refreshed at ingestion by refresh group, one -/// ingestion run per group and span, then the others; answer every query; drop the -/// states past their retention; and answer the queries again with the -/// query-time reference. -fn run_data_plane( - plan_json: &str, - samples: &[RawDataSample], - evaluation_ms: i64, - store: &mut SummaryStore, -) -> DataPlaneRun { - run_data_plane_on( - plan_json, - samples, - evaluation_ms, - store, - IngestionBuilds::PerEvaluation, - ) -} - -/// When this stand-in runtime refreshes a materialized view an ingestion -/// run refreshes from raw data samples. -#[derive(Clone, Copy)] -enum IngestionBuilds { - /// For each evaluation, over windows ending at that evaluation. - PerEvaluation, - /// Tumbling, as a runtime that maintains tumbling windows as samples arrive: each - /// once, over windows ending on the grid of its window's width anchored - /// at the epoch (tumbling window end ≡ 0 mod width, as the backend data plane - /// builds them); an evaluation reads the newest one at or before it. - Tumbling, -} - -/// [`run_data_plane`] with at-ingestion views refreshed as `ingestion` says. -fn run_data_plane_on( - plan_json: &str, - samples: &[RawDataSample], - evaluation_ms: i64, - store: &mut SummaryStore, - ingestion: IngestionBuilds, -) -> DataPlaneRun { - // The installed plan, read back from its file and checked. - let installed: InstallablePlan = serde_json::from_str(plan_json).unwrap(); - installed.validate().unwrap(); - // The DAG the executor runs. - let dag = &installed.dag; - let views = installed.materialized_views().unwrap(); - // Every input of the query run, keyed by the node it is bound to. - let mut inputs = BTreeMap::new(); - // How many materialized views this evaluation refreshes. - let mut outputs_built = 0; - // Each view's state key and the time span of its data at this - // evaluation, `(start_ms, end_ms]`, by node id. - let mut planned: BTreeMap = BTreeMap::new(); - for view in &views { - // The evaluation its windows end at: this one, or for a tumbling - // at-ingestion view the newest point of its grid at or before it. - let built_for_ms = match (ingestion, view.refresh()) { - (IngestionBuilds::Tumbling, MaterializedViewRefresh::AtIngestion) => view - .inputs - .raw_data_windows() - .first() - .map_or(evaluation_ms, |window| { - evaluation_ms - evaluation_ms.rem_euclid(window.width_ms()) - }), - _ => evaluation_ms, - }; - // The span of the data it summarizes: its windows' and the views' - // it reads spans together. - let span = view - .inputs - .raw_data_windows() - .iter() - .map(|window| window.bounds_ms(built_for_ms)) - .chain( - view.inputs - .materialized_views - .iter() - .map(|id| planned[&(*id as u64)].1), - ) - .reduce(|(a, b), (c, d)| (a.min(c), b.max(d))) - .expect("a materialized view reads something"); - planned.insert(view.node_id as u64, (state_key(view, built_for_ms), span)); - } - // The runs that refresh the views the store lacks: for each refresh - // group, one ingestion run per span its views' data covers, with every - // view of the group over that span as a root (a node they share runs - // once); then each view refreshed on first read in this evaluation's - // query run. - let mut refreshes: Vec<(Vec<&MaterializedView>, Scope)> = Vec::new(); - for group in installed.ingestion_refresh_groups().unwrap() { - let mut by_span: BTreeMap<(i64, i64), Vec<&MaterializedView>> = BTreeMap::new(); - for member in &group.views { - let view = views - .iter() - .find(|view| view.node_id == member.node_id) - .unwrap(); - by_span - .entry(planned[&(view.node_id as u64)].1) - .or_default() - .push(view); - } - for (span, members) in by_span { - let scope = Scope::Ingestion { - window_start_ms: span.0, - window_end_ms: span.1, - }; - refreshes.push((members, scope)); - } - } - for view in views - .iter() - .filter(|view| view.refresh() == MaterializedViewRefresh::OnFirstRead) - { - let scope = Scope::Query { - evaluation_time_ms: evaluation_ms, - }; - refreshes.push((vec![view], scope)); - } - let raw_data_reads_before = RAW_DATA_READS.with(Cell::get); - for (members, scope) in refreshes { - // The views of the run whose state the store lacks, one per state. - let mut roots: Vec<&MaterializedView> = Vec::new(); - for view in members { - let key = &planned[&(view.node_id as u64)].0; - if !store.contains_key(key) - && !roots - .iter() - .any(|root| &planned[&(root.node_id as u64)].0 == key) - { - roots.push(view); - } - } - if roots.is_empty() { - continue; - } - // The views the run reads that it does not refresh, from the - // store; its raw data is bound at its `Scan`s. - let mut sources = BTreeMap::new(); - for id in roots - .iter() - .flat_map(|view| &view.inputs.materialized_views) - { - if !roots.iter().any(|root| root.node_id == *id) { - let key = &planned[&(*id as u64)].0; - sources.insert(*id as u64, store[key].state.clone()); - } - } - let root_ids: Vec = roots.iter().map(|view| view.node_id as u64).collect(); - // Each root's batches, in one batch per root, as the store keeps it. - let outputs = run(dag, sources, samples, &root_ids, scope); - for (view, batches) in roots.into_iter().zip(outputs) { - let (key, span) = &planned[&(view.node_id as u64)]; - let schema = batches[0].schema().clone(); - let state_rows = batches.iter().flat_map(|b| b.to_rows().to_vec()).collect(); - store.insert( - key.clone(), - StoredState { - state: Batch::try_new(schema, state_rows).unwrap(), - start_ms: span.0, - retention_ms: view.retention_ms(), - }, - ); - outputs_built += 1; - } - } - let raw_data_reads_refreshing = RAW_DATA_READS.with(Cell::get) - raw_data_reads_before; - // Each view's state and the span of its data, by node id. - let mut built: BTreeMap = BTreeMap::new(); - for view in &views { - let (key, span) = &planned[&(view.node_id as u64)]; - // One stored state serves every view with its key, so it is kept - // for the largest retention any of them requires. - let stored = store.get_mut(key).unwrap(); - stored.retention_ms = stored.retention_ms.max(view.retention_ms()); - built.insert(view.node_id as u64, (stored.state.clone(), *span)); - } - // The query run reads the views it reaches from the queries' roots - // without passing another view; those below are read only through the - // views refreshed from them. - let mut pending = query_roots(&installed); - let mut seen = std::collections::BTreeSet::new(); - while let Some(id) = pending.pop() { - if !seen.insert(id) { - continue; - } - match built.get(&id) { - Some((state, _)) => { - inputs.insert(id, state.clone()); - } - None => pending.extend( - dag.edges - .iter() - .filter(|edge| edge.consumer as u64 == id) - .map(|edge| edge.producer as u64), - ), - } - } - // The query evaluation, which reads the stored states. - let query = Scope::Query { - evaluation_time_ms: evaluation_ms, - }; - // Each query's answers: (series labels, value) per series. - let answers_per_query = run( - dag, - inputs, - samples, - &query_roots(&installed), - query.clone(), - ) - .iter() - .map(|batches| answers(batches)) - .collect(); - // The store drops each state whose data starts more than its retention - // before this evaluation. - store.retain(|_, state| state.start_ms >= evaluation_ms - state.retention_ms as i64); - - // The correctness reference: nothing stored, every window read from - // the raw samples at query time. - let reference = asap_planner::test_support::query_time_reference(&installed).unwrap(); - // The reference's DAG. - let reference_dag = &reference.dag; - // The reference's answers, which the installed plan's must equal. - let reference_answers_per_query = run( - reference_dag, - BTreeMap::new(), - samples, - &query_roots(&reference), - query, - ) - .iter() - .map(|batches| answers(batches)) - .collect(); - DataPlaneRun { - answers_per_query, - reference_answers_per_query, - outputs_built, - raw_data_reads_refreshing, - } -} - -/// The sketch algorithm of the tumbling windows view `id` of `plan` builds. -fn tumbling_window_sketch(plan: &InstallablePlan, id: usize) -> SketchAlgorithm { - let node = plan.dag.nodes.iter().find(|node| node.id == id).unwrap(); - match &node.payload { - asap_executor::planner::ir::physical_export::PhysicalASAPOperatorPayload::ASAP( - ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), - .. - }, - ) => *kind.algorithm(), - other => panic!("not a sketch tumbling window: {other:?}"), - } -} - -/// Issue #1151's example: a p90 at rank error 0.01 over a minute and a p99 -/// at relative error 0.01 over five minutes, both every 10 s, over 1,000 -/// ingesting non-negative `latency_ms` gauges sampled every 10 s, on a -/// deployment that keeps no raw data: a KLL chain and a DDSketch chain, -/// both of 10 s tumbling windows, kept at ingestion over one `Scan`. Their tumbling windows are one refresh group, so each -/// 10 s window is read from raw data once, in one ingestion run that builds -/// both tumbling windows ending there: 30 reads for the 30 tumbling window windows of the first -/// evaluation (36 when each tumbling window view had its own run). Each state equals -/// the one a run of its view alone builds, and both queries answer as the -/// query-time reference. -#[tokio::test] -async fn tumbling_windows_of_one_refresh_window_read_each_raw_window_once() { - let entry = |query: &str, metric: &str| { - format!( - r#"{{"query": {query:?}, "demand": {{"fixed_interval": 10000}}, - "requirements": {{"accuracy": {{"accuracy_metric": {metric:?}, "max_error": 0.01, - "failure_probability": 0.01, "accuracy_scope": "whole_result"}}}}, - "predictability": {{"predictable": {{"known_at": null}}}}}}"# - ) - }; - let workload: PlanningWorkload = serde_json::from_str(&format!( - r#"{{"query_workload": {{"language": "promql", "repeating_queries": [{}, {}]}}, - "data_workload": {{"arrival": {{"ingesting": {{"rows_per_second": 100.0}}}}, - "sources": {{"latency_ms": {{"series_count": 1000, "sample_interval": 10000, - "metric_type": "gauge", "values_non_negative": true}}}}}}}}"#, - entry("quantile_over_time(0.9, latency_ms[1m])", "rank"), - entry("quantile_over_time(0.99, latency_ms[5m])", "relative"), - )) - .unwrap(); - let capabilities = DeploymentCapabilities { - raw_data_retained: false, - ..DeploymentCapabilities::default() - }; - let plan_json = plan_json_for(workload, &capabilities).await; - let installed: InstallablePlan = serde_json::from_str(&plan_json).unwrap(); - installed.check_compiles().unwrap(); - // One refresh group: the six 10 s KLL tumbling windows and the thirty 10 s - // DDSketch tumbling windows. - let groups = installed.ingestion_refresh_groups().unwrap(); - assert_eq!(groups.len(), 1, "{groups:#?}"); - assert_eq!( - groups[0].refresh_window, - Some(RefreshWindow { - width_ms: 10_000, - phase_ms: 0 - }) - ); - let sketches: Vec = groups[0] - .views - .iter() - .map(|view| tumbling_window_sketch(&installed, view.node_id)) - .collect(); - let count = |algorithm| sketches.iter().filter(|a| **a == algorithm).count(); - assert_eq!(count(SketchAlgorithm::Kll), 6, "{sketches:?}"); - assert_eq!(count(SketchAlgorithm::DDSketch), 30, "{sketches:?}"); - - let samples = remote_write_samples(); - let mut store = SummaryStore::new(); - let evaluated = run_data_plane(&plan_json, &samples, EVALUATION_MS, &mut store); - assert_eq!(evaluated.outputs_built, 36); - assert_eq!(evaluated.raw_data_reads_refreshing, 30); - assert_eq!( - evaluated.answers_per_query, - evaluated.reference_answers_per_query - ); - assert_eq!(evaluated.answers_per_query[0].len(), 2); - - // Each tumbling window's state equals the one an ingestion run of its view alone - // builds. - for view in &groups[0].views { - let span = view.inputs.raw_data_windows()[0].bounds_ms(EVALUATION_MS); - let scope = Scope::Ingestion { - window_start_ms: span.0, - window_end_ms: span.1, - }; - let alone = run( - &installed.dag, - BTreeMap::new(), - &samples, - &[view.node_id as u64], - scope, - ) - .remove(0); - // `Value` has no equality; its debug form shows each state. - let rows = format!( - "{:?}", - alone - .iter() - .flat_map(|b| b.to_rows().to_vec()) - .collect::>() - ); - let stored = &store[&state_key(view, EVALUATION_MS)].state; - assert_eq!( - format!("{:?}", stored.to_rows()), - rows, - "view {}", - view.node_id - ); - } -} - -/// #509 Example 4b, as a runtime runs it: one `plan_installable` call, -/// `plan.json`, then each tumbling window view refreshed in an ingestion run and merged -/// in the query run. One p99 per series, read from the hour's oldest tumbling window, -/// equal to the query-time reference; the other metric is not read. -#[tokio::test] -async fn a_runtime_installs_and_runs_the_plan_from_one_facade_call() { - let plan_json = plan_json(&["quantile_over_time(0.99, latency_ms[60m])"]).await; - let run = run_data_plane( - &plan_json, - &remote_write_samples(), - EVALUATION_MS, - &mut SummaryStore::new(), - ); - let installed: InstallablePlan = serde_json::from_str(&plan_json).unwrap(); - let views = installed.materialized_views().unwrap(); - assert!(!views.is_empty()); - assert_eq!(run.outputs_built, views.len()); - // The p99 of the hour comes from its oldest tumbling window, tumbling window 5. - assert_answers_from_tumbling_window(&run.answers_per_query[0], 5); - assert_eq!(run.answers_per_query[0][0].0["instance"], "a"); - assert_eq!(run.answers_per_query, run.reference_answers_per_query); -} - /// The plan file carries Stage 3's predicted usage of the selected plan, so /// a runtime reports it against what it measures without planning again: /// for Example 4b's tumbling windows maintained at ingestion time, its nodes' prices @@ -795,392 +299,6 @@ async fn the_plan_carries_stage3s_predicted_usage() { .any(|node| node.installed_node_ids == [root])); } -/// The query's label matcher selects series `a` only: the runtime feeds -/// every sample, and the window's selector applies the plan's `Scan`. -#[tokio::test] -async fn label_matchers_select_the_series_a_tumbling_window_reads() { - let plan_json = - plan_json(&[r#"quantile_over_time(0.99, latency_ms{instance="a"}[60m])"#]).await; - let run = run_data_plane( - &plan_json, - &remote_write_samples(), - EVALUATION_MS, - &mut SummaryStore::new(), - ); - let answered = &run.answers_per_query[0]; - assert_eq!(answered.len(), 1, "{answered:?}"); - assert_eq!(answered[0].0["instance"], "a"); - assert_answers_from_tumbling_window(answered, 5); - assert_eq!(run.answers_per_query, run.reference_answers_per_query); -} - -/// The p99 over the last hour and over the hour before the last 10 -/// minutes may share selected tumbling windows. The runtime refreshes each distinct -/// state key once, independent of the optimizer's tumbling-window widths. Both answers -/// equal the reference. -#[tokio::test] -async fn an_hour_and_its_offset_twin_both_equal_the_reference() { - let plan_json = plan_json(&[ - "quantile_over_time(0.99, latency_ms[60m])", - "quantile_over_time(0.99, latency_ms[60m] offset 10m)", - ]) - .await; - let installed: InstallablePlan = serde_json::from_str(&plan_json).unwrap(); - let distinct_states: BTreeSet<_> = installed - .materialized_views() - .unwrap() - .iter() - .map(|view| state_key(view, EVALUATION_MS)) - .collect(); - assert!(!distinct_states.is_empty()); - let run = run_data_plane( - &plan_json, - &remote_write_samples(), - EVALUATION_MS, - &mut SummaryStore::new(), - ); - assert_eq!(run.outputs_built, distinct_states.len()); - assert_eq!(run.answers_per_query.len(), 2); - // The last hour's p99 comes from tumbling window 5, the offset hour's from tumbling window 6. - assert_answers_from_tumbling_window(&run.answers_per_query[0], 5); - assert_answers_from_tumbling_window(&run.answers_per_query[1], 6); - assert_eq!(run.answers_per_query, run.reference_answers_per_query); -} - -/// Two copies of `sum by (job) (sum_over_time(m[10m]))`, each every 20 s, -/// one at phase 0 and one at phase 5 s, over 6 ingesting series of `m` -/// sampled every second. Tumbling windows shared by both must end on both schedules: -/// 20 s tumbling windows end only on the first's, so the second would read -/// `(t − 605 s, t − 5 s]` instead of `(t − 600 s, t]`. -#[tokio::test] -async fn queries_at_two_phases_read_tumbling_windows_on_both_schedules() { - assert_phased_copies_answer_as_reference(&[0, 5_000]).await; -} - -/// The same two copies, both at phase 5 s. A runtime anchors tumbling windows at the -/// epoch and the plan carries no tumbling-window origin, so 20 s tumbling windows would end 5 s -/// before every evaluation: the tumbling windows must also end on the phase itself. -#[tokio::test] -async fn queries_at_one_nonzero_phase_read_tumbling_windows_anchored_at_the_epoch() { - assert_phased_copies_answer_as_reference(&[5_000, 5_000]).await; -} - -/// Copies of `sum by (job) (sum_over_time(m[10m]))`, each every 20 s, copy -/// `q` at phase `phases_ms[q]`, over 6 ingesting series of `m` sampled -/// every second. Run by a runtime that refreshes each ingestion tumbling window once, on -/// the grid of its width anchored at the epoch, each answers as the -/// query-time reference at its own evaluation. -async fn assert_phased_copies_answer_as_reference(phases_ms: &[i64]) { - let entries: Vec = phases_ms - .iter() - .map(|phase_ms| { - format!( - r#"{{"query": "sum by (job) (sum_over_time(m[10m]))", - "demand": {{"fixed_interval_at": {{"interval": 20000, "evaluation_phase": {phase_ms}}}}}, - "requirements": {{}}, - "predictability": {{"predictable": {{"known_at": null}}}}}}"# - ) - }) - .collect(); - let workload: PlanningWorkload = serde_json::from_str(&format!( - r#"{{"query_workload": {{"language": "promql", "repeating_queries": [{}]}}, - "data_workload": {{"arrival": {{"ingesting": {{"rows_per_second": 6.0}}}}, - "sources": {{"m": {{"series_count": 6, "sample_interval": 1000}}}}}}}}"#, - entries.join(",") - )) - .unwrap(); - // A deployment that keeps no raw data samples: the windows are views. - let capabilities = DeploymentCapabilities { - raw_data_retained: false, - ..DeploymentCapabilities::default() - }; - let plan_json = plan_json_for(workload, &capabilities).await; - let plan: InstallablePlan = serde_json::from_str(&plan_json).unwrap(); - let views = plan.materialized_views().unwrap(); - assert!( - views - .iter() - .any(|view| view.refresh() == MaterializedViewRefresh::AtIngestion), - "{views:#?}" - ); - // Samples of `m`, `job` a (instances 1-3) and b (4-6), every second - // from 11 minutes before the first evaluation to 10 s after it; each - // value distinct, so a window off by 5 s at either edge answers - // differently. - let samples: Vec = (1..=6i64) - .flat_map(|instance| { - (0..680i64).map(move |step| RawDataSample { - labels: BTreeMap::from([ - ("__name__".to_string(), "m".to_string()), - ( - "job".to_string(), - if instance <= 3 { "a" } else { "b" }.to_string(), - ), - ("instance".to_string(), instance.to_string()), - ]), - timestamp_ms: EVALUATION_MS - 660_000 + step * 1_000, - value: (instance * 1000 + step * step % 997) as f64, - }) - }) - .collect(); - // The tumbling windows stored across every evaluation. - let mut store = SummaryStore::new(); - // Copy `q` evaluates `phases_ms[q]` after `EVALUATION_MS` (on the - // 20 s grid); each reads the tumbling windows the store holds by then. - for (query, phase_ms) in phases_ms.iter().enumerate() { - let evaluation_ms = EVALUATION_MS + phase_ms; - let run = run_data_plane_on( - &plan_json, - &samples, - evaluation_ms, - &mut store, - IngestionBuilds::Tumbling, - ); - assert_eq!(run.answers_per_query[query].len(), 2, "one sum per job"); - assert_eq!( - run.answers_per_query[query], run.reference_answers_per_query[query], - "query {query} at {evaluation_ms}" - ); - } -} - -/// A query whose window equals its interval, with no declared phase: one -/// tumbling window per evaluation, maintained at ingestion time, answers -/// every evaluation without raw data. `quantile_over_time(0.99, m[5m])` -/// every 5 minutes at rank error 0.01 (a KLL per series) had no plan. -#[tokio::test] -async fn a_window_equal_to_its_interval_is_kept_at_ingestion_time() { - assert_answers_as_reference_without_raw_data( - "quantile_over_time(0.99, m[5m])", - r#"{"fixed_interval": 300000}"#, - RANK_ERROR_1_PERCENT, - &[0, 300_000, 600_000], - ) - .await; - assert_answers_as_reference_without_raw_data( - "sum by (job) (sum_over_time(m[20s]))", - r#"{"fixed_interval": 20000}"#, - "{}", - &[0, 20_000, 40_000, 60_000], - ) - .await; -} - -/// The same 20 s window every 20 s at phase 5 s: a 20 s window on the -/// epoch grid would end 5 s before every evaluation, so the plan must not -/// read one; it still answers every evaluation as the reference. -#[tokio::test] -async fn a_window_equal_to_its_interval_at_an_offset_phase_answers_as_reference() { - assert_answers_as_reference_without_raw_data( - "sum by (job) (sum_over_time(m[20s]))", - r#"{"fixed_interval_at": {"interval": 20000, "evaluation_phase": 5000}}"#, - "{}", - &[5_000, 25_000, 45_000, 65_000], - ) - .await; -} - -/// Rank error 0.01 with failure probability 0.01, as `requirements`. -const RANK_ERROR_1_PERCENT: &str = r#"{"accuracy": {"accuracy_metric": "rank", "max_error": 0.01, - "failure_probability": 0.01, "accuracy_scope": "whole_result"}}"#; - -/// Plans `query` with `demand` and `requirements` over 6 ingesting series of `m` sampled every -/// second, for a deployment that keeps no raw data, and runs it at -/// `EVALUATION_MS` plus each of `evaluation_offsets_ms` on a runtime that -/// refreshes each ingestion state once, on the epoch grid of its width. The -/// plan keeps state at ingestion time and reads no raw data at query time; -/// every answer equals the query-time reference. -async fn assert_answers_as_reference_without_raw_data( - query: &str, - demand: &str, - requirements: &str, - evaluation_offsets_ms: &[i64], -) { - let workload: PlanningWorkload = serde_json::from_str(&format!( - r#"{{"query_workload": {{"language": "promql", "repeating_queries": [ - {{"query": {query:?}, "demand": {demand}, "requirements": {requirements}, - "predictability": {{"predictable": {{"known_at": null}}}}}}]}}, - "data_workload": {{"arrival": {{"ingesting": {{"rows_per_second": 6.0}}}}, - "sources": {{"m": {{"series_count": 6, "sample_interval": 1000}}}}}}}}"# - )) - .unwrap(); - let capabilities = DeploymentCapabilities { - raw_data_retained: false, - ..DeploymentCapabilities::default() - }; - let plan_json = plan_json_for(workload, &capabilities).await; - let plan: InstallablePlan = serde_json::from_str(&plan_json).unwrap(); - let views = plan.materialized_views().unwrap(); - assert!( - views - .iter() - .any(|view| view.refresh() == MaterializedViewRefresh::AtIngestion), - "{views:#?}" - ); - assert!( - plan.raw_data_retention().unwrap().is_empty(), - "no raw data at query time" - ); - // Samples of `m`, `job` a (instances 1-3) and b (4-6), every second - // from 6 minutes before `EVALUATION_MS` to past the last evaluation; - // each value distinct, so a window off by a second answers differently. - let last_ms = EVALUATION_MS + evaluation_offsets_ms.iter().max().unwrap(); - let steps = (last_ms - EVALUATION_MS + 360_000) / 1_000 + 10; - let samples: Vec = (1..=6i64) - .flat_map(|instance| { - (0..steps).map(move |step| RawDataSample { - labels: BTreeMap::from([ - ("__name__".to_string(), "m".to_string()), - ( - "job".to_string(), - if instance <= 3 { "a" } else { "b" }.to_string(), - ), - ("instance".to_string(), instance.to_string()), - ]), - timestamp_ms: EVALUATION_MS - 360_000 + step * 1_000, - value: (instance * 1000 + step * step % 997) as f64, - }) - }) - .collect(); - // The states stored across every evaluation. - let mut store = SummaryStore::new(); - for offset_ms in evaluation_offsets_ms { - let evaluation_ms = EVALUATION_MS + offset_ms; - let run = run_data_plane_on( - &plan_json, - &samples, - evaluation_ms, - &mut store, - IngestionBuilds::Tumbling, - ); - assert!(!run.answers_per_query[0].is_empty()); - assert_eq!( - run.answers_per_query, run.reference_answers_per_query, - "{query} at {evaluation_ms}" - ); - } -} - -/// The same exact query twice, one copy every 10 s and one every 15 s -/// (hash-consed into one memo group), on a deployment that keeps no raw -/// data. A tumbling-window chain is built at ingestion time once per -/// tumbling window, on the grid of its width anchored at the epoch, so a -/// chain either copy reads must have a width dividing that copy's interval: -/// a 15 s chain read every 10 s ends up to 10 s before the evaluation -/// (#1173). Every evaluation of both copies over one minute answers as the -/// query-time reference. -#[tokio::test] -async fn queries_at_two_intervals_read_chains_on_their_own_grids() { - for query in [ - "sum by (job) (rate(m[1m]))", - "sum by (job) (sum_over_time(m[1m]))", - ] { - assert_copies_at_intervals_answer_as_reference(query, &[10_000, 15_000]).await; - } -} - -/// Copies of `query`, copy `q` every `intervals_ms[q]`, over 6 ingesting -/// series of `m` sampled every second, on a deployment that keeps no raw -/// data. Run by a runtime that refreshes each ingestion tumbling window -/// once, on the grid of its width anchored at the epoch, every evaluation -/// of every copy over one minute answers as the query-time reference. -async fn assert_copies_at_intervals_answer_as_reference(query: &str, intervals_ms: &[i64]) { - let entries: Vec = intervals_ms - .iter() - .map(|interval_ms| { - format!( - r#"{{"query": {query:?}, - "demand": {{"fixed_interval": {interval_ms}}}, - "requirements": {{}}, - "predictability": {{"predictable": {{"known_at": null}}}}}}"# - ) - }) - .collect(); - let workload: PlanningWorkload = serde_json::from_str(&format!( - r#"{{"query_workload": {{"language": "promql", "repeating_queries": [{}]}}, - "data_workload": {{"arrival": {{"ingesting": {{"rows_per_second": 6.0}}}}, - "sources": {{"m": {{"series_count": 6, "sample_interval": 1000}}}}}}}}"#, - entries.join(",") - )) - .unwrap(); - // A deployment that keeps no raw data samples: the windows are views. - let capabilities = DeploymentCapabilities { - raw_data_retained: false, - ..DeploymentCapabilities::default() - }; - let plan_json = plan_json_for(workload, &capabilities).await; - // Samples of `m`, `job` a (instances 1-3) and b (4-6), every second - // from 2 minutes before `EVALUATION_MS` to past the last evaluation; - // each value distinct, so a window off by a second answers differently. - let samples: Vec = (1..=6i64) - .flat_map(|instance| { - (0..200i64).map(move |step| RawDataSample { - labels: BTreeMap::from([ - ("__name__".to_string(), "m".to_string()), - ( - "job".to_string(), - if instance <= 3 { "a" } else { "b" }.to_string(), - ), - ("instance".to_string(), instance.to_string()), - ]), - timestamp_ms: EVALUATION_MS - 120_000 + step * 1_000, - value: (instance * 1000 + step * step % 997) as f64, - }) - }) - .collect(); - // Every evaluation of every copy over one minute, in time order. - let mut evaluations: Vec<(usize, i64)> = intervals_ms - .iter() - .enumerate() - .flat_map(|(query, &interval_ms)| { - (0..=60_000 / interval_ms).map(move |k| (query, EVALUATION_MS + k * interval_ms)) - }) - .collect(); - evaluations.sort_by_key(|&(query, evaluation_ms)| (evaluation_ms, query)); - // The states stored across every evaluation. - let mut store = SummaryStore::new(); - for (copy, evaluation_ms) in evaluations { - let run = run_data_plane_on( - &plan_json, - &samples, - evaluation_ms, - &mut store, - IngestionBuilds::Tumbling, - ); - assert_eq!(run.answers_per_query[copy].len(), 2, "one answer per job"); - assert_eq!( - run.answers_per_query[copy], run.reference_answers_per_query[copy], - "{query}, copy {copy} (every {} ms) at {evaluation_ms}", - intervals_ms[copy] - ); - } -} - -/// A plan file of another DAG wire version is refused, and so is the -/// reference of a tampered plan file, with an error rather than a panic. -#[tokio::test] -async fn a_plan_file_of_another_version_or_tampered_is_refused() { - let plan_json = plan_json(&["quantile_over_time(0.99, latency_ms[60m])"]).await; - let mut other_version: InstallablePlan = serde_json::from_str(&plan_json).unwrap(); - other_version.wire_version += 1; - assert!(matches!( - other_version.validate(), - Err(InstallablePlanError::UnsupportedWireVersion(_)) - )); - let mut tampered: InstallablePlan = serde_json::from_str(&plan_json).unwrap(); - tampered.query_texts.clear(); - assert!(matches!( - asap_planner::test_support::query_time_reference(&tampered), - Err(InstallablePlanError::QueriesDoNotMatchRoots { .. }) - )); - let mut missing_node: InstallablePlan = serde_json::from_str(&plan_json).unwrap(); - missing_node.dag.nodes.remove(0); - assert!(matches!( - asap_planner::test_support::query_time_reference(&missing_node), - Err(InstallablePlanError::InvalidDag(_)) - )); -} - /// `sum(latency_ms)` reads an instant selector: per series, only the latest /// sample in its 1 s lookback counts. Series `a` has samples at T-900 ms (1) /// and T-100 ms (10), series `b` at T-200 ms (100): the sum is 10 + 100. @@ -1217,682 +335,21 @@ async fn an_instant_selector_reads_the_latest_sample_per_series() { assert_eq!(answered[0].1, 110.0); } -/// A deployment that keeps raw data samples and query-time results across -/// evaluations (#509 Example 4's B3), with a dashboard panel -/// (`max_over_time(latency_ms[5m])` every minute over 1M series sampled -/// every 1.5 s, answered as the last hour at 1-minute steps): Stage 3 -/// keeps five 1-minute tumbling windows refreshed at query time, and keeps raw data -/// for refreshing the oldest tumbling window at the oldest step (5 min + 1 h). The first -/// evaluation, with an empty store, refreshes all five in its query run; the -/// next, a minute later, reuses the four still in its window and refreshes -/// only the newest. Each -/// evaluation's answers (each series' newest-tumbling-window maximum) equal the -/// reference. -#[tokio::test] -async fn query_time_results_kept_across_evaluations_are_built_in_query_runs() { - let capabilities = DeploymentCapabilities { - query_time_results_retained: true, - ..DeploymentCapabilities::default() - }; - let workload: PlanningWorkload = serde_json::from_str( - r#"{"query_workload": {"language": "promql", "repeating_queries": [{ - "query": "max_over_time(latency_ms[5m])", - "demand": {"fixed_interval": 60000}, - "requirements": {}, - "predictability": {"predictable": {"known_at": null}}, - "range_evaluation": {"step_ms": 60000, "span_ms": 3600000}}]}, - "data_workload": {"arrival": {"ingesting": {"rows_per_second": 666666.67}}, - "sources": {"latency_ms": {"series_count": 1000000, "sample_interval": 1500}}}}"#, - ) - .unwrap(); - let plan_json = plan_json_for(workload, &capabilities).await; - let installed: InstallablePlan = serde_json::from_str(&plan_json).unwrap(); - let views = installed.materialized_views().unwrap(); - assert_eq!(views.len(), 5); - assert!(views - .iter() - .all(|v| v.refresh() == MaterializedViewRefresh::OnFirstRead - && v.inputs.raw_data_windows()[0].width_ms() == 60_000)); - // Raw data is kept for refreshing the oldest tumbling window at the oldest step: - // the five tumbling windows' 5 minutes plus the hour of steps, as long as the tumbling windows - // themselves are kept. - let tumbling_window_lookback_ms = views - .iter() - .flat_map(|v| v.inputs.raw_data_windows()) - .map(asap_types::test_support::lookback_ms) - .max(); - assert_eq!(tumbling_window_lookback_ms, Some(300_000)); - assert!(views.iter().all(|v| v.retention_ms() == 3_900_000)); - let retention = installed.raw_data_retention().unwrap(); - assert_eq!( - retention - .iter() - .map(|r| r.lookback_ms()) - .collect::>(), - vec![Some(3_900_000)], - "{retention:?}" - ); - // The tumbling window cache the data plane keeps across evaluations. - let mut store = SummaryStore::new(); - let first = run_data_plane( - &plan_json, - &remote_write_samples(), - EVALUATION_MS, - &mut store, - ); - assert_eq!(first.outputs_built, 5); - assert_answers_from_tumbling_window(&first.answers_per_query[0], 0); - assert_eq!(first.answers_per_query, first.reference_answers_per_query); - let next = run_data_plane( - &plan_json, - &remote_write_samples(), - EVALUATION_MS + 60_000, - &mut store, - ); - assert_eq!( - next.outputs_built, 1, - "four tumbling windows reused from the cache" - ); - assert_eq!(next.answers_per_query, next.reference_answers_per_query); -} - -/// An exact per-series quantile under a group average (issue #770), on a -/// deployment that keeps raw data samples: Stage 3 selects the exact plan, which -/// reads each series' raw data samples in the window at query time, and the -/// executor runs it. Series `a` (1..=10) and `b` (10, 20, ..., 100) are in -/// `east`, `c` (constantly 5) in `west`; a sample of each before the window -/// is not read. Per series, p99 interpolates between its two largest -/// samples (rank 0.99 × 9 = 8.91): 9.91 for `a`, 99.1 for `b`, 5 for `c`. -#[tokio::test] -async fn an_exact_per_series_quantile_runs_from_raw_data_samples() { - let workload: PlanningWorkload = serde_json::from_str( - r#"{"query_workload": {"language": "promql", "repeating_queries": [{ - "query": "avg by (region) (quantile_over_time(0.99, latency_ms[100s]))", - "demand": {"fixed_interval": 60000}, - "requirements": {}, - "predictability": {"predictable": {"known_at": null}}}]}, - "data_workload": {"arrival": {"ingesting": {"rows_per_second": 3.0}}, - "sources": {"latency_ms": {"series_count": 3, "sample_interval": 10000}}}}"#, - ) - .unwrap(); - let plan_json = plan_json_for(workload, &DeploymentCapabilities::default()).await; - let installed: InstallablePlan = serde_json::from_str(&plan_json).unwrap(); - assert!( - installed.materialized_views().unwrap().is_empty(), - "the exact plan" - ); - assert_eq!(installed.raw_data_retention().unwrap().len(), 1); - // Ten samples per series in the window, one every 10 s, and one before it. - let mut samples = Vec::new(); - for (instance, region, value) in [ - ("a", "east", (|step| step as f64) as fn(i64) -> f64), - ("b", "east", |step| 10.0 * step as f64), - ("c", "west", |_| 5.0), - ] { - for step in 0..=10 { - let mut sample = sample( - "latency_ms", - instance, - EVALUATION_MS - 105_000 + 10_000 * step, - if step == 0 { 1e9 } else { value(step) }, - ); - sample - .labels - .insert("region".to_string(), region.to_string()); - samples.push(sample); - } - } - let run = run_data_plane( - &plan_json, - &samples, - EVALUATION_MS, - &mut SummaryStore::new(), - ); - let answered = &run.answers_per_query[0]; - assert_eq!(answered.len(), 2, "{answered:?}"); - for ((labels, value), (region, expected)) in answered - .iter() - .zip([("east", (9.91 + 99.1) / 2.0), ("west", 5.0)]) - { - assert_eq!(labels["region"], region); - assert!((value - expected).abs() < 1e-9, "{region}: {value}"); - } - assert_eq!(run.answers_per_query, run.reference_answers_per_query); -} - // ── A ratio of two relative-error quantiles (issue #852) ──────────────────── -/// Repeating `queries`, each every 10 minutes at error 0.05 in each -/// summary's own metric (W-RangeHeavy leaves the metric unset) and δ 0.01, -/// over 1,000 ingesting non-negative `latency_ms` gauges sampled -/// every second. -fn relative_workload(queries: &[&str]) -> PlanningWorkload { - let entries: Vec = queries - .iter() - .map(|query| { - format!( - r#"{{"query": {query:?}, "demand": {{"fixed_interval": 600000}}, - "requirements": {{"accuracy": {{"max_error": 0.05, "failure_probability": 0.01, - "accuracy_scope": "whole_result"}}}}, - "predictability": {{"predictable": {{"known_at": null}}}}}}"# - ) - }) - .collect(); - serde_json::from_str(&format!( - r#"{{"query_workload": {{"language": "promql", "repeating_queries": [{}]}}, - "data_workload": {{"arrival": {{"ingesting": {{"rows_per_second": 1000.0}}}}, - "sources": {{"latency_ms": {{"series_count": 1000, "sample_interval": 1000, - "metric_type": "gauge", "values_non_negative": true}}}}}}}}"#, - entries.join(",") - )) - .unwrap() -} - -/// The DDSketch α of every sketch `plan` builds; panics on any other sketch. -fn ddsketch_alphas(plan: &InstallablePlan) -> Vec { - plan.dag - .nodes - .iter() - .filter_map(|node| match &node.payload { - asap_executor::planner::ir::physical_export::PhysicalASAPOperatorPayload::ASAP( - ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), - .. - } - | ASAPOp::SummaryTumblingWindowChain { - family: FieldDataType::Sketch(kind, _), - .. - }, - ) => match kind.params() { - SketchParams::DDSketch { alpha } => Some(*alpha), - other => panic!("not a DDSketch: {other:?}"), - }, - _ => None, - }) - .collect() -} +// ── Materialized views refreshed from materialized views (issue #781) ────────────────── -/// PromQL `quantile_over_time(q, …)` of `values`: linear interpolation -/// between the sorted values at rank `q·(n − 1)`. -fn promql_quantile(q: f64, values: &mut [f64]) -> f64 { - values.sort_by(f64::total_cmp); - let rank = q * (values.len() - 1) as f64; - let (low, high) = (rank.floor() as usize, rank.ceil() as usize); - values[low] + (values[high] - values[low]) * (rank - low as f64) +/// The time selection `(from_ms, to_ms]` relative to the evaluation. +fn relative_time(from_ms: i64, to_ms: i64) -> Vec { + vec![SelectionBox::new( + Default::default(), + Some((Bound::Excluded(from_ms), Bound::Included(to_ms))), + )] } -/// The p90/p50 ratio of an hour (W-RangeHeavy template 8) on a deployment -/// that keeps no raw data samples: both quantiles come from DDSketches -/// sized for a share of the target (α = 1 − 1.05^{−1/2} ≈ 0.0241, so -/// `(a + b)/(1 − b) ≤ 0.05`), maintained in tumbling windows; nothing is read from -/// raw data at query time. Run on data, each series' ratio is within 0.05 -/// of the exact ratio of its hour's samples. -#[tokio::test] -async fn a_ratio_of_two_quantiles_runs_from_ddsketches_within_its_relative_target() { - let query = - "quantile_over_time(0.9, latency_ms[60m]) / quantile_over_time(0.5, latency_ms[60m])"; - let capabilities = DeploymentCapabilities { - raw_data_retained: false, - ..DeploymentCapabilities::default() - }; - let plan_json = plan_json_for(relative_workload(&[query]), &capabilities).await; - let installed: InstallablePlan = serde_json::from_str(&plan_json).unwrap(); - assert!( - installed.raw_data_retention().unwrap().is_empty(), - "no raw scan" - ); - let alphas = ddsketch_alphas(&installed); - assert!(!alphas.is_empty(), "DDSketch states"); - let share = 1.0 - 1.05f64.powf(-0.5); - for alpha in &alphas { - assert!(*alpha <= share + 1e-12, "α {alpha} > {share}"); - } - let samples = remote_write_samples(); - let run = run_data_plane( - &plan_json, - &samples, - EVALUATION_MS, - &mut SummaryStore::new(), - ); - let answered = &run.answers_per_query[0]; - assert_eq!(answered.len(), 2, "{answered:?}"); - for (labels, value) in answered { - let mut hour: Vec = samples - .iter() - .filter(|s| { - s.labels["__name__"] == "latency_ms" - && s.labels["instance"] == labels["instance"] - && s.timestamp_ms > EVALUATION_MS - 3_600_000 - && s.timestamp_ms <= EVALUATION_MS - }) - .map(|s| s.value) - .collect(); - let exact = promql_quantile(0.9, &mut hour) / promql_quantile(0.5, &mut hour); - let error = (value - exact).abs() / exact; - assert!(error <= 0.05, "{labels:?}: {value} vs {exact} ({error})"); - } -} - -/// The ratio's numerator is also asked for on its own (W-RangeHeavy -/// templates 5 and 8): the p90 estimate is shared, so its state is sized -/// for its strictest reader, the ratio's share; the plan still reads no raw -/// data samples. -#[tokio::test] -#[ignore = "slow without a planning budget: over 30 min in a debug build; #1197"] -async fn a_quantile_shared_with_a_ratio_is_sized_for_the_ratio() { - let capabilities = DeploymentCapabilities { - raw_data_retained: false, - ..DeploymentCapabilities::default() - }; - let plan_json = plan_json_for( - relative_workload(&[ - "quantile_over_time(0.9, latency_ms[60m])", - "quantile_over_time(0.9, latency_ms[60m]) / quantile_over_time(0.5, latency_ms[60m])", - ]), - &capabilities, - ) - .await; - let installed: InstallablePlan = serde_json::from_str(&plan_json).unwrap(); - assert!( - installed.raw_data_retention().unwrap().is_empty(), - "no raw scan" - ); - let share = 1.0 - 1.05f64.powf(-0.5); - for alpha in ddsketch_alphas(&installed) { - assert!(alpha <= share + 1e-12, "α {alpha} > {share}"); - } - let run = run_data_plane( - &plan_json, - &remote_write_samples(), - EVALUATION_MS, - &mut SummaryStore::new(), - ); - assert_eq!(run.answers_per_query, run.reference_answers_per_query); -} - -// ── Materialized views refreshed from materialized views (issue #781) ────────────────── - -/// Stage 1 to Stage 3, then every node `store` names stored as it says, -/// the others as Stage 2 chose, and the plan timed again: how a test makes -/// the plan store a shape Stage 3 does not select today. -struct StoredAsForced { - store: fn(&OperatorNode) -> Option, -} - -impl OptimizationPass for StoredAsForced { - fn name(&self) -> &'static str { - "stored-as-forced" - } - - fn optimize(&self, input: OptimizationInput<'_>) -> Result { - let mut output = Stage1ToStage3Pass.optimize(input)?; - // Each node untimed once, so shared inputs stay shared, and its - // materialization in `assignment`. - let mut untimed: HashMap<*const OperatorNode, Rc> = HashMap::new(); - let mut assignment = MaterializationAssignment::default(); - let roots: Vec> = output - .plans - .iter() - .map(|plan| untime(&plan.root, self.store, &mut untimed, &mut assignment)) - .collect(); - // A node read both at ingestion and at query time is copied, one - // copy per timing, as Stage 2 does. - let (roots, assignment) = split_shared_by_timing(&roots, &assignment); - let roots: Vec = roots.into_iter().map(QueryRoot::Operator).collect(); - let timed = apply_materialization_timings_to_roots(&roots, &assignment).unwrap(); - for (plan, root) in output.plans.iter_mut().zip(timed) { - let QueryRoot::Operator(root) = root else { - unreachable!("operator roots in, operator roots out") - }; - plan.root = root; - } - Ok(output) - } -} - -/// `node` untimed, with `store`'s materialization (or its own) recorded in -/// `assignment` for every node of its sub-DAG. -fn untime( - node: &Rc, - store: fn(&OperatorNode) -> Option, - untimed: &mut HashMap<*const OperatorNode, Rc>, - assignment: &mut MaterializationAssignment, -) -> Rc { - if let Some(done) = untimed.get(&Rc::as_ptr(node)) { - return Rc::clone(done); - } - let mut copy = (**node).clone(); - copy.operator = node - .operator - .map_children(|child| untime(child, store, untimed, assignment)); - copy.timing = None; - copy.materialization = Materialization::NotMaterialized; - let copy = Rc::new(copy); - // A shared non-summary node Stage 2 built once per evaluation is not - // kept: a build forced to ingestion time above it cannot read a - // query-time result. - let own = match node.asap() { - Some( - ASAPOp::SummaryAgg { .. } - | ASAPOp::SummaryTumblingWindowChain { .. } - | ASAPOp::SummaryMerge { .. }, - ) => node.materialization, - _ => Materialization::NotMaterialized, - }; - let materialization = store(node).unwrap_or(own); - if materialization != Materialization::NotMaterialized { - assignment.set(©, materialization); - } - untimed.insert(Rc::as_ptr(node), Rc::clone(©)); - copy -} - -/// Every `SummaryAgg` built at ingestion time. -fn summaries_at_ingestion(node: &OperatorNode) -> Option { - matches!(node.asap(), Some(ASAPOp::SummaryAgg { .. })) - .then_some(Materialization::MatAtIngestion(Medium::Memory)) -} - -/// Every `SummaryMerge` built at query time and kept across evaluations. -fn merges_kept(node: &OperatorNode) -> Option { - matches!(node.asap(), Some(ASAPOp::SummaryMerge { .. })).then_some( - Materialization::MatAtQueryKeptAcrossEvaluations(Medium::Memory), - ) -} - -/// Exact PromQL `queries`, every minute, over 6 ingesting series of `m` -/// sampled every 10 s; `tail` is appended to each entry's JSON (e.g. its -/// range evaluation). -fn exact_workload(queries: &[&str], tail: &str) -> PlanningWorkload { - let entries: Vec = queries - .iter() - .map(|query| { - format!( - r#"{{"query": {query:?}, "demand": {{"fixed_interval": 60000}}, - "requirements": {{}}, - "predictability": {{"predictable": {{"known_at": null}}}}{tail}}}"# - ) - }) - .collect(); - serde_json::from_str(&format!( - r#"{{"query_workload": {{"language": "promql", "repeating_queries": [{}]}}, - "data_workload": {{"arrival": {{"ingesting": {{"rows_per_second": 0.6}}}}, - "sources": {{"m": {{"series_count": 6, "sample_interval": 10000}}}}}}}}"#, - entries.join(",") - )) - .unwrap() -} - -/// The plan file of `workload` on `capabilities`, with what `store` names -/// stored. -async fn forced_plan_json( - workload: &PlanningWorkload, - capabilities: &DeploymentCapabilities, - store: fn(&OperatorNode) -> Option, -) -> String { - let pass = StoredAsForced { store }; - let mut input = UserInput::new( - workload, - FrontendInput::Promql, - PlanningModels::builtin().with_deployment_capabilities(capabilities), - ); - input.pass = Some(&pass); - serde_json::to_string(&plan_installable(input).await.unwrap().0).unwrap() -} - -/// Samples of `m` for six series, `job` a (instances 1, 2, 3) and b (4, 5, -/// 6), every 10 s over the last 10 minutes; each value distinct. -fn job_samples() -> Vec { - let mut samples = Vec::new(); - for instance in 1..=6 { - for step in 0..60 { - samples.push(RawDataSample { - labels: BTreeMap::from([ - ("__name__".to_string(), "m".to_string()), - ( - "job".to_string(), - if instance <= 3 { "a" } else { "b" }.to_string(), - ), - ("instance".to_string(), instance.to_string()), - ]), - timestamp_ms: EVALUATION_MS - 595_000 + step * 10_000, - value: (instance * 1000 + (step * 37) % 101) as f64, - }); - } - } - samples -} - -/// The time selection `(from_ms, to_ms]` relative to the evaluation. -fn relative_time(from_ms: i64, to_ms: i64) -> Vec { - vec![SelectionBox::new( - Default::default(), - Some((Bound::Excluded(from_ms), Bound::Included(to_ms))), - )] -} - -/// A per-group maximum and a per-group sum (`max by (job)` and -/// `sum by (job, instance)`), both refreshed at ingestion from one stored -/// per-series sum: Stage 1 shares the per-series sums the two queries read -/// (over scans that project other labels). The per-series sum is listed -/// first, then each group output, which reads only it, with its time -/// selection under a definition of its own; refreshed in that order from -/// stored inputs, they answer as the query-time reference. (A coarse sum -/// in place of the maximum rolls up from the fine sum instead: -/// [`a_coarse_sum_rolls_up_from_the_stored_fine_sum`].) -#[tokio::test] -async fn group_outputs_from_one_stored_per_series_sum_are_listed_and_built() { - let workload = exact_workload( - &[ - "max by (job) (sum_over_time(m[1m]))", - "sum by (job, instance) (sum_over_time(m[1m]))", - ], - "", - ); - let plan_json = forced_plan_json( - &workload, - &DeploymentCapabilities::default(), - summaries_at_ingestion, - ) - .await; - let plan: InstallablePlan = serde_json::from_str(&plan_json).unwrap(); - let views = plan.materialized_views().unwrap(); - let [per_series, sums @ ..] = views.as_slice() else { - panic!("{:#?}", views); - }; - assert_eq!(sums.len(), 2, "{:#?}", views); - assert!(per_series.inputs.materialized_views.is_empty()); - assert_eq!(per_series.inputs.raw_data_windows().len(), 1); - for sum in sums { - assert_eq!(sum.inputs.materialized_views, [per_series.node_id]); - assert!(sum.inputs.raw_data_windows().is_empty()); - assert_ne!( - sum.coverage().definition_id(), - per_series.coverage().definition_id() - ); - } - assert_ne!( - sums[0].coverage().definition_id(), - sums[1].coverage().definition_id() - ); - for output in &views { - assert_eq!(output.refresh(), MaterializedViewRefresh::AtIngestion); - assert_eq!(output.coverage().selection(), relative_time(-60_000, 0)); - assert_eq!(output.retention_ms(), 60_000); - } - assert!(plan.raw_data_retention().unwrap().is_empty()); - let run = run_data_plane( - &plan_json, - &job_samples(), - EVALUATION_MS, - &mut SummaryStore::new(), - ); - assert_eq!(run.outputs_built, 3); - assert_eq!(run.answers_per_query[0].len(), 2, "one maximum per job"); - assert_eq!(run.answers_per_query[1].len(), 6, "one sum per series"); - assert_eq!(run.answers_per_query, run.reference_answers_per_query); -} - -/// A coarse and a fine sum (`sum by (job)` and `sum by (job, instance)`): -/// the fine grouping is priced at no more than the 6 series, as the -/// per-series sum is, so the coarse sum rolls up from it and the plan -/// stores two outputs, the second reading only the first. Built in that -/// order, they answer as the query-time reference. -#[tokio::test] -async fn a_coarse_sum_rolls_up_from_the_stored_fine_sum() { - let workload = exact_workload( - &[ - "sum by (job) (sum_over_time(m[1m]))", - "sum by (job, instance) (sum_over_time(m[1m]))", - ], - "", - ); - let plan_json = forced_plan_json( - &workload, - &DeploymentCapabilities::default(), - summaries_at_ingestion, - ) - .await; - let plan: InstallablePlan = serde_json::from_str(&plan_json).unwrap(); - let views = plan.materialized_views().unwrap(); - let [first, rolled_up] = views.as_slice() else { - panic!("{:#?}", views); - }; - assert!(first.inputs.materialized_views.is_empty()); - assert_eq!(rolled_up.inputs.materialized_views, [first.node_id]); - assert!(rolled_up.inputs.raw_data_windows().is_empty()); - let run = run_data_plane( - &plan_json, - &job_samples(), - EVALUATION_MS, - &mut SummaryStore::new(), - ); - assert_eq!(run.outputs_built, 2); - assert_eq!(run.answers_per_query[0].len(), 2, "one sum per job"); - assert_eq!(run.answers_per_query[1].len(), 6, "one sum per series"); - assert_eq!(run.answers_per_query, run.reference_answers_per_query); -} - -/// A maximum across series (`max by (job)`) refreshed at ingestion from the -/// stored per-series maxima, which the per-series query reads too: listed -/// after them with their time selection, under a definition of its own. -#[tokio::test] -async fn a_summary_across_series_from_per_series_summaries_is_listed_and_built() { - let workload = exact_workload( - &[ - "max by (job) (max_over_time(m[1m]))", - "max_over_time(m[1m])", - ], - "", - ); - let plan_json = forced_plan_json( - &workload, - &DeploymentCapabilities::default(), - summaries_at_ingestion, - ) - .await; - let plan: InstallablePlan = serde_json::from_str(&plan_json).unwrap(); - let views = plan.materialized_views().unwrap(); - let [per_series, across] = views.as_slice() else { - panic!("{:#?}", views); - }; - assert!(per_series.inputs.materialized_views.is_empty()); - assert_eq!(across.inputs.materialized_views, [per_series.node_id]); - assert!(across.inputs.raw_data_windows().is_empty()); - for output in [per_series, across] { - assert_eq!(output.refresh(), MaterializedViewRefresh::AtIngestion); - assert_eq!(output.coverage().selection(), relative_time(-60_000, 0)); - } - assert_ne!( - across.coverage().definition_id(), - per_series.coverage().definition_id() - ); - let run = run_data_plane( - &plan_json, - &job_samples(), - EVALUATION_MS, - &mut SummaryStore::new(), - ); - assert_eq!(run.outputs_built, 2); - assert_eq!(run.answers_per_query[0].len(), 2, "one maximum per job"); - assert_eq!(run.answers_per_query[1].len(), 6, "one maximum per series"); - assert_eq!(run.answers_per_query, run.reference_answers_per_query); -} - -/// A dashboard's 5-minute max over the last hour at 1-minute steps (#509 -/// Example 4's B3) keeps five 1-minute tumbling windows; with its merge kept too, the -/// merge is listed after the tumbling windows it reads, with their definition and the -/// union of their times, refreshed in the query run and kept for its window -/// plus the hour of steps; its tumbling windows are kept at least that long. Built -/// from the stored tumbling windows it answers as the reference, and a minute later -/// only the newest tumbling window and the new merge are refreshed. -#[tokio::test] -async fn a_kept_sliding_window_merge_is_listed_and_built() { - let capabilities = DeploymentCapabilities { - query_time_results_retained: true, - ..DeploymentCapabilities::default() - }; - let mut workload = exact_workload( - &["max_over_time(m[5m])"], - r#", "range_evaluation": {"step_ms": 60000, "span_ms": 3600000}"#, - ); - // Declared as sampled every second, so that keeping the tumbling windows is - // cheaper than rebuilding the window at every step; the samples the - // run reads are every 10 s, and exact answers do not depend on it. - let facts = workload - .data_workload - .as_mut() - .and_then(|data| data.sources.get_mut("m")) - .unwrap(); - facts.sample_interval = Some(asap_types::workload::DurationMs(1_000)); - let plan_json = forced_plan_json(&workload, &capabilities, merges_kept).await; - let plan: InstallablePlan = serde_json::from_str(&plan_json).unwrap(); - let views = plan.materialized_views().unwrap(); - let (merge, tumbling_windows) = views.split_last().unwrap(); - assert_eq!(tumbling_windows.len(), 5); - assert_eq!( - merge.inputs.materialized_views, - tumbling_windows - .iter() - .map(|tumbling_window| tumbling_window.node_id) - .collect::>() - ); - assert!(merge.inputs.raw_data_windows().is_empty()); - assert_eq!(merge.refresh(), MaterializedViewRefresh::OnFirstRead); - assert_eq!(merge.coverage().selection(), relative_time(-300_000, 0)); - assert_eq!(merge.retention_ms(), 300_000 + 3_600_000); - for (i, tumbling_window) in tumbling_windows.iter().enumerate() { - let i = i as i64; - assert_eq!( - tumbling_window.coverage().definition_id(), - merge.coverage().definition_id() - ); - assert_eq!( - tumbling_window.coverage().selection(), - relative_time(-(i + 1) * 60_000, -i * 60_000) - ); - assert!(tumbling_window.retention_ms() >= merge.retention_ms()); - } - let mut store = SummaryStore::new(); - let first = run_data_plane(&plan_json, &job_samples(), EVALUATION_MS, &mut store); - assert_eq!(first.outputs_built, 6); - assert_eq!(first.answers_per_query, first.reference_answers_per_query); - let next = run_data_plane( - &plan_json, - &job_samples(), - EVALUATION_MS + 60_000, - &mut store, - ); - assert_eq!( - next.outputs_built, 2, - "the newest tumbling window and the new merge" - ); - assert_eq!(next.answers_per_query, next.reference_answers_per_query); -} - -/// Example 4b's selected tumbling windows cover the hour and share one definition id, -/// each with its own time selection; each reads its tumbling window of -/// raw data samples and nothing stored. +/// Example 4b's selected tumbling windows cover the hour and share one definition id, +/// each with its own time selection; each reads its tumbling window of +/// raw data samples and nothing stored. #[tokio::test] async fn tumbling_windows_of_an_hour_share_one_definition_with_distinct_times() { let plan: InstallablePlan = @@ -1930,124 +387,6 @@ async fn tumbling_windows_of_an_hour_share_one_definition_with_distinct_times() } } -/// PromQL `topk` returns whole input series: `topk by (region) (2, -/// sum_over_time(data[1m]))` answers, per region, the two series with the -/// largest sums, each with its own full label set (region, service and -/// instance; `sum_over_time` drops the metric name), not only `region`. -/// Series `e1`..`e3` are in `east` with sums 6, 60 and 30; `w1`, `w2` in -/// `west` with sums 9 and 3. -#[tokio::test] -async fn topk_answers_carry_the_chosen_series_labels() { - let workload: PlanningWorkload = serde_json::from_str( - r#"{"query_workload": {"language": "promql", "repeating_queries": [{ - "query": "topk by (region) (2, sum_over_time(data[1m]))", - "demand": {"fixed_interval": 60000}, - "requirements": {}, - "predictability": {"predictable": {"known_at": null}}}]}, - "data_workload": {"arrival": {"ingesting": {"rows_per_second": 0.5}}, - "sources": {"data": {"series_count": 5, "sample_interval": 10000}}}}"#, - ) - .unwrap(); - let plan_json = plan_json_for(workload, &DeploymentCapabilities::default()).await; - let samples = topk_samples(); - let run = run_data_plane( - &plan_json, - &samples, - EVALUATION_MS, - &mut SummaryStore::new(), - ); - assert_full_series_answers(&run, 4); - let labels = |instance: &str, region: &str, service: &str| { - BTreeMap::from([ - ("instance".to_string(), instance.to_string()), - ("region".to_string(), region.to_string()), - ("service".to_string(), service.to_string()), - ]) - }; - let mut answered = run.answers_per_query[0].clone(); - answered.sort_by(|a, b| a.0.cmp(&b.0)); - let mut expected = vec![ - (labels("e2", "east", "web"), 60.0), - (labels("e3", "east", "db"), 30.0), - (labels("w1", "west", "api"), 9.0), - (labels("w2", "west", "web"), 3.0), - ]; - expected.sort_by(|a, b| a.0.cmp(&b.0)); - assert_eq!(answered.len(), expected.len(), "{answered:?}"); - for ((labels, value), (want_labels, want)) in answered.iter().zip(&expected) { - assert_eq!(labels, want_labels); - assert!((value - want).abs() < 1e-9, "{labels:?}: {value}"); - } - assert_eq!(run.answers_per_query, run.reference_answers_per_query); -} - -/// Samples of `data` for `topk_answers_carry_the_chosen_series_labels`: -/// three per series inside the last minute, each a third of its sum. -fn topk_samples() -> Vec { - let mut samples = Vec::new(); - for (instance, region, service, sum) in [ - ("e1", "east", "api", 6.0), - ("e2", "east", "web", 60.0), - ("e3", "east", "db", 30.0), - ("w1", "west", "api", 9.0), - ("w2", "west", "web", 3.0), - ] { - // In time order, as a counter's samples arrive. - for step in (1..=3).rev() { - let mut sample = sample("data", instance, EVALUATION_MS - step * 15_000, sum / 3.0); - sample.labels.insert("region".into(), region.into()); - sample.labels.insert("service".into(), service.into()); - samples.push(sample); - } - } - samples -} - -/// The first query's `expected` answers, each a whole input series (it -/// carries its `instance`), and the same as the query-time reference's. -fn assert_full_series_answers(run: &DataPlaneRun, expected: usize) { - let answered = &run.answers_per_query[0]; - assert_eq!(answered.len(), expected, "{answered:?}"); - for (labels, _) in answered { - assert!(labels.contains_key("instance"), "{labels:?}"); - assert!(labels.contains_key("service"), "{labels:?}"); - } - assert_eq!(run.answers_per_query, run.reference_answers_per_query); -} - -/// Every top-k and bottom-k form, grouped or not, over sums, counts and -/// rates, answers whole input series: `topk(2, ...)` the two of five, -/// `by (region)` per region, `bottomk` likewise. -#[tokio::test] -async fn every_topk_form_answers_whole_series() { - for (query, expected) in [ - ("topk(2, sum_over_time(data[1m]))", 2), - ("topk by (region) (1, count_over_time(data[1m]))", 2), - ("topk by (region) (1, rate(data[1m]))", 2), - ("bottomk by (region) (1, sum_over_time(data[1m]))", 2), - ("bottomk(3, sum_over_time(data[1m]))", 3), - ] { - let workload: PlanningWorkload = serde_json::from_str(&format!( - r#"{{"query_workload": {{"language": "promql", "repeating_queries": [{{ - "query": {query:?}, - "demand": {{"fixed_interval": 60000}}, - "requirements": {{}}, - "predictability": {{"predictable": {{"known_at": null}}}}}}]}}, - "data_workload": {{"arrival": {{"ingesting": {{"rows_per_second": 0.5}}}}, - "sources": {{"data": {{"series_count": 5, "sample_interval": 10000}}}}}}}}"# - )) - .unwrap(); - let plan_json = plan_json_for(workload, &DeploymentCapabilities::default()).await; - let run = run_data_plane( - &plan_json, - &topk_samples(), - EVALUATION_MS, - &mut SummaryStore::new(), - ); - assert_full_series_answers(&run, expected); - } -} - /// The MVP benchmark's p99 query: the per-series p99 of a gauge over 100 s, /// averaged per (region, service, job), every second, within 0.05 at /// failure probability 0.3, over 1,000 `data` series sampled every 100 ms, @@ -2059,469 +398,12 @@ fn p99_avg_workload(values_non_negative: bool) -> PlanningWorkload { "demand": {{"fixed_interval": 1000}}, "requirements": {{"accuracy": {{"max_error": 0.05, "failure_probability": 0.3, "accuracy_scope": "whole_result"}}}}, - "predictability": {{"predictable": {{"known_at": null}}}}}}]}}, - "data_workload": {{"arrival": {{"ingesting": {{"rows_per_second": 10000.0}}}}, - "sources": {{"data": {{"series_count": 1000, "metric_type": "gauge", - "sample_interval": 100, "values_non_negative": {values_non_negative}}}}}}}}}"# - )) - .unwrap() -} - -/// [`p99_avg_workload`] planned for a deployment that keeps no raw data samples. -async fn plan_p99_avg( - values_non_negative: bool, -) -> Result<(InstallablePlan, Option), String> { - let capabilities = DeploymentCapabilities { - raw_data_retained: false, - ..DeploymentCapabilities::default() - }; - plan_installable(UserInput::new( - &p99_avg_workload(values_non_negative), - FrontendInput::Promql, - PlanningModels::builtin().with_deployment_capabilities(&capabilities), - )) - .await - .map_err(|error| error.to_string()) -} - -/// Without raw data samples, the only valid plan of the p99 query keeps one -/// DDSketch per series in one-second tumbling windows built at ingestion and merges -/// the window's tumbling windows at query time. A quantile -/// of merged tumbling windows is still one of the samples, so a declared non-negative -/// gauge carries the sketch's relative bound through `avg`: that plan is -/// selected, its answers equal the query-time reference's and are within -/// 5% of the exact averages. -#[tokio::test] -async fn a_non_negative_gauges_merged_ddsketch_tumbling_windows_carry_the_relative_bound_through_avg( -) { - let (plan, selection) = plan_p99_avg(true).await.expect("a valid plan"); - let selection = selection.as_ref().expect("Stage 3's report"); - let views = plan.materialized_views().unwrap(); - assert!(selection.cost_per_second.is_some()); - assert!(!views.is_empty(), "tumbling windows built at ingestion"); - assert_eq!( - stored_definitions(&plan, "DDSketch").len(), - 1, - "one DDSketch" - ); - assert!( - plan.raw_data_retention().unwrap().is_empty(), - "no raw samples read" - ); - // The query's guarantee, with `data`'s declared sign and its 1,000 - // series bound as Stage 3 binds them: a relative error of at most 0.05 - // at failure probability at most 0.3 (DDSketch's bound is - // deterministic). - let root = plan.dag.roots[0]; - let guarantee = plan - .dag - .nodes - .iter() - .find(|node| node.id == root) - .and_then(|node| node.guarantee.clone()) - .expect("the query's guarantee"); - let sign = asap_accuracy::test_support::raw_data_samples_sign_statistic("data"); - let bound = guarantee.bind(&|statistic| match statistic { - "input_row_count" => Some(1_000.0), - _ => (statistic == sign).then_some(1.0), - }); - assert_eq!( - bound.spec.accuracy_metric, - asap_types::ir::properties::AccuracyMetric::Relative - ); - let error = bound.spec.error.evaluate().expect("a numeric bound"); - assert!(error <= 0.05, "{error}"); - let failure = bound - .spec - .failure_probability - .evaluate() - .expect("a numeric failure probability"); - assert!(failure <= 0.3, "{failure}"); - - // Eight non-negative series, one sample every 100 ms over the last - // 110 s: two per (region, service, job) group. - let evaluation_ms = EVALUATION_MS; - let mut samples = Vec::new(); - let mut per_group: BTreeMap<(String, String, String), Vec>> = BTreeMap::new(); - for series in 0..8 { - let region = format!("r{}", series % 2); - let service = format!("s{}", series / 4); - let job = "j".to_string(); - let mut window = Vec::new(); - for step in 0..1_100i64 { - let timestamp_ms = evaluation_ms - 110_000 + 50 + step * 100; - let value = 10.0 * (series + 1) as f64 + ((step * 37 + series * 11) % 997) as f64; - samples.push(RawDataSample { - labels: BTreeMap::from([ - ("__name__".to_string(), "data".to_string()), - ("region".to_string(), region.clone()), - ("service".to_string(), service.clone()), - ("job".to_string(), job.clone()), - ("instance".to_string(), format!("i{series}")), - ]), - timestamp_ms, - value, - }); - if timestamp_ms > evaluation_ms - 100_000 { - window.push(value); - } - } - per_group - .entry((region, service, job)) - .or_default() - .push(window); - } - let run = run_data_plane( - &serde_json::to_string(&plan).unwrap(), - &samples, - evaluation_ms, - &mut SummaryStore::new(), - ); - assert_eq!(run.answers_per_query, run.reference_answers_per_query); - let answers = &run.answers_per_query[0]; - assert_eq!(answers.len(), per_group.len()); - for (labels, value) in answers { - let key = ( - labels["region"].clone(), - labels["service"].clone(), - labels["job"].clone(), - ); - // The exact answer: the mean of the group's per-series p99s, each - // PromQL's interpolated quantile of the series' window. - let exact = per_group[&key] - .iter() - .map(|window| { - let mut sorted = window.clone(); - sorted.sort_by(f64::total_cmp); - let rank = 0.99 * (sorted.len() - 1) as f64; - let (low, high) = (rank.floor() as usize, rank.ceil() as usize); - sorted[low] + (sorted[high] - sorted[low]) * (rank - low as f64) - }) - .sum::() - / per_group[&key].len() as f64; - assert!( - (value - exact).abs() <= 0.05 * exact, - "{labels:?}: {value} vs exact {exact}" - ); - } -} - -/// Without a non-negative declaration, relative sketch error cannot propagate -/// through the average, so this no-raw deployment has no valid plan: plan -/// selection selects none. Pass 1 prunes the quantile sketches for that -/// reason (`NoErrorBoundForQueryTarget`) and Stage 2 the exact candidates, -/// which scan raw data at query time. -#[tokio::test] -async fn an_undeclared_gauges_merged_ddsketch_tumbling_windows_get_no_relative_bound_through_avg() { - let error = plan_p99_avg(false).await.expect_err("no valid plan"); - assert_plan_selection_selected_none(&error); - // Pass 1 prunes both quantile sketches for that reason: DDSketch's - // relative bound does not pass through the average of an undeclared - // gauge, and no rule carries KLL's rank bound through it. - let capabilities = DeploymentCapabilities { - raw_data_retained: false, - ..DeploymentCapabilities::default() - }; - let models = PlanningModels::builtin().with_deployment_capabilities(&capabilities); - let no_bound = pass1_pruned_for_no_error_bound(&p99_avg_workload(false), models); - assert!( - no_bound.contains(&"DDSketch".to_string()) && no_bound.contains(&"Kll".to_string()), - "{no_bound:?}" - ); - // Declared non-negative, DDSketch's bound passes through the average. - let no_bound = pass1_pruned_for_no_error_bound(&p99_avg_workload(true), models); - assert!(!no_bound.contains(&"DDSketch".to_string()), "{no_bound:?}"); -} - -/// The summaries Pass 1 pruned planning `workload` (PromQL) with `models` -/// because no query reading them can meet its target -/// ([`PruneReason::NoErrorBoundForQueryTarget`]), as the facade's planning -/// run reports them (`PlanningRun::per_computation_candidates`, its -/// `pruned_candidates`): each family's name (e.g. `"Kll"`). -fn pass1_pruned_for_no_error_bound( - workload: &PlanningWorkload, - models: PlanningModels<'_>, -) -> Vec { - let roots = asap_frontend_promql::lower_promql_query_workload(workload) - .unwrap() - .into_iter() - .map(|root| match root { - QueryRoot::Operator(node) => { - QueryRoot::Operator(with_promql_series_identity(&node).unwrap_or(node)) - } - scalar => scalar, - }) - .collect(); - let entries: Vec<_> = workload.query_workload.entries().collect(); - let data = workload.data_workload.clone().unwrap_or_default(); - let run = run_planning( - roots, - &entries, - &data, - models, - PlanSearchSettings { inspect: true }, - ) - .unwrap(); - run.per_computation_candidates - .expect("Pass 1's output") - .pruned_candidates - .iter() - .filter(|pruned| pruned.reason == PruneReason::NoErrorBoundForQueryTarget) - .filter_map(|pruned| match &pruned.candidate { - HowToComputeAggregateMeasure::Summary(family) => Some(format!("{family:?}")), - _ => None, - }) - .collect() -} - -/// `error` says plan selection found no valid plan: no candidate is valid. -/// Not a solver, realizer or input failure. -fn assert_plan_selection_selected_none(error: &str) { - assert!( - error.contains("plan selection: no valid candidate"), - "{error}" - ); -} - -/// `avg by (g) (quantile_over_time(0.99, x[100s]))` every 10 s at rank -/// error 0.01 and δ 0.01, over 100 ingesting `x` series sampled every -/// second: ASAPQuery's random mixed workload query. Stage 1 has no -/// analytical rule for an average of quantile estimates, so every sketch -/// candidate has no guarantee. -fn average_of_p99s_workload() -> PlanningWorkload { - serde_json::from_str( - r#"{"query_workload": {"language": "promql", "repeating_queries": [{ - "query": "avg by (g) (quantile_over_time(0.99, x[100s]))", - "demand": {"fixed_interval": 10000}, - "requirements": {"accuracy": {"accuracy_metric": "rank", "max_error": 0.01, - "failure_probability": 0.01, "accuracy_scope": "per_value"}}, - "predictability": {"predictable": {"known_at": null}}}]}, - "data_workload": {"arrival": {"ingesting": {"rows_per_second": 100.0}}, - "sources": {"x": {"series_count": 100, "sample_interval": 1000}}}}"#, - ) - .unwrap() -} - -/// Raw `x` samples of series `a` and `b` in group `g="1"` and `c` in group -/// `g="2"`, one every second from 200 s before `EVALUATION_MS` to it, off -/// the second boundaries: series `i`'s value is `100 × i + 1..=101`, all -/// positive, which DDSketch needs at run time (nothing declares it here). -fn grouped_samples() -> Vec { - let mut samples = Vec::new(); - for (i, (instance, g)) in [("a", "1"), ("b", "1"), ("c", "2")].iter().enumerate() { - for step in 0..200 { - samples.push(RawDataSample { - labels: BTreeMap::from([ - ("__name__".to_string(), "x".to_string()), - ("instance".to_string(), instance.to_string()), - ("g".to_string(), g.to_string()), - ]), - timestamp_ms: EVALUATION_MS - 199_500 + step * 1_000, - value: (100 * i) as f64 + 1.0 + ((step * 37 + 11 * i as i64) % 101) as f64, - }); - } - } - samples -} - -/// Quantile queries over one metric and window with different `by` labels -/// (ASAPQuery's seed-862 quantile queries) share one per-series DDSketch: -/// planned jointly on a deployment that keeps no raw data samples, every stored -/// DDSketch has one definition, and each query answers as its query-time -/// reference. Series of `data` carry `region`, `service` and `job`; values -/// are positive. -#[tokio::test] -async fn quantile_queries_with_different_groupings_share_one_per_series_sketch() { - let plan_json = jointly_planned( - &[ - "avg by (region) (quantile_over_time(0.99, data[100s]))", - "avg by (region, service, job) (quantile_over_time(0.9, data[100s]))", - "avg by (service) (quantile_over_time(0.5, data[100s]))", - ], - r#"{"accuracy": {"accuracy_metric": "relative", "max_error": 0.05, - "failure_probability": 0.01, "accuracy_scope": "per_value"}}"#, - ) - .await; - let plan: InstallablePlan = serde_json::from_str(&plan_json).unwrap(); - let sketches = stored_definitions(&plan, "DDSketch"); - assert_eq!(sketches.len(), 1, "{sketches:?}"); - let run = run_data_plane( - &plan_json, - &grouped_series_samples(), - EVALUATION_MS, - &mut SummaryStore::new(), - ); - assert_eq!(run.answers_per_query[0].len(), 2, "one answer per region"); - assert_eq!(run.answers_per_query[1].len(), 6); - assert_eq!(run.answers_per_query[2].len(), 3, "one answer per service"); - assert_eq!(run.answers_per_query, run.reference_answers_per_query); -} - -/// The six series of `data` the sharing tests read: `region` east/west, -/// `service` api/web/db, `job` batch/online; values positive, one every -/// 10 s over the last two minutes, in time order. -fn grouped_series_samples() -> Vec { - let mut samples = Vec::new(); - for (instance, region, service, job) in [ - ("s1", "east", "api", "online"), - ("s2", "east", "web", "online"), - ("s3", "east", "db", "batch"), - ("s4", "west", "api", "online"), - ("s5", "west", "web", "batch"), - ("s6", "west", "db", "batch"), - ] { - let base: f64 = instance[1..].parse().unwrap(); - for step in (0..12).rev() { - let mut sample = sample( - "data", - instance, - EVALUATION_MS - 5_000 - step * 10_000, - base * 10.0 + ((step * 37) % 11) as f64, - ); - sample.labels.insert("region".into(), region.into()); - sample.labels.insert("service".into(), service.into()); - sample.labels.insert("job".into(), job.into()); - samples.push(sample); - } - } - samples -} - -/// The exact answer of `avg by (g) (quantile_over_time(0.99, x[100s]))` at -/// `EVALUATION_MS` over [`grouped_samples`]: PromQL's p99 (linear -/// interpolation at rank `0.99 × (n − 1)`) of each series' samples in -/// `(EVALUATION_MS − 100 s, EVALUATION_MS]`, averaged per group. -fn exact_average_of_p99s(samples: &[RawDataSample]) -> BTreeMap { - let mut per_series: BTreeMap<(String, String), Vec> = BTreeMap::new(); - for s in samples - .iter() - .filter(|s| EVALUATION_MS - 100_000 < s.timestamp_ms && s.timestamp_ms <= EVALUATION_MS) - { - per_series - .entry((s.labels["g"].clone(), s.labels["instance"].clone())) - .or_default() - .push(s.value); - } - let mut per_group: BTreeMap> = BTreeMap::new(); - for ((g, _), mut values) in per_series { - values.sort_by(f64::total_cmp); - let rank = 0.99 * (values.len() - 1) as f64; - let (low, high) = (rank.floor() as usize, rank.ceil() as usize); - let p99 = values[low] + (values[high] - values[low]) * (rank - low as f64); - per_group.entry(g).or_default().push(p99); - } - per_group - .into_iter() - .map(|(g, p99s)| (g, p99s.iter().sum::() / p99s.len() as f64)) - .collect() -} - -/// Issue #774: without raw data samples retained, every candidate of an average -/// of p99s fails a check (the exact ones scan raw data samples at query time, -/// the sketch ones have no guarantee), so planning fails. With -/// `PlanningModels::ignore_accuracy_guarantees`, the cheapest -/// sketch plan is selected, the report marks its output as accepted without -/// a guarantee, and it runs: equal to its query-time reference and within a -/// loose tolerance of the exact answer. -#[tokio::test] -async fn ignoring_accuracy_guarantees_selects_a_sketch_plan_that_runs() { - let workload = average_of_p99s_workload(); - let capabilities = DeploymentCapabilities { - raw_data_retained: false, - ..DeploymentCapabilities::default() - }; - let plan = |ignore: bool| { - let workload = workload.clone(); - let capabilities = capabilities.clone(); - async move { - plan_installable(UserInput::new( - &workload, - FrontendInput::Promql, - PlanningModels::builtin() - .with_deployment_capabilities(&capabilities) - .with_ignore_accuracy_guarantees(ignore), - )) - .await - } - }; - // Without the setting, no candidate is valid. - let error = plan(false) - .await - .expect_err("no candidate is valid without the setting"); - assert_plan_selection_selected_none(&error.to_string()); - // Pass 1 prunes every quantile sketch for it: no rule carries a rank - // bound through the average, and a relative one does not meet a rank - // target. With the setting, Stage 3 may accept an output without its - // guarantee, so Pass 1 prunes none. - let models = PlanningModels::builtin().with_deployment_capabilities(&capabilities); - let no_bound = pass1_pruned_for_no_error_bound(&workload, models); - assert!( - no_bound.contains(&"DDSketch".to_string()) && no_bound.contains(&"Kll".to_string()), - "{no_bound:?}" - ); - assert_eq!( - pass1_pruned_for_no_error_bound(&workload, models.with_ignore_accuracy_guarantees(true)), - Vec::::new() - ); - // With it. - let (installed, report) = plan(true).await.expect("a sketch plan is selected"); - // A per-series KLL or DDSketch plan. - let families: Vec = installed - .dag - .nodes - .iter() - .filter_map(|node| match &node.payload { - asap_executor::planner::ir::physical_export::PhysicalASAPOperatorPayload::ASAP( - ASAPOp::SummaryAgg { family, .. } - | ASAPOp::SummaryTumblingWindowChain { family, .. }, - ) => Some(format!("{family:?}")), - _ => None, - }) - .collect(); - // Its sketches (beside the average's exact sum and count) are KLL or - // DDSketch. - let sketches: Vec<&String> = families - .iter() - .filter(|f| f.starts_with("Sketch")) - .collect(); - assert!( - !sketches.is_empty() - && sketches - .iter() - .all(|f| f.contains("Kll") || f.contains("DDSketch")), - "{families:?}" - ); - // The report marks the query's output as accepted without a guarantee. - let report = report.as_ref().expect("Stage 3's report"); - assert_eq!(report.queries_accepted_without_guarantee, vec![0]); - // It runs, equal to its query-time reference ... - let plan_json = serde_json::to_string(&installed).unwrap(); - let samples = grouped_samples(); - let run = run_data_plane( - &plan_json, - &samples, - EVALUATION_MS, - &mut SummaryStore::new(), - ); - assert_eq!(run.answers_per_query, run.reference_answers_per_query); - // ... and within 5 (of values spanning 100 per series) of the exact - // average of p99s, per group. - let exact = exact_average_of_p99s(&samples); - let answered: BTreeMap = run.answers_per_query[0] - .iter() - .map(|(labels, value)| (labels["g"].clone(), *value)) - .collect(); - assert_eq!( - answered.keys().collect::>(), - exact.keys().collect::>() - ); - for (g, value) in &answered { - assert!( - (value - exact[g]).abs() <= 5.0, - "g={g}: {value} vs exact {}", - exact[g] - ); - } + "predictability": {{"predictable": {{"known_at": null}}}}}}]}}, + "data_workload": {{"arrival": {{"ingesting": {{"rows_per_second": 10000.0}}}}, + "sources": {{"data": {{"series_count": 1000, "metric_type": "gauge", + "sample_interval": 100, "values_non_negative": {values_non_negative}}}}}}}}}"# + )) + .unwrap() } /// The MVP benchmark's p99 query over an undeclared gauge, which has no @@ -2553,184 +435,6 @@ async fn ignoring_accuracy_guarantees_plans_the_undeclared_p99_query() { ); } -/// [`p99_avg_workload`]'s query with `outer` in place of `avg`, planned -/// for a deployment that keeps no raw data samples. -async fn plan_p99( - outer: &str, - values_non_negative: bool, - ignore_accuracy: bool, -) -> (InstallablePlan, Option) { - let mut json = serde_json::to_value(p99_avg_workload(values_non_negative)).unwrap(); - json["query_workload"]["repeating_queries"][0]["query"] = - format!("{outer} by (region, service, job) (quantile_over_time(0.99, data[100s]))").into(); - let workload: PlanningWorkload = serde_json::from_value(json).unwrap(); - let capabilities = DeploymentCapabilities { - raw_data_retained: false, - // A DDSketch kernel fixture, not a comparison of sketch families. - supported_summaries: ignore_accuracy - .then(|| only_summary_family(SketchAlgorithm::DDSketch)), - ..DeploymentCapabilities::default() - }; - let settings = CostInputs::default(); - let models = PlanningModels::builtin() - .with_deployment_capabilities(&capabilities) - .with_cost_inputs(&settings) - .with_ignore_accuracy_guarantees(ignore_accuracy); - plan_installable(UserInput::new(&workload, FrontendInput::Promql, models)) - .await - .expect("a valid plan") -} - -/// `data` samples of series `0..series`, `value(series, step)` every -/// `interval_ms` over the `steps` steps ending just before `EVALUATION_MS`, -/// two series per (region, service, job) group; and each group's series' -/// windows `(EVALUATION_MS - 100 s, EVALUATION_MS]`. -#[allow(clippy::type_complexity)] -fn data_samples( - series: usize, - steps: i64, - interval_ms: i64, - value: impl Fn(usize, i64) -> f64, -) -> ( - Vec, - BTreeMap<(String, String, String), Vec>>, -) { - let mut samples = Vec::new(); - let mut per_group: BTreeMap<(String, String, String), Vec>> = BTreeMap::new(); - for s in 0..series { - let (region, service, job) = ( - format!("r{}", s % 2), - format!("s{}", s / 4), - "j".to_string(), - ); - let mut window = Vec::new(); - for step in 0..steps { - let timestamp_ms = EVALUATION_MS - steps * interval_ms + 50 + step * interval_ms; - let v = value(s, step); - samples.push(RawDataSample { - labels: BTreeMap::from([ - ("__name__".to_string(), "data".to_string()), - ("region".to_string(), region.clone()), - ("service".to_string(), service.clone()), - ("job".to_string(), job.clone()), - ("instance".to_string(), format!("i{s}")), - ]), - timestamp_ms, - value: v, - }); - if timestamp_ms > EVALUATION_MS - 100_000 { - window.push(v); - } - } - per_group - .entry((region, service, job)) - .or_default() - .push(window); - } - (samples, per_group) -} - -/// PromQL's interpolated `quantile_over_time(q, ...)` of `window`. -fn interpolated_quantile(q: f64, window: &[f64]) -> f64 { - let mut sorted = window.to_vec(); - sorted.sort_by(f64::total_cmp); - let rank = q * (sorted.len() - 1) as f64; - let (low, high) = (rank.floor() as usize, rank.ceil() as usize); - sorted[low] + (sorted[high] - sorted[low]) * (rank - low as f64) -} - -/// Runs `plan` over `samples` and checks every group's answer against -/// `combine` of its series' exact p99s, within the relative error 0.05, and -/// against the query-time reference. -fn assert_p99s_within_alpha( - plan: &InstallablePlan, - samples: &[RawDataSample], - per_group: &BTreeMap<(String, String, String), Vec>>, - combine: impl Fn(&[f64]) -> f64, -) { - let run = run_data_plane( - &serde_json::to_string(plan).unwrap(), - samples, - EVALUATION_MS, - &mut SummaryStore::new(), - ); - assert_eq!(run.answers_per_query, run.reference_answers_per_query); - let answers = &run.answers_per_query[0]; - assert_eq!(answers.len(), per_group.len()); - for (labels, value) in answers { - let key = ( - labels["region"].clone(), - labels["service"].clone(), - labels["job"].clone(), - ); - let p99s: Vec = per_group[&key] - .iter() - .map(|window| interpolated_quantile(0.99, window)) - .collect(); - let exact = combine(&p99s); - assert!( - (value - exact).abs() <= 0.05 * exact.abs(), - "{labels:?}: {value} vs exact {exact}" - ); - } -} - -/// A non-negative gauge that reports exactly 0 is still summarized by the -/// selected DDSketch tumbling windows (zero has its own bucket): each series' window -/// holds the samples 0, 0, 3 and 5 scaled by the series, and every answer -/// equals the query-time reference's and is within the relative error 0.05 -/// of the exact average of PromQL's interpolated p99s. -#[tokio::test] -async fn a_ddsketch_over_a_gauge_reporting_zero_runs_within_alpha() { - let (plan, _) = plan_p99("avg", true, false).await; - let views = plan.materialized_views().unwrap(); - assert!(!views.is_empty(), "tumbling windows built at ingestion"); - let (samples, per_group) = data_samples(8, 4, 20_000, |s, step| { - (s + 1) as f64 * [0.0, 0.0, 3.0, 5.0][step as usize] - }); - assert!(per_group.values().flatten().all(|window| window.len() == 4)); - let average = |p99s: &[f64]| p99s.iter().sum::() / p99s.len() as f64; - assert_p99s_within_alpha(&plan, &samples, &per_group, average); - // Many samples, a third of them zero. - let (samples, per_group) = data_samples(8, 1_100, 100, |s, step| { - (s + 1) as f64 * [0.0, 3.0, 5.0][(step % 3) as usize] - }); - assert_p99s_within_alpha(&plan, &samples, &per_group, average); -} - -/// Signed samples (negative, zero and positive) are summarized by the -/// selected DDSketch tumbling windows, through their negative store and zero bucket: -/// Signed interpolation has no general relative guarantee, so this kernel -/// fixture explicitly bypasses accuracy admission. Every answer equals the -/// query-time reference's and is -/// within 0.05 of the exact value, including series whose samples are all -/// negative. -#[tokio::test] -async fn a_ddsketch_over_signed_samples_runs_within_alpha() { - let (plan, report) = plan_p99("max", false, true).await; - let views = plan.materialized_views().unwrap(); - assert!( - !stored_definitions(&plan, "DDSketch").is_empty(), - "exercise signed DDS state" - ); - assert_eq!( - report.as_ref().unwrap().queries_accepted_without_guarantee, - vec![0] - ); - assert!(!views.is_empty(), "tumbling windows built at ingestion"); - let max = |p99s: &[f64]| p99s.iter().copied().fold(f64::NEG_INFINITY, f64::max); - // Series 0..4 cross zero; series 4..8 are all negative. - let (samples, per_group) = data_samples(8, 1_100, 100, |s, step| { - let scale = (s + 1) as f64; - if s < 4 { - scale * [-7.0, -3.0, 0.0, 2.0, 4.0][(step % 5) as usize] - } else { - scale * [-9.0, -8.0, -5.0, -1.0][(step % 4) as usize] - } - }); - assert_p99s_within_alpha(&plan, &samples, &per_group, max); -} - /// The ASAPQuery workloads' counter top-k query `topk(10, sum by (job) /// ((data_total[5m])))` for `f` `increase` or `rate`, every minute /// within 0.05 at failure probability 0.3, over 24 `data_total` counter @@ -2750,111 +454,6 @@ fn counter_topk_workload(function: &str) -> PlanningWorkload { .unwrap() } -/// The per-second rate of series `instance` (1 or 2) of job `job` (1..=12) -/// in [`counter_raw_data_samples`]: distinct for every series and every job's sum. -fn counter_rate(job: i64, instance: i64) -> f64 { - job as f64 + 0.1 * instance as f64 -} - -/// `data_total` samples of 12 jobs with 2 series each, every 10 s from -/// 595 s before `EVALUATION_MS`: each a counter growing linearly at -/// [`counter_rate`], so its 5-minute `increase` is 300 × its rate (the -/// samples 5 s inside each window end extrapolate to them exactly). -fn counter_raw_data_samples() -> Vec { - let mut samples = Vec::new(); - for job in 1..=12 { - for instance in 1..=2 { - for step in 0..60 { - let mut s = sample( - "data_total", - &format!("{job}-{instance}"), - EVALUATION_MS - 595_000 + step * 10_000, - counter_rate(job, instance) * (5 + step * 10) as f64, - ); - s.labels.insert("job".into(), format!("j{job:02}")); - samples.push(s); - } - } - } - samples -} - -/// Issue: with guarantees ignored and no raw data samples, the counter top-k -/// queries selected exact per-series tumbling windows followed by a query-time -/// CountSketchWithHeap top-k the executor cannot compile ("invalid keyed -/// evaluation shape or capacity"). The selected plan must be one the -/// executor compiles; it runs, equal to its query-time reference and to the -/// exact top 10 jobs (jobs 3..=12, each the sum of its two series). -#[tokio::test] -async fn ignoring_guarantees_counter_topk_plans_compile_and_answer_exactly() { - let capabilities = DeploymentCapabilities { - raw_data_retained: false, - ..DeploymentCapabilities::default() - }; - let settings = CostInputs::default(); - for (function, seconds) in [("increase", 300.0), ("rate", 1.0)] { - let plan = plan_installable(UserInput::new( - &counter_topk_workload(function), - FrontendInput::Promql, - PlanningModels::builtin() - .with_deployment_capabilities(&capabilities) - .with_cost_inputs(&settings) - .with_ignore_accuracy_guarantees(true), - )) - .await - .unwrap_or_else(|error| panic!("{function}: {error}")) - .0; - // An exact plan: no heap over the per-job sums, which have one row - // per job. - assert!( - !plan.dag.nodes.iter().any(|node| matches!( - &node.payload, - asap_executor::planner::ir::physical_export::PhysicalASAPOperatorPayload::ASAP( - ASAPOp::SummaryAgg { - family: asap_executor::planner::ir::schema::FieldDataType::Sketch(..), - .. - } - ) - )), - "{function}" - ); - // The plan compiles as a runtime compiles it. - plan.check_compiles().unwrap(); - let plan_json = serde_json::to_string(&plan).unwrap(); - let run = run_data_plane( - &plan_json, - &counter_raw_data_samples(), - EVALUATION_MS, - &mut SummaryStore::new(), - ); - assert_eq!( - run.answers_per_query, run.reference_answers_per_query, - "{function}" - ); - let expected: Vec<(String, f64)> = (3..=12) - .map(|job| { - ( - format!("j{job:02}"), - (counter_rate(job, 1) + counter_rate(job, 2)) * seconds, - ) - }) - .collect(); - let mut answered: Vec<(String, f64)> = run.answers_per_query[0] - .iter() - .map(|(labels, value)| (labels["job"].clone(), *value)) - .collect(); - answered.sort_by(|a, b| a.0.cmp(&b.0)); - assert_eq!(answered.len(), expected.len(), "{function}: {answered:?}"); - for ((job, value), (want_job, want)) in answered.iter().zip(&expected) { - assert_eq!(job, want_job, "{function}"); - assert!( - (value - want).abs() <= 1e-9 * want, - "{function} {job}: {value} vs {want}" - ); - } - } -} - /// `InstallablePlan::check_compiles` reports what the Planner's executor /// refuses: the counter top-k plan with its answer column's type changed /// from Float64 to Utf8 does not compile, with the executor's reason. @@ -2895,897 +494,6 @@ async fn check_compiles_reports_the_executors_error() { ); } -/// `queries` every 10 s at `accuracy` (JSON) over six ingesting series of -/// `data`, a non-negative gauge, planned jointly on a deployment that keeps -/// no raw data samples: the plan file. -async fn jointly_planned(queries: &[&str], accuracy: &str) -> String { - let entries: Vec = queries - .iter() - .map(|query| { - format!( - r#"{{"query": {query:?}, "demand": {{"fixed_interval": 10000}}, - "requirements": {accuracy}, - "predictability": {{"predictable": {{"known_at": null}}}}}}"# - ) - }) - .collect(); - let workload: PlanningWorkload = serde_json::from_str(&format!( - r#"{{"query_workload": {{"language": "promql", "repeating_queries": [{}]}}, - "data_workload": {{"arrival": {{"ingesting": {{"rows_per_second": 0.6}}}}, - "sources": {{"data": {{"series_count": 6, "sample_interval": 10000, - "metric_type": "gauge", "values_non_negative": true}}}}}}}}"#, - entries.join(",") - )) - .unwrap(); - let capabilities = DeploymentCapabilities { - raw_data_retained: false, - ..DeploymentCapabilities::default() - }; - plan_json_for(workload, &capabilities).await -} - -/// The definition ids of `plan`'s materialized views whose node's payload -/// mentions `family` (e.g. `"DDSketch"`, `"Sum"`). -fn stored_definitions(plan: &InstallablePlan, family: &str) -> BTreeSet { - plan.materialized_views() - .unwrap() - .iter() - .filter(|view| { - let node = plan - .dag - .nodes - .iter() - .find(|n| n.id == view.node_id) - .unwrap(); - serde_json::to_string(&node.payload) - .unwrap() - .contains(family) - }) - .map(|view| view.coverage().definition_id().to_owned()) - .collect() -} - -/// A fine and a coarse sum over one metric and window (ASAPQuery's -/// sum-fine and sum-coarse) store one Sum: both group sums are computed -/// from one stored per-series Sum (Stage 1 shares the per-series sums the -/// two queries' scans read), so every stored Sum has one definition, and -/// both answer as the query-time reference. -#[tokio::test] -async fn fine_and_coarse_sums_store_one_sum() { - let plan_json = jointly_planned( - &[ - "sum by (region, service, job) (sum_over_time(data[100s]))", - "sum by (region) (sum_over_time(data[100s]))", - ], - "{}", - ) - .await; - let plan: InstallablePlan = serde_json::from_str(&plan_json).unwrap(); - let sums = stored_definitions(&plan, "\"Sum\""); - assert_eq!(sums.len(), 1, "{sums:?}"); - let run = run_data_plane( - &plan_json, - &grouped_series_samples(), - EVALUATION_MS, - &mut SummaryStore::new(), - ); - assert_eq!(run.answers_per_query[0].len(), 6); - assert_eq!(run.answers_per_query[1].len(), 2, "one sum per region"); - assert_eq!(run.answers_per_query, run.reference_answers_per_query); -} - -/// Per-series answers planned beside a grouping of the same per-series -/// values (over a scan with other labels) answer as their query-time -/// reference, and the two share one stored per-series state: a top-k of -/// per-series sums keeps each chosen series' labels, and per-series -/// DDSketch quantiles keep every series' labels. -#[tokio::test] -async fn per_series_answers_beside_a_grouping_share_one_state_and_answer_as_the_reference() { - for (queries, family, accuracy, answers) in [ - ( - [ - "topk(2, sum_over_time(data[100s]))", - "sum by (region) (sum_over_time(data[100s]))", - ], - "\"Sum\"", - "{}", - 2, - ), - ( - [ - "quantile_over_time(0.9, data[100s])", - "avg by (region) (quantile_over_time(0.99, data[100s]))", - ], - "DDSketch", - r#"{"accuracy": {"accuracy_metric": "relative", "max_error": 0.05, - "failure_probability": 0.01, "accuracy_scope": "per_value"}}"#, - 6, - ), - ] { - let plan_json = jointly_planned(&queries, accuracy).await; - let plan: InstallablePlan = serde_json::from_str(&plan_json).unwrap(); - assert_eq!(stored_definitions(&plan, family).len(), 1, "{queries:?}"); - let run = run_data_plane( - &plan_json, - &grouped_series_samples(), - EVALUATION_MS, - &mut SummaryStore::new(), - ); - assert_eq!(run.answers_per_query[0].len(), answers, "{queries:?}"); - assert_eq!(run.answers_per_query[1].len(), 2, "{queries:?}: per region"); - assert_eq!( - run.answers_per_query, run.reference_answers_per_query, - "{queries:?}" - ); - } -} - -/// A heap top-k query `query` (the ASAPQuery workloads' top-k-sum or -/// counter top-k shape) every 10 s, within 0.05 at failure probability 0.3, -/// over 10,000 series of `metric` sampled every 100 ms, with `facts` (its -/// type and sign, as JSON fields). -fn heap_topk_workload(query: &str, metric: &str, facts: &str) -> PlanningWorkload { - serde_json::from_str(&format!( - r#"{{"query_workload": {{"language": "promql", "repeating_queries": [{{ - "query": {query:?}, - "demand": {{"fixed_interval": 10000}}, - "requirements": {{"accuracy": {{"max_error": 0.05, "failure_probability": 0.3}}}}, - "predictability": {{"predictable": {{"known_at": null}}}}}}]}}, - "data_workload": {{"arrival": {{"ingesting": {{"rows_per_second": 100000.0}}}}, - "sources": {{"{metric}": {{"series_count": 10000, "sample_interval": 100, - {facts}}}}}}}}}"# - )) - .unwrap() -} - -/// The executor's support for `family` alone: a deployment whose only -/// sketch family is `family` (exact plans stay allowed). -fn only_summary_family(family: SketchAlgorithm) -> Vec { - asap_executor::supported_summary_families() - .into_iter() - .filter(|supported| supported.family == family) - .collect() -} - -/// A deployment's accuracy model that accepts, whatever its guarantee, only -/// a query output that reads a heap of this algorithm, and no other output: -/// it forces the cheapest plan with that heap, as a deployment that requires -/// the heap would. -struct HeapPlansOnly(SketchAlgorithm); - -impl asap_accuracy::AccuracyModel for HeapPlansOnly { - fn satisfies( - &self, - _guarantee: &asap_types::ir::properties::AccuracyGuarantee, - _target: &asap_types::ir::properties::AccuracyTarget, - ) -> bool { - false - } - - fn accepts_output_regardless_of_guarantee( - &self, - output: &QueryRoot, - _target: &asap_types::ir::properties::AccuracyTarget, - ) -> bool { - let QueryRoot::Operator(root) = output else { - return false; - }; - OperatorNode::reachable(root).iter().any(|node| { - matches!(node.asap(), Some(ASAPOp::SummaryAgg { family: FieldDataType::Sketch(kind, _), .. } - | ASAPOp::SummaryTumblingWindowChain { family: FieldDataType::Sketch(kind, _), .. }) - if *kind.algorithm() == self.0) - }) - } -} - -/// `workload` planned with accuracy guarantees ignored (a top-k estimate -/// has none) on a deployment that keeps no raw samples; with `heap`, the -/// deployment's accuracy model accepts only outputs that read a heap of that -/// algorithm ([`HeapPlansOnly`]). -async fn plan_ignoring_guarantees( - workload: &PlanningWorkload, - heap: Option, -) -> Result { - let capabilities = DeploymentCapabilities { - raw_data_retained: false, - ..DeploymentCapabilities::default() - }; - // Ignoring guarantees replaces the accuracy model, so a forced heap goes - // through its own accuracy model instead. - let settings = CostInputs::default(); - let heap_only = heap.map(HeapPlansOnly); - let models = PlanningModels::builtin() - .with_deployment_capabilities(&capabilities) - .with_cost_inputs(&settings) - .with_ignore_accuracy_guarantees(heap.is_none()); - let models = match &heap_only { - Some(model) => models.with_accuracy(model), - None => models, - }; - plan_installable(UserInput::new(workload, FrontendInput::Promql, models)) - .await - .map(|(plan, _)| plan) - .map_err(|e| e.to_string()) -} - -/// The heap sketch `plan` builds: its algorithm and width. -fn heap_of(plan: &InstallablePlan) -> Option<(SketchAlgorithm, u32)> { - plan.dag.nodes.iter().find_map(|node| match &node.payload { - asap_executor::planner::ir::physical_export::PhysicalASAPOperatorPayload::ASAP( - ASAPOp::SummaryAgg { family, .. } | ASAPOp::SummaryTumblingWindowChain { family, .. }, - ) => match family { - FieldDataType::Sketch(kind, _) => match kind.params() { - SketchParams::CmsWithHeap { width, .. } - | SketchParams::CountSketchWithHeap { width, .. } => { - Some((*kind.algorithm(), *width)) - } - _ => None, - }, - _ => None, - }, - _ => None, - }) -} - -/// The heap of `algorithm`'s point-error bound on an item of a partition -/// whose items' true values are `values`: `ε·L1` for Count-Min (ε = e/w, -/// an overestimate), `ε·L2` for Count Sketch (ε = √(3/w)), which is at most -/// `ε·L1`. -fn heap_bound(algorithm: &SketchAlgorithm, width: u32, values: &[f64]) -> f64 { - let width = f64::from(width); - match algorithm { - SketchAlgorithm::CmsWithHeap => { - std::f64::consts::E / width * values.iter().map(|v| v.abs()).sum::() - } - SketchAlgorithm::CountSketchWithHeap => { - (3.0 / width).sqrt() * values.iter().map(|v| v * v).sum::().sqrt() - } - other => panic!("not a heap: {other:?}"), - } -} - -/// Checks a heap top-k plan's `answers` against the `truth` (each item's -/// exact value, by partition): per partition, `k` answers, the true top `k` -/// recalled, each item named by its labels (`item_label`, and `partition` -/// when partitioned), and each estimate within the heap's bound. -fn assert_heap_topk_answers( - answers: &[Answer], - truth: &BTreeMap>, - (partition, item_label, k): (Option<&str>, &str, usize), - (algorithm, width): (&SketchAlgorithm, u32), -) { - let mut answered: BTreeMap> = BTreeMap::new(); - for (labels, value) in answers { - // A range function's result drops the metric name, as the exact - // top-k's does. - assert!(!labels.contains_key("__name__"), "{labels:?}"); - let group = partition.map_or(String::new(), |p| labels[p].clone()); - let item = labels - .get(item_label) - .unwrap_or_else(|| panic!("{item_label} kept: {labels:?}")); - answered - .entry(group) - .or_default() - .push((item.clone(), *value)); - } - assert_eq!( - answered.keys().collect::>(), - truth.keys().collect::>() - ); - for (group, items) in &answered { - let exact = &truth[group]; - let bound = heap_bound( - algorithm, - width, - &exact.values().copied().collect::>(), - ); - let mut ranked: Vec<(&String, &f64)> = exact.iter().collect(); - ranked.sort_by(|a, b| b.1.total_cmp(a.1)); - let top: BTreeSet<&String> = ranked.iter().take(k).map(|(item, _)| *item).collect(); - assert_eq!(items.len(), k, "{group}: {items:?}"); - let recalled = items.iter().filter(|(item, _)| top.contains(item)).count(); - assert_eq!(recalled, k, "{group}: recall of {top:?} in {items:?}"); - for (item, estimate) in items { - let error = estimate - exact[item]; - assert!( - error.abs() <= bound + 1e-9 * exact[item].abs(), - "{group}/{item}: {estimate} vs exact {}, bound {bound}", - exact[item] - ); - if *algorithm == SketchAlgorithm::CmsWithHeap { - assert!( - error >= -1e-9 * exact[item].abs(), - "Count-Min overestimates" - ); - } - } - } -} - -/// `data` samples of 2 regions with 40 series each, six inside the last -/// minute per series: series `r` (rank 1..=40) of a region sums to -/// `1000 / r^1.1` (west at half that), a skewed ranking. -fn skewed_series_samples() -> (Vec, BTreeMap>) { - let mut samples = Vec::new(); - let mut truth: BTreeMap> = BTreeMap::new(); - for (region, scale) in [("east", 1.0), ("west", 0.5)] { - for rank in 1..=40 { - let instance = format!("{region}-{rank:02}"); - let sum = scale * 1000.0 / f64::from(rank).powf(1.1); - for step in 0..6 { - let mut s = sample( - "data", - &instance, - EVALUATION_MS - 55_000 + step * 10_000, - sum / 6.0, - ); - s.labels.insert("region".into(), region.into()); - samples.push(s); - } - truth - .entry(region.to_string()) - .or_default() - .insert(instance, sum); - } - } - (samples, truth) -} - -/// `workload`'s plan with only `algorithm`'s heap plans accepted -/// ([`HeapPlansOnly`] forces the heap; guarantees are not ignored): it builds that heap, compiles as a runtime compiles it, and -/// both it and its query-time reference answer `samples` with the true top -/// `k` per partition, each named by its labels and within the heap's bound -/// of `truth` ([`assert_heap_topk_answers`]). -async fn assert_forced_heap_answers( - workload: &PlanningWorkload, - algorithm: SketchAlgorithm, - (samples, truth): (&[RawDataSample], &BTreeMap>), - labels: (Option<&str>, &str, usize), -) { - let plan = plan_ignoring_guarantees(workload, Some(algorithm)) - .await - .unwrap_or_else(|error| panic!("{algorithm:?}: {error}")); - let (built, width) = heap_of(&plan).expect("a heap plan"); - assert_eq!(built, algorithm); - plan.check_compiles().unwrap(); - let plan_json = serde_json::to_string(&plan).unwrap(); - let run = run_data_plane(&plan_json, samples, EVALUATION_MS, &mut SummaryStore::new()); - for answers in [ - &run.answers_per_query[0], - &run.reference_answers_per_query[0], - ] { - assert_heap_topk_answers(answers, truth, labels, (&algorithm, width)); - } -} - -/// The ASAPQuery workloads' top-k-sum shape over a gauge, -/// `topk by (region) (3, sum_over_time(data[1m]))`: the fused -/// whole-expression Count Sketch heap (one per region over the raw -/// samples, ranking series identities), forced with guarantees ignored, -/// compiles, and answers each region's true top 3 series, with their full -/// labels, within its bound of their exact sums. (A Count-Min heap needs -/// the weights proven non-negative.) -#[tokio::test] -#[ignore = "fails on main too: Stage 3 rejects the forced heap plan (SelectedPlanFailedCheck, AccuracyTargetMissed); not a wall-clock issue; #1015"] -async fn a_forced_heap_topk_sum_plan_compiles_and_answers_within_its_bound() { - let workload = heap_topk_workload( - "topk by (region) (3, sum_over_time(data[1m]))", - "data", - r#""metric_type": "gauge""#, - ); - let (samples, truth) = skewed_series_samples(); - assert_forced_heap_answers( - &workload, - SketchAlgorithm::CountSketchWithHeap, - (&samples, &truth), - (Some("region"), "instance", 3), - ) - .await; -} - -/// `data` samples of 2 regions with 40 series each inside the last minute: -/// series `r` (rank 1..=40) has `round(200 / r^1.1)` samples (at least 1), -/// a skewed count ranking. -fn skewed_count_samples() -> (Vec, BTreeMap>) { - let mut samples = Vec::new(); - let mut truth: BTreeMap> = BTreeMap::new(); - for region in ["east", "west"] { - for rank in 1..=40 { - let instance = format!("{region}-{rank:02}"); - let count = (200.0 / f64::from(rank).powf(1.1)).round().max(1.0) as i64; - for step in 0..count { - // Spread over (EVALUATION_MS - 59 s, EVALUATION_MS - 1 s]. - let at = EVALUATION_MS - 1_000 - step * 58_000 / count; - let mut s = sample("data", &instance, at, 1.0 + step as f64); - s.labels.insert("region".into(), region.into()); - samples.push(s); - } - truth - .entry(region.to_string()) - .or_default() - .insert(instance, count as f64); - } - } - samples.sort_by_key(|s| s.timestamp_ms); - (samples, truth) -} - -/// `topk by (region) (3, count_over_time(data[1m]))`: each forced fused -/// heap (each sample counts 1) compiles and answers each region's true top -/// 3 series, with their labels, within its bound of their counts. -#[tokio::test] -#[ignore = "fails on main too: Stage 3 rejects the forced heap plan (SelectedPlanFailedCheck, AccuracyTargetMissed); not a wall-clock issue; #1015"] -async fn a_forced_heap_topk_count_plan_compiles_and_answers_within_its_bound() { - let workload = heap_topk_workload( - "topk by (region) (3, count_over_time(data[1m]))", - "data", - r#""metric_type": "gauge""#, - ); - let (samples, truth) = skewed_count_samples(); - for algorithm in [ - SketchAlgorithm::CmsWithHeap, - SketchAlgorithm::CountSketchWithHeap, - ] { - assert_forced_heap_answers( - &workload, - algorithm, - (&samples, &truth), - (Some("region"), "instance", 3), - ) - .await; - } -} - -/// The per-second rate of series `instance` (1 or 2) of job `job` (1..=12) -/// in [`skewed_counter_raw_data_samples`]: `50 / job^1.2`, plus a little per -/// instance, a skewed ranking of the jobs' sums. -fn skewed_counter_rate(job: i64, instance: i64) -> f64 { - 50.0 / (job as f64).powf(1.2) + 0.01 * instance as f64 -} - -/// `data_total` samples of 12 jobs with 2 series each, every 10 s from 595 s -/// before `EVALUATION_MS`, each a counter growing linearly at -/// [`skewed_counter_rate`], as [`counter_raw_data_samples`]. -fn skewed_counter_raw_data_samples() -> Vec { - let mut samples = Vec::new(); - for job in 1..=12 { - for instance in 1..=2 { - for step in 0..60 { - let mut s = sample( - "data_total", - &format!("{job}-{instance}"), - EVALUATION_MS - 595_000 + step * 10_000, - skewed_counter_rate(job, instance) * (5 + step * 10) as f64, - ); - s.labels.insert("job".into(), format!("j{job:02}")); - samples.push(s); - } - } - } - samples -} - -/// The ASAPQuery workloads' counter top-k, `topk(3, sum by (job) -/// ((data_total[5m])))` for `f` `increase` or `rate`: each forced fused -/// heap (one over the per-series values, ranking jobs) compiles and answers -/// the true top 3 jobs, with their `job` labels, within its bound of their -/// exact sums. -#[tokio::test] -#[ignore = "fails on main too: Stage 3 rejects the forced heap plan (SelectedPlanFailedCheck, AccuracyTargetMissed); not a wall-clock issue; #1015"] -async fn a_forced_heap_counter_topk_plan_compiles_and_answers_within_its_bound() { - let samples = skewed_counter_raw_data_samples(); - for (function, seconds) in [("increase", 300.0), ("rate", 1.0)] { - let workload = heap_topk_workload( - &format!("topk(3, sum by (job) ({function}(data_total[5m])))"), - "data_total", - r#""metric_type": "counter""#, - ); - // Each job's exact sum over its two series. - let truth = BTreeMap::from([( - String::new(), - (1..=12) - .map(|job| { - let sum = skewed_counter_rate(job, 1) + skewed_counter_rate(job, 2); - (format!("j{job:02}"), sum * seconds) - }) - .collect::>(), - )]); - for algorithm in [ - SketchAlgorithm::CmsWithHeap, - SketchAlgorithm::CountSketchWithHeap, - ] { - assert_forced_heap_answers(&workload, algorithm, (&samples, &truth), (None, "job", 3)) - .await; - } - } -} - -/// `data` samples of 40 regions with 3 series each, inside the last -/// minute: region `r` (rank 1..=40) sums to `1000 / r^1.1` over -/// its series (a skewed ranking of the regions) and holds `3 · -/// round(200 / r^1.1)` samples (a skewed count ranking), returned with each -/// region's exact sum and exact count. -fn skewed_region_raw_data_samples() -> ( - Vec, - BTreeMap, - BTreeMap, -) { - let mut samples = Vec::new(); - let mut sums = BTreeMap::new(); - let mut counts = BTreeMap::new(); - for rank in 1..=40i64 { - let region = format!("r{rank:02}"); - let sum = 1000.0 / (rank as f64).powf(1.1); - let per_series_steps = (200.0 / (rank as f64).powf(1.1)).round().max(1.0) as i64; - for series in 0..3 { - let instance = format!("{region}-{series}"); - for step in 0..per_series_steps { - // Spread over (EVALUATION_MS - 59 s, EVALUATION_MS - 1 s]. - let at = EVALUATION_MS - 1_000 - step * 58_000 / per_series_steps; - let mut s = sample("data", &instance, at, sum / 3.0 / per_series_steps as f64); - s.labels.insert("region".into(), region.clone()); - samples.push(s); - } - } - sums.insert(region.clone(), sum); - counts.insert(region, (3 * per_series_steps) as f64); - } - samples.sort_by_key(|s| s.timestamp_ms); - (samples, sums, counts) -} - -/// The ASAPQuery workloads' grouped top-k-sum shape, -/// `topk(3, sum by (region) ((data[1m])))` for `f` `sum_over_time` or -/// `count_over_time`: each forced fused heap (one over the raw data samples, -/// keyed by `region`, weighted by the sample value or by one) compiles and -/// answers the true top 3 regions within its bound of their exact sums or -/// counts, each named by its `region` label alone, as the exact top-k's. -/// A Count-Min heap over sample values needs the gauge declared -/// non-negative; a Count Sketch heap takes any gauge. -#[tokio::test] -#[ignore = "fails on main too: Stage 3 rejects the forced heap plan (SelectedPlanFailedCheck, AccuracyTargetMissed); not a wall-clock issue; #1015"] -async fn a_forced_heap_grouped_topk_sum_plan_compiles_and_answers_within_its_bound() { - let (samples, sums, counts) = skewed_region_raw_data_samples(); - let gauge = r#""metric_type": "gauge""#; - let non_negative = r#""metric_type": "gauge", "values_non_negative": true"#; - for (function, truth, facts, algorithm) in [ - ( - "sum_over_time", - &sums, - non_negative, - SketchAlgorithm::CmsWithHeap, - ), - ( - "sum_over_time", - &sums, - gauge, - SketchAlgorithm::CountSketchWithHeap, - ), - ( - "count_over_time", - &counts, - gauge, - SketchAlgorithm::CmsWithHeap, - ), - ( - "count_over_time", - &counts, - gauge, - SketchAlgorithm::CountSketchWithHeap, - ), - ] { - let query = format!("topk(3, sum by (region) ({function}(data[1m])))"); - let workload = heap_topk_workload(&query, "data", facts); - let truth = BTreeMap::from([(String::new(), truth.clone())]); - let plan = plan_ignoring_guarantees(&workload, Some(algorithm)) - .await - .unwrap_or_else(|error| panic!("{query} {algorithm:?}: {error}")); - let (built, width) = heap_of(&plan).expect("a heap plan"); - assert_eq!(built, algorithm, "{query}"); - plan.check_compiles().unwrap(); - let plan_json = serde_json::to_string(&plan).unwrap(); - let run = run_data_plane( - &plan_json, - &samples, - EVALUATION_MS, - &mut SummaryStore::new(), - ); - for answers in [ - &run.answers_per_query[0], - &run.reference_answers_per_query[0], - ] { - assert_heap_topk_answers(answers, &truth, (None, "region", 3), (&algorithm, width)); - for (labels, _) in answers.iter() { - assert_eq!( - labels.keys().collect::>(), - ["region"], - "{query} {algorithm:?}: the group label alone" - ); - } - } - // The installed plan (stored states) answers as its query-time - // reference does, within the heap's bound. - let bound = heap_bound( - &algorithm, - width, - &truth[""].values().copied().collect::>(), - ); - let reference: BTreeMap<_, _> = run.reference_answers_per_query[0] - .iter() - .map(|(labels, value)| (labels.clone(), *value)) - .collect(); - for (labels, value) in &run.answers_per_query[0] { - let expected = reference - .get(labels) - .unwrap_or_else(|| panic!("{query} {algorithm:?}: {labels:?} in the reference")); - assert!( - (value - expected).abs() <= bound, - "{query} {algorithm:?}: {value} vs {expected}" - ); - } - } -} - // ── Classic-histogram `histogram_quantile` (issue #890) ───────────────────── -/// The classic bucket metric of the histogram tests. -const BUCKET_METRIC: &str = "http_request_duration_seconds_bucket"; - -/// Exact PromQL `queries`, every minute, over ingesting classic-bucket -/// series of [`BUCKET_METRIC`] sampled every 10 s. -fn exact_bucket_workload(queries: &[&str]) -> PlanningWorkload { - let entries: Vec = queries - .iter() - .map(|query| { - format!( - r#"{{"query": {query:?}, "demand": {{"fixed_interval": 60000}}, - "requirements": {{"accuracy": {{"max_error": 0.0, "failure_probability": 0.0}}}}, - "predictability": {{"predictable": {{"known_at": null}}}}}}"# - ) - }) - .collect(); - serde_json::from_str(&format!( - r#"{{"query_workload": {{"language": "promql", "repeating_queries": [{}]}}, - "data_workload": {{"arrival": {{"ingesting": {{"rows_per_second": 2.4}}}}, - "sources": {{"{BUCKET_METRIC}": {{"series_count": 24, "sample_interval": 10000}}}}}}}}"#, - entries.join(",") - )) - .unwrap() -} - -/// Cumulative bucket counters of [`BUCKET_METRIC`], one sample every 10 s -/// over the 5 minutes before `EVALUATION_MS` (first 295 s before it, last -/// 5 s before it), each rising at a constant per-second rate from 1000, so -/// `rate(…[5m])` of each is its slope. Per `(service, job)`, the slopes of -/// buckets `le` 0.1, 0.5, 1 and `+Inf`, each on instances `1` and `2`: -/// - `orders`/`api`: 0.5, 1.5, 2, 2 (summed over instances: 1, 3, 4, 4); -/// - `batch`/`worker`: 0.5, 0.5, 0.5, 2.5 (most observations above 1, in -/// `+Inf` alone; summed: 1, 1, 1, 5); -/// - `idle`/`idle`: 0 everywhere (no observations). -fn bucket_samples() -> Vec { - let mut samples = Vec::new(); - for (service, job, slopes) in [ - ("orders", "api", [0.5, 1.5, 2.0, 2.0]), - ("batch", "worker", [0.5, 0.5, 0.5, 2.5]), - ("idle", "idle", [0.0; 4]), - ] { - for (le, slope) in ["0.1", "0.5", "1", "+Inf"].into_iter().zip(slopes) { - for instance in ["1", "2"] { - for step in 0..30 { - samples.push(RawDataSample { - labels: BTreeMap::from([ - ("__name__".to_string(), BUCKET_METRIC.to_string()), - ("service".to_string(), service.to_string()), - ("job".to_string(), job.to_string()), - ("instance".to_string(), instance.to_string()), - ("le".to_string(), le.to_string()), - ]), - timestamp_ms: EVALUATION_MS - 295_000 + step * 10_000, - value: 1000.0 + slope * 10.0 * step as f64, - }); - } - } - } - } - samples -} - -/// The plan file of exact `queries` over [`bucket_samples`]' metric, on a -/// deployment that keeps raw data samples. -async fn bucket_plan_json(queries: &[&str]) -> String { - plan_json_for( - exact_bucket_workload(queries), - &DeploymentCapabilities::default(), - ) - .await -} - -/// `histogram_quantile` over classic buckets as the issue's dashboard -/// writes it: p50, p95 and p99 of the summed bucket rates of one service, -/// in one workload. Stage 3 selects an exact plan the executor compiles, -/// and it answers with Prometheus' cumulative-bucket interpolation: over -/// summed rates 1, 3, 4, 4 at `le` 0.1, 0.5, 1, `+Inf`, rank 0.5 · 4 = 2 -/// lies in the 0.5 bucket, 0.1 + 0.4 · (2 − 1) / (3 − 1) = 0.3; rank 3.8 -/// and 3.96 in the 1 bucket, 0.5 + 0.5 · 0.8 = 0.9 and 0.5 + 0.5 · 0.96 = -/// 0.98. The result has no labels: `sum by (le)` keeps only `le`, which -/// the quantile drops. -#[tokio::test] -async fn classic_histogram_quantiles_run_with_prometheus_interpolation() { - let selector = format!(r#"rate({BUCKET_METRIC}{{service="orders"}}[5m])"#); - let queries: Vec = [0.5, 0.95, 0.99] - .iter() - .map(|q| format!("histogram_quantile({q}, sum by (le) ({selector}))")) - .collect(); - let queries: Vec<&str> = queries.iter().map(String::as_str).collect(); - let plan_json = bucket_plan_json(&queries).await; - let run = run_data_plane( - &plan_json, - &bucket_samples(), - EVALUATION_MS, - &mut SummaryStore::new(), - ); - assert_eq!(run.answers_per_query.len(), 3); - for (answers, expected) in run.answers_per_query.iter().zip([0.3, 0.9, 0.98]) { - assert_eq!(answers.len(), 1, "{answers:?}"); - let (labels, value) = &answers[0]; - assert!(labels.is_empty(), "le is dropped: {labels:?}"); - assert!((value - expected).abs() < 1e-9, "{value} vs {expected}"); - } - assert_eq!(run.answers_per_query, run.reference_answers_per_query); -} - -/// One p99 per `job` (`sum by (job, le)`): the result keeps `job` and -/// drops `le`. `worker`'s rank 0.99 · 5 = 4.95 lies past every finite -/// bucket (most observations are in `+Inf` alone), so Prometheus answers -/// the largest finite bound, 1; `idle` has no observations, so NaN. The -/// p99 alert (`> 0.5`) keeps `api` (0.98) and `worker` (1) and drops -/// `idle` (NaN compares false). A selector that matches no series gives an -/// empty result. Over one series' buckets, without `sum`, the result keeps -/// that series' labels but `le` (and the name `rate` dropped). -#[tokio::test] -async fn a_classic_histogram_quantile_keeps_labels_and_handles_inf_and_empty_input() { - let by_job = format!("histogram_quantile(0.99, sum by (job, le) (rate({BUCKET_METRIC}[5m])))"); - let alert = format!("{by_job} > 0.5"); - let empty = format!( - r#"histogram_quantile(0.5, sum by (le) (rate({BUCKET_METRIC}{{service="missing"}}[5m])))"# - ); - // Without `sum`, each series is its own histogram: `instance` 1's - // buckets alone (rates 0.5, 1.5, 2, 2), keeping its labels but `le`. - let per_series = format!( - r#"histogram_quantile(0.99, rate({BUCKET_METRIC}{{service="orders",instance="1"}}[5m]))"# - ); - let plan_json = bucket_plan_json(&[&by_job, &alert, &empty, &per_series]).await; - let run = run_data_plane( - &plan_json, - &bucket_samples(), - EVALUATION_MS, - &mut SummaryStore::new(), - ); - let job = |answers: &[Answer]| -> Vec<(String, f64)> { - answers - .iter() - .map(|(labels, value)| { - assert_eq!(labels.keys().collect::>(), ["job"], "{labels:?}"); - (labels["job"].clone(), *value) - }) - .collect() - }; - for answers in [&run.answers_per_query, &run.reference_answers_per_query] { - let per_job = job(&answers[0]); - assert_eq!(per_job.len(), 3, "{per_job:?}"); - assert_eq!(per_job[0].0, "api"); - assert!((per_job[0].1 - 0.98).abs() < 1e-9, "{per_job:?}"); - assert_eq!(per_job[1].0, "idle"); - assert!(per_job[1].1.is_nan(), "{per_job:?}"); - assert_eq!(per_job[2], ("worker".to_string(), 1.0)); - let alerting = job(&answers[1]); - assert_eq!(alerting.len(), 2, "{alerting:?}"); - assert_eq!(alerting[0].0, "api"); - assert_eq!(alerting[1], ("worker".to_string(), 1.0)); - assert!(answers[2].is_empty(), "{:?}", answers[2]); - let [(labels, value)] = answers[3].as_slice() else { - panic!("{:?}", answers[3]); - }; - assert_eq!( - labels.keys().collect::>(), - ["instance", "job", "service"], - "{labels:?}" - ); - assert!((value - 0.98).abs() < 1e-9, "{value}"); - } -} - // ── PromQL absent() (issue #895) ───────────────────────────────────────────── - -/// The Prometheus stale marker, which Remote Write carries when a series ends. -const STALE_MARKER: f64 = f64::from_bits(0x7ff0_0000_0000_0002); - -/// The answers of the exact plan of PromQL `query` over `samples` of `m` -/// (6 series, every 10 s), with the query-time reference's. -async fn exact_answers(query: &str, samples: &[RawDataSample]) -> Vec { - let plan_json = plan_json_for( - exact_workload(&[query], ""), - &DeploymentCapabilities::default(), - ) - .await; - let run = run_data_plane(&plan_json, samples, EVALUATION_MS, &mut SummaryStore::new()); - assert_eq!(run.answers_per_query, run.reference_answers_per_query); - run.answers_per_query[0].clone() -} - -/// A sample of `m` at `ago_ms` before `EVALUATION_MS` with `labels`. -fn m_sample(labels: &[(&str, &str)], ago_ms: i64, value: f64) -> RawDataSample { - let mut sample = sample("m", "1", EVALUATION_MS - ago_ms, value); - for (name, label) in labels { - sample.labels.insert(name.to_string(), label.to_string()); - } - sample -} - -/// `absent(v)` answers as Prometheus' `funcAbsent`: one sample of value 1 -/// when `v` selects no series at the evaluation time, labelled with the -/// selector's equality matchers (`job`, not the negated `instance` and -/// `env`); no sample when some series matches, whatever its value -/// (0 included). A series whose newest sample is the stale marker has -/// ended, so `absent` answers 1 again. -#[tokio::test] -async fn absent_answers_as_prometheus_through_the_installed_plan() { - let query = r#"absent(m{job="a",instance!="3",env!="prod"})"#; - let one = vec![(BTreeMap::from([("job".to_string(), "a".to_string())]), 1.0)]; - // No sample at all. - assert_eq!(exact_answers(query, &[]).await, one); - // Only series the matchers reject: job b, env prod, instance 3. - let rejected = [ - m_sample(&[("job", "b")], 1_000, 1.0), - m_sample(&[("job", "a"), ("env", "prod")], 1_000, 1.0), - { - let mut other = m_sample(&[("job", "a")], 1_000, 1.0); - other.labels.insert("instance".into(), "3".into()); - other - }, - ]; - assert_eq!(exact_answers(query, &rejected).await, one); - // A matching series, valued 0 or not: no sample. - for value in [0.0, 5.0] { - let samples = [m_sample(&[("job", "a")], 1_000, value)]; - assert_eq!(exact_answers(query, &samples).await, vec![], "{value}"); - } - // The last matching series ended (its newest sample is stale). - let ended = [ - m_sample(&[("job", "a")], 5_000, 5.0), - m_sample(&[("job", "a")], 1_000, STALE_MARKER), - ]; - assert_eq!(exact_answers(query, &ended).await, one); -} - -/// `absent(m)` (no matcher) synthesizes a label-less sample; the missing-job -/// alert shape (several equality matchers) keeps every matcher's label. -#[tokio::test] -async fn absent_labels_come_from_the_equality_matchers_only() { - assert_eq!( - exact_answers("absent(m)", &[]).await, - vec![(BTreeMap::new(), 1.0)] - ); - let alert = r#"absent(m{kubernetes_namespace="mw-content-history-reconcile-enrich",release="production",job_name="mw_content_history_reconcile_enrich"})"#; - let labels: BTreeMap = [ - ( - "kubernetes_namespace", - "mw-content-history-reconcile-enrich", - ), - ("release", "production"), - ("job_name", "mw_content_history_reconcile_enrich"), - ] - .into_iter() - .map(|(name, value)| (name.to_string(), value.to_string())) - .collect(); - assert_eq!(exact_answers(alert, &[]).await, vec![(labels.clone(), 1.0)]); - let running = [m_sample( - &labels - .iter() - .map(|(name, value)| (name.as_str(), value.as_str())) - .collect::>(), - 1_000, - 3600.0, - )]; - assert_eq!(exact_answers(alert, &running).await, vec![]); -} diff --git a/crates/planner/tests/sql_per_group_topk.rs b/crates/planner/tests/sql_per_group_topk.rs deleted file mode 100644 index d4ea3a199..000000000 --- a/crates/planner/tests/sql_per_group_topk.rs +++ /dev/null @@ -1,405 +0,0 @@ -//! SQL's per-group top-k idiom, `ROW_NUMBER() OVER (PARTITION BY city -//! ORDER BY COUNT(*) DESC) … WHERE rn <= k`, planned with -//! `plan_installable` and run with the Planner's executor: the exact plan -//! compiles and answers each city's top hosts with their counts, the same -//! rows PromQL `topk by (city)` answers over the same data, and the plan -//! Stage 3 selects under an approximate target installs (each materialized -//! view it keeps refreshes). - -use std::collections::{BTreeMap, BTreeSet}; -use std::sync::Arc; - -use asap_executor::physical_planner::compile; -use asap_executor::runtime::{Limits, RunContext, Scope}; -use asap_executor::sources::InMemoryDataStore; -use asap_executor::values::{Batch, Value}; -use asap_planner::{ - plan_installable, test_support::query_time_reference, FrontendInput, InstallablePlan, - InstallablePlanError, PlanningModels, PlanningWorkload, SqlCatalog, UserInput, -}; -use asap_types::ir::operator::Source as DataSource; -use asap_types::ir::physical_export::PhysicalASAPOperatorPayload; -use asap_types::ir::schema::{DataType, Field, Schema}; -use asap_types::ir::NonASAPOp; -use futures::{executor::block_on, StreamExt}; - -use crate::time_series; -use time_series::{time_series_tables, RawDataSample}; - -/// The evaluation time of every run. -const EVALUATION_MS: i64 = 1_800_000_000_000; - -/// The issue's per-city top 5 hosts by row count over the last 5 minutes. -const TOP_5_HOSTS_PER_CITY: &str = "SELECT city, host, c FROM (SELECT city, host, COUNT(*) AS c, \ - ROW_NUMBER() OVER (PARTITION BY city ORDER BY COUNT(*) DESC) AS rn FROM metrics \ - WHERE ts > now() - INTERVAL '5' MINUTE GROUP BY city, host) WHERE rn <= 5"; - -/// The issue's global top 5 hosts by row count over the last 5 minutes. -const TOP_5_HOSTS: &str = "SELECT host, COUNT(*) AS c FROM metrics \ - WHERE ts > now() - INTERVAL '5' MINUTE GROUP BY host ORDER BY c DESC LIMIT 5"; - -/// The same per-city top 5 in PromQL, each table row a sample of the -/// series `metrics{city, host}`. -const PROMQL_TOP_5_HOSTS_PER_CITY: &str = - "topk by (city) (5, sum by (city, host) (count_over_time(metrics[5m])))"; - -/// `metrics(ts, city, host, value)`, as the issue declares it (all but -/// `ts` nullable). -fn metrics_schema() -> Schema { - Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("city", DataType::Utf8, true), - Field::plain("host", DataType::Utf8, true), - Field::plain("value", DataType::Float64, true), - ], - 0, - vec![], - ) -} - -/// Repeating `queries` in `language` (its workload JSON), every 10 s at -/// `accuracy`, over `metrics` ingesting 666.67 rows per second from 100 -/// series: the issue's workload. -fn workload(language: &str, queries: &[&str], accuracy: &str) -> PlanningWorkload { - let entries: Vec = queries - .iter() - .map(|query| { - format!( - r#"{{"query": {query:?}, "demand": {{"fixed_interval": 10000}}, - "requirements": {{"accuracy": {accuracy}}}, - "predictability": {{"predictable": {{"known_at": null}}}}}}"# - ) - }) - .collect(); - serde_json::from_str(&format!( - r#"{{"query_workload": {{"language": {language}, "repeating_queries": [{}]}}, - "data_workload": {{"arrival": {{"ingesting": {{"rows_per_second": 666.67}}}}, - "sources": {{"metrics": {{"series_count": 100}}}}}}}}"#, - entries.join(",") - )) - .unwrap() -} - -const EXACT: &str = r#"{"max_error": 0.0, "failure_probability": 0.0}"#; -const APPROXIMATE: &str = r#"{"max_error": 0.01, "failure_probability": 0.001}"#; - -async fn plan_sql(queries: &[&str], accuracy: &str) -> InstallablePlan { - let catalog = SqlCatalog::new().with_table("metrics", metrics_schema()); - plan_installable(UserInput::new( - &workload(r#"{"sql": "datafusion_sql"}"#, queries, accuracy), - FrontendInput::Sql { catalog: &catalog }, - PlanningModels::builtin(), - )) - .await - .expect("plan_installable plans the SQL workload") - .0 -} - -async fn plan_promql(query: &str) -> InstallablePlan { - plan_installable(UserInput::new( - &workload(r#""promql""#, &[query], EXACT), - FrontendInput::Promql, - PlanningModels::builtin(), - )) - .await - .expect("plan_installable plans the PromQL workload") - .0 -} - -/// Rows per `(city, host)` in the last 5 minutes. `nyc` has seven hosts -/// with distinct counts at the 5th place (`h5` 5, `h6` and `h7` 3), so the -/// top 5 does not depend on how ties are broken; `sf` has two hosts, fewer -/// than 5. `h7` also has 100 rows from before the window, which no answer -/// counts. -const RECENT_ROWS: [(&str, &str, i64); 9] = [ - ("nyc", "h1", 9), - ("nyc", "h2", 8), - ("nyc", "h3", 7), - ("nyc", "h4", 6), - ("nyc", "h5", 5), - ("nyc", "h6", 3), - ("nyc", "h7", 3), - ("sf", "s1", 4), - ("sf", "s2", 2), -]; - -/// The table rows: per `(city, host)`, its [`RECENT_ROWS`] count one second -/// apart ending a second before the evaluation, and `h7`'s 100 older rows -/// 10 to 11.65 minutes before it. -fn rows() -> Vec<(i64, &'static str, &'static str)> { - let mut rows = Vec::new(); - for (city, host, count) in RECENT_ROWS { - for i in 0..count { - rows.push((EVALUATION_MS - 1000 * (i + 1), city, host)); - } - } - for i in 0..100 { - rows.push((EVALUATION_MS - 600_000 - 1000 * i, "nyc", "h7")); - } - rows -} - -/// The exact answer: each city's top 5 hosts by recent row count. -fn expected_top_5_per_city() -> BTreeSet<(String, String, i64)> { - let mut per_city: BTreeMap<&str, Vec<(&str, i64)>> = BTreeMap::new(); - for (city, host, count) in RECENT_ROWS { - per_city.entry(city).or_default().push((host, count)); - } - per_city - .into_iter() - .flat_map(|(city, mut hosts)| { - hosts.sort_by_key(|&(_, count)| std::cmp::Reverse(count)); - hosts - .into_iter() - .take(5) - .map(move |(host, count)| (city.to_string(), host.to_string(), count)) - }) - .collect() -} - -/// Each answer row's `city` (when it has one), `host` and the ranked count -/// (the column named `ranked`), read by name. -fn ranked_rows(batches: &[Batch], ranked: &str) -> BTreeSet<(String, String, i64)> { - batches - .iter() - .flat_map(|batch| { - let names: Vec<&str> = batch - .schema() - .fields - .iter() - .map(|f| f.name.as_str()) - .collect(); - let column = |name: &str| names.iter().position(|n| *n == name); - let (city, host, count) = ( - column("city"), - column("host").unwrap_or_else(|| panic!("a host column in {names:?}")), - column(ranked).unwrap_or_else(|| panic!("a {ranked} column in {names:?}")), - ); - batch - .to_rows() - .into_iter() - .map(move |row| { - let text = |index: usize| match &row[index] { - Value::Utf8(text) => text.to_string(), - other => panic!("a label: {other:?}"), - }; - let count = match row[count] { - Value::Int64(n) => n, - Value::Float64(x) if x.fract() == 0.0 => x as i64, - ref other => panic!("a count: {other:?}"), - }; - (city.map(text).unwrap_or_default(), text(host), count) - }) - .collect::>() - }) - .collect() -} - -/// Query `query` of `plan` with every node at query time, at -/// [`EVALUATION_MS`], with `metrics` bound to [`rows`]: its batches. -fn run_sql(plan: &InstallablePlan, query: usize) -> Vec { - let reference = plan - .dag - .with_every_node_at_query_time() - .expect("the plan runs at query time"); - let schema = reference - .nodes - .iter() - .find_map(|node| match &node.payload { - PhysicalASAPOperatorPayload::NonASAP(NonASAPOp::Scan { schema, .. }) => { - Some(Arc::new(schema.clone())) - } - _ => None, - }) - .expect("the plan scans the table"); - let rows = rows() - .into_iter() - .map(|(ts, city, host)| { - vec![ - Value::Timestamp(ts), - Value::Utf8(city.into()), - Value::Utf8(host.into()), - Value::Float64(1.0), - ] - }) - .collect(); - let batch = Batch::try_new(schema.clone(), rows).unwrap(); - let store = InMemoryDataStore::default() - .with_raw_data( - DataSource::Table { - table_ref: "metrics".into(), - }, - schema, - vec![batch], - ) - .unwrap(); - let roots = [reference.roots[query] as u64]; - let bound = compile(&reference, &[], &roots) - .and_then(|compiled| compiled.instantiate(Arc::new(store))) - .expect("the plan binds to the table"); - let batches = collect(bound.execute(&roots, query_scope()).unwrap().remove(0)); - batches -} - -/// The PromQL plan's query-time reference at [`EVALUATION_MS`], each table -/// row of [`rows`] a sample of `metrics{city, host}`: its batches. -fn run_promql(plan: &InstallablePlan) -> Vec { - let samples: Vec = rows() - .into_iter() - .map(|(timestamp_ms, city, host)| RawDataSample { - labels: BTreeMap::from([ - ("__name__".to_string(), "metrics".to_string()), - ("city".to_string(), city.to_string()), - ("host".to_string(), host.to_string()), - ]), - timestamp_ms, - value: 1.0, - }) - .collect(); - let reference = query_time_reference(plan).unwrap(); - let dag = &reference.dag; - let roots = [reference.dag.roots[0] as u64]; - let compiled = compile(dag, &[], &roots).expect("the executor compiles the plan"); - // The samples bound at the plan's `Scan` of `metrics`. - let instance = compiled - .instantiate(Arc::new(time_series_tables(dag, &samples))) - .unwrap(); - let batches = collect( - instance - .execute(compiled.roots(), query_scope()) - .unwrap() - .remove(0), - ); - batches -} - -fn query_scope() -> RunContext { - RunContext::new( - Scope::Query { - evaluation_time_ms: EVALUATION_MS, - }, - Limits::default(), - ) - .unwrap() -} - -/// Every batch of one query's `output`. -fn collect(mut output: impl futures::Stream> + Unpin) -> Vec -where - B: std::ops::Deref, - E: std::fmt::Debug, -{ - block_on(async { - let mut batches = Vec::new(); - while let Some(batch) = output.next().await { - batches.push((*batch.expect("the query runs")).clone()); - } - batches - }) -} - -/// The exact per-city top 5 compiles and answers each city's top hosts -/// with their counts, `sf` with its two. -#[tokio::test] -async fn an_exact_sql_per_city_top_5_runs() { - let plan = plan_sql(&[TOP_5_HOSTS_PER_CITY], EXACT).await; - plan.check_compiles().unwrap(); - let answered = ranked_rows(&run_sql(&plan, 0), "topk_5"); - assert_eq!(answered, expected_top_5_per_city()); -} - -/// SQL's `ROW_NUMBER` per-city top 5 and PromQL's `topk by (city)` answer -/// the same rows over the same data. -#[tokio::test] -async fn sql_and_promql_per_city_top_5_answer_alike() { - let sql = ranked_rows( - &run_sql(&plan_sql(&[TOP_5_HOSTS_PER_CITY], EXACT).await, 0), - "topk_5", - ); - let promql = ranked_rows( - &run_promql(&plan_promql(PROMQL_TOP_5_HOSTS_PER_CITY).await), - "topk_5", - ); - assert_eq!(sql, promql); - assert_eq!(sql, expected_top_5_per_city()); -} - -/// The global top 5 answers its hosts, not only their counts. -#[tokio::test] -async fn an_exact_sql_top_5_answers_its_hosts() { - let plan = plan_sql(&[TOP_5_HOSTS], EXACT).await; - let answered: BTreeSet<(String, i64)> = ranked_rows(&run_sql(&plan, 0), "topk_5") - .into_iter() - .map(|(_, host, count)| (host, count)) - .collect(); - let expected = [("h1", 9), ("h2", 8), ("h3", 7), ("h4", 6), ("h5", 5)] - .map(|(host, count)| (host.to_string(), count)); - assert_eq!(answered, BTreeSet::from(expected)); -} - -/// The issue's two queries under ε 0.01, δ 0.001: the plan Stage 3 selects -/// installs, so the executor refreshes every materialized view it keeps and -/// compiles its query run. -#[tokio::test] -async fn the_selected_approximate_plan_installs_and_compiles() { - let plan = plan_sql(&[TOP_5_HOSTS, TOP_5_HOSTS_PER_CITY], APPROXIMATE).await; - plan.check_compiles().unwrap(); -} - -/// With `rn` selected, the per-city top 5 keeps its window function, which -/// the executor does not run, so no plan of the workload compiles. Stage 3 -/// does not compile candidates; installation compiles the selected plan -/// (each view's refresh, then the query run) and refuses it with the -/// executor's reason rather than returning a plan a runtime cannot run. -#[tokio::test] -async fn a_plan_the_executor_cannot_run_is_refused_at_installation() { - let with_rn = - TOP_5_HOSTS_PER_CITY.replace("SELECT city, host, c FROM", "SELECT city, host, c, rn FROM"); - let catalog = SqlCatalog::new().with_table("metrics", metrics_schema()); - let planned = plan_installable(UserInput::new( - &workload( - r#"{"sql": "datafusion_sql"}"#, - &[TOP_5_HOSTS, &with_rn], - APPROXIMATE, - ), - FrontendInput::Sql { catalog: &catalog }, - PlanningModels::builtin(), - )) - .await; - match planned { - Err(asap_planner::PlanError::Installable( - error @ (InstallablePlanError::MaterializedViewNotExecutable { .. } - | InstallablePlanError::QueryRunNotExecutable { .. }), - )) => { - let diagnostic = error.to_string(); - assert!( - diagnostic.contains("value operation has no native implementation"), - "{diagnostic}" - ); - } - other => panic!("no plan is installed: {:?}", other.map(|_| ())), - } -} - -/// A per-city top 5 the heavy-hitter `TopK` does not cover runs exactly as -/// the per-partition `Limit{Sort}`, with the query's own columns: ordered by -/// the count then the host (a tie-breaking second key), and with a SELECT -/// list that drops the partition column. -#[tokio::test] -async fn generic_per_city_top_5_forms_run_exactly() { - let tie_broken = TOP_5_HOSTS_PER_CITY.replace("COUNT(*) DESC)", "COUNT(*) DESC, host)"); - let plan = plan_sql(&[&tie_broken], EXACT).await; - assert_eq!( - ranked_rows(&run_sql(&plan, 0), "c"), - expected_top_5_per_city() - ); - let hosts_only = - TOP_5_HOSTS_PER_CITY.replace("SELECT city, host, c FROM", "SELECT host, c FROM"); - let plan = plan_sql(&[&hosts_only], EXACT).await; - let expected: BTreeSet<_> = expected_top_5_per_city() - .into_iter() - .map(|(_, host, count)| (String::new(), host, count)) - .collect(); - assert_eq!(ranked_rows(&run_sql(&plan, 0), "c"), expected); -} diff --git a/crates/planner/tests/sql_table_scans.rs b/crates/planner/tests/sql_table_scans.rs index 69dd65cf4..9dd9b41a2 100644 --- a/crates/planner/tests/sql_table_scans.rs +++ b/crates/planner/tests/sql_table_scans.rs @@ -12,18 +12,16 @@ use std::sync::Arc; use asap_executor::physical_planner::compile; use asap_executor::runtime::{Limits, RunContext, Scope}; -use asap_executor::sources::{ - DataStore, InMemoryDataStore, MaterializedViewRequest, RawDataRequest, -}; +use asap_executor::sources::InMemoryDataStore; use asap_executor::values::{Batch, Value}; use asap_planner::{ plan_installable, DeploymentCapabilities, FrontendInput, InstallablePlan, MaterializedViewRefresh, PlanningModels, PlanningWorkload, SqlCatalog, UserInput, }; -use asap_types::ir::materialized_view::MaterializedView; + use asap_types::ir::operator::Source; use asap_types::ir::physical_export::PhysicalASAPOperatorPayload; -use asap_types::ir::schema::{DataType, Field, FieldDataType, Schema}; +use asap_types::ir::schema::{DataType, Field, Schema}; use asap_types::ir::NonASAPOp; use futures::{executor::block_on, StreamExt}; @@ -78,20 +76,6 @@ const APPROXIMATE: &str = r#"{"accuracy_metric": "relative", "max_error": 0.05, /// No approximation. const EXACT: &str = r#"{"max_error": 0.0, "failure_probability": 0.0}"#; -async fn plan(query: &str, accuracy: &str) -> InstallablePlan { - plan_all(&[query], accuracy).await -} - -async fn plan_all(queries: &[&str], accuracy: &str) -> InstallablePlan { - plan_for( - queries, - accuracy, - MINUTE_MS as u64, - &DeploymentCapabilities::default(), - ) - .await -} - /// [`plan_all`] for a deployment that keeps no raw data: every window is /// read by state maintained at ingestion time. async fn plan_without_raw_data(queries: &[&str], accuracy: &str) -> InstallablePlan { @@ -147,74 +131,6 @@ fn rows() -> Vec> { .collect() } -/// Runs query 0 of `plan`, every node at query time (its reference plan, -/// which reads the table's rows wherever the plan keeps state), at -/// [`EVALUATION_MS`] with `http` bound to [`rows`]: its output field types -/// and its rows. A `WHERE ts >= now() - INTERVAL` window is a `TimeRange` -/// the executor runs as a filter on `ts` against the evaluation time. -fn run(plan: &InstallablePlan) -> (Vec, Vec>) { - run_query(plan, 0) -} - -/// [`run`] for query `query` of `plan`. -fn run_query(plan: &InstallablePlan, query: usize) -> (Vec, Vec>) { - let reference = plan - .dag - .with_every_node_at_query_time() - .expect("the plan runs at query time"); - // The table's rows in the schema the plan's scan reads (its columns - // qualified by the table). - let schema = reference - .nodes - .iter() - .find_map(|node| match &node.payload { - PhysicalASAPOperatorPayload::NonASAP(NonASAPOp::Scan { schema, .. }) => { - Some(Arc::new(schema.clone())) - } - _ => None, - }) - .expect("the plan scans the table"); - let batch = Batch::try_new(schema.clone(), rows()).unwrap(); - let store = InMemoryDataStore::default() - .with_raw_data( - Source::Table { - table_ref: "http".into(), - }, - schema, - vec![batch], - ) - .unwrap(); - let roots = [reference.roots[query] as u64]; - let bound = compile(&reference, &[], &roots) - .and_then(|compiled| compiled.instantiate(Arc::new(store))) - .expect("the plan binds to the table"); - let context = RunContext::new( - Scope::Query { - evaluation_time_ms: EVALUATION_MS, - }, - Limits::default(), - ) - .unwrap(); - block_on(async { - let mut output = bound.execute(&roots, context).unwrap().remove(0); - let mut fields = None; - let mut rows = Vec::new(); - while let Some(batch) = output.next().await { - let batch = batch.expect("the query runs"); - fields.get_or_insert_with(|| { - batch - .schema() - .fields - .iter() - .map(|f| f.dtype.clone()) - .collect() - }); - rows.extend(batch.to_rows()); - } - (fields.unwrap_or_default(), rows) - }) -} - /// The answer rows by their first column, each value as it prints (its /// type and value). fn by_key(rows: Vec>) -> BTreeMap { @@ -235,17 +151,6 @@ fn shown(value: &Value) -> String { } } -/// The planned output type of query 0's root. -fn planned_output(plan: &InstallablePlan) -> Vec { - let root = plan.dag.roots[0]; - let node = plan.dag.nodes.iter().find(|n| n.id == root).unwrap(); - node.output_schema - .fields - .iter() - .map(|f| f.dtype.clone()) - .collect() -} - const DISTINCT_USERS_LAST_5_MINUTES: &str = "SELECT service, COUNT(DISTINCT user_id) AS value \ FROM http WHERE ts >= now() - INTERVAL '5 minutes' GROUP BY service"; @@ -258,159 +163,6 @@ fn expected_distinct_users() -> BTreeMap { .collect() } -#[tokio::test] -async fn a_distinct_count_over_an_int64_id_in_a_now_window_runs_from_a_sketch() { - let plan = plan(DISTINCT_USERS_LAST_5_MINUTES, APPROXIMATE).await; - let selected = format!("{:?}", plan.dag.nodes); - assert!( - selected.contains("SummaryAgg"), - "a distinct-count sketch answers it: {selected}" - ); - let (fields, rows) = run(&plan); - assert_eq!( - fields, - planned_output(&plan), - "the executor's output type is the plan's" - ); - // The sketch is exact below its capacity. - assert_eq!(by_key(rows), expected_distinct_users()); -} - -#[tokio::test] -async fn an_exact_distinct_count_over_an_int64_id_in_a_now_window_runs() { - let plan = plan(DISTINCT_USERS_LAST_5_MINUTES, EXACT).await; - let (fields, rows) = run(&plan); - assert_eq!( - fields, - planned_output(&plan), - "the executor's output type is the plan's" - ); - assert_eq!(by_key(rows), expected_distinct_users()); -} - -#[tokio::test] -async fn now_is_the_evaluation_time() { - let plan = plan( - "SELECT service, COUNT(*) AS value FROM http \ - WHERE ts > now() - INTERVAL '1 minute' GROUP BY service", - EXACT, - ) - .await; - let (_, rows) = run(&plan); - // The 60 rows after the minute before the evaluation: 20 per service. - let expected: BTreeMap = (0..3) - .map(|s| (format!("s{s}"), shown(&Value::Int64(20)))) - .collect(); - assert_eq!(by_key(rows), expected); -} - -/// SQL `percentile_cont` of `values`: linear interpolation at rank -/// `q * (n - 1)` over the sorted values. -fn percentile_cont(q: f64, mut values: Vec) -> f64 { - values.sort_by(f64::total_cmp); - let rank = q * (values.len() - 1) as f64; - let (low, high) = (rank.floor() as usize, rank.ceil() as usize); - values[low] + (values[high] - values[low]) * (rank - low as f64) -} - -#[tokio::test] -async fn an_exact_grouped_sql_percentile_runs() { - let plan = plan( - "SELECT service, approx_percentile_cont(latency, 0.99) AS value FROM http \ - WHERE ts >= now() - INTERVAL '5 minutes' GROUP BY service", - EXACT, - ) - .await; - let (fields, rows) = run(&plan); - assert_eq!( - fields, - planned_output(&plan), - "the executor's output type is the plan's" - ); - // Each service's latencies `i` over the last 5 minutes. The window is a - // range selector's `(T - 5m, T]`, so the row stamped exactly `T - 5m` - // (i = 899) is not read: i >= 900. - let expected: BTreeMap = (0..3i64) - .map(|s| { - let latencies = (900..1200) - .filter(|i| i % 3 == s) - .map(|i| i as f64) - .collect(); - ( - format!("s{s}"), - shown(&Value::Float64(percentile_cont(0.99, latencies))), - ) - }) - .collect(); - assert_eq!(by_key(rows), expected); -} - -/// A global percentile over an empty window is NULL (SQL), and the plan -/// types it nullable. -#[tokio::test] -async fn an_exact_global_sql_percentile_of_no_row_is_null() { - let plan = plan( - "SELECT approx_percentile_cont(latency, 0.5) AS value FROM http \ - WHERE ts > now()", - EXACT, - ) - .await; - let (fields, rows) = run(&plan); - assert_eq!( - fields, - planned_output(&plan), - "the executor's output type is the plan's" - ); - assert_eq!(rows.len(), 1); - assert!(matches!(rows[0][0], Value::Null), "{:?}", rows[0][0]); -} - -/// A 5 and a 15 minute distinct count over one table: the plan builds them -/// in tumbling windows, each tumbling window a `TimeRange` over a shifted table `Scan`, -/// which the executor runs as filters on `ts`. Both answer exactly (the -/// sketch is exact below its capacity). -#[tokio::test] -async fn distinct_counts_over_5_and_15_minute_windows_run_from_tumbling_windows() { - let fifteen = DISTINCT_USERS_LAST_5_MINUTES.replace("5 minutes", "15 minutes"); - let plan = plan_all(&[DISTINCT_USERS_LAST_5_MINUTES, &fifteen], APPROXIMATE).await; - let selected = format!("{:?}", plan.dag.nodes); - assert!( - selected.contains("TimeShift"), - "tumbling windows: {selected}" - ); - assert_eq!(by_key(run_query(&plan, 0).1), expected_distinct_users()); - // The last 15 minutes (i >= 300): every user `i % 50` of each service - // (3 and 50 are coprime) and its 7 recent ones. - let expected: BTreeMap = (0..3) - .map(|s| (format!("s{s}"), shown(&Value::Int64(57)))) - .collect(); - assert_eq!(by_key(run_query(&plan, 1).1), expected); -} - -/// Exact row counts over 5 and 15 minutes: a count tells a row read twice -/// or missed at a tumbling window's edge, which a distinct count cannot. Each tumbling window is -/// `(T - s - w, T - s]`, so the 15-minute tumbling windows read every row after -/// `T - 15m` exactly once: 300 per service (the row at `T - 15m` is not -/// read), and 100 in the last 5 minutes. -#[tokio::test] -async fn row_counts_over_5_and_15_minute_windows_read_each_row_once() { - let five = "SELECT service, COUNT(*) AS value FROM http \ - WHERE ts >= now() - INTERVAL '5 minutes' GROUP BY service"; - let fifteen = five.replace("5 minutes", "15 minutes"); - let plan = plan_all(&[five, &fifteen], EXACT).await; - let selected = format!("{:?}", plan.dag.nodes); - assert!( - selected.contains("TimeShift"), - "tumbling windows: {selected}" - ); - for (query, rows_per_service) in [(0, 100), (1, 300)] { - let expected: BTreeMap = (0..3) - .map(|s| (format!("s{s}"), shown(&Value::Int64(rows_per_service)))) - .collect(); - assert_eq!(by_key(run_query(&plan, query).1), expected, "query {query}"); - } -} - /// Runs `ids` of `dag` in `scope` with each of `inputs` bound to the node /// its key names and `table` bound as `http`: the batches of each id's /// output. @@ -546,7 +298,7 @@ fn run_data_plane(plan: &InstallablePlan) -> Vec>> { .collect() } -/// A 5 and a 15 minute distinct count kept in tumbling windows at ingestion +/// A 5 and a 15 minute distinct count materialized in tumbling windows at ingestion /// time (the deployment keeps no raw data): each tumbling window is built in an ingestion run from the table rows of its /// window, and both queries answer exactly from the tumbling windows (the sketch is /// exact below its capacity), as the query-time reference does. @@ -641,106 +393,3 @@ async fn a_window_equal_to_its_interval_is_built_at_ingestion_time_from_table_ro .collect(); assert_eq!(by_key(run_data_plane(&plan).remove(0)), expected); } - -/// Each run's requested `(start, end]` time range, in order. -type RequestedTimeRanges = std::sync::Mutex>>; - -/// A store whose `http` table records each run's requested time range and -/// returns every row (it ignores the request). -struct RecordingStore { - rows: InMemoryDataStore, - requested_time_ranges_ms: Arc, -} -impl DataStore for RecordingStore { - fn read_raw_data( - &self, - source: &Source, - request: &RawDataRequest, - context: RunContext, - ) -> Result, asap_executor::Error> { - self.requested_time_ranges_ms - .lock() - .unwrap() - .push(request.time_range_ms); - self.rows.read_raw_data(source, request, context) - } - fn read_materialized_view( - &self, - view: &MaterializedView, - request: &MaterializedViewRequest, - context: RunContext, - ) -> Result, asap_executor::Error> { - self.rows.read_materialized_view(view, request, context) - } -} - -// The store's `http` table of a `ts >= now() - INTERVAL '5 minutes'` query is -// asked for `(T - 5m, T]` at evaluation time `T`; it returns all 20 minutes -// of rows, and the answer is the one over the window. -#[tokio::test] -async fn a_data_store_is_asked_for_the_query_window() { - // The user's query, planned exactly over the `http` table. - let plan = plan( - "SELECT service, COUNT(*) AS value FROM http \ - WHERE ts >= now() - INTERVAL '5 minutes' GROUP BY service", - EXACT, - ) - .await; - // The plan with every node at query time: it reads the table at the Scan. - let reference = plan.dag.with_every_node_at_query_time().unwrap(); - // The schema the plan's Scan reads. - let schema = reference - .nodes - .iter() - .find_map(|node| match &node.payload { - PhysicalASAPOperatorPayload::NonASAP(NonASAPOp::Scan { schema, .. }) => { - Some(Arc::new(schema.clone())) - } - _ => None, - }) - .unwrap(); - // Each run's requested time range, as the store saw it. - let requested_time_ranges_ms = Arc::new(RequestedTimeRanges::default()); - // The deployment's store: `http` → its rows, recording each request. - let store = RecordingStore { - rows: InMemoryDataStore::default() - .with_raw_data( - Source::Table { - table_ref: "http".into(), - }, - schema.clone(), - vec![Batch::try_new(schema, rows()).unwrap()], - ) - .unwrap(), - requested_time_ranges_ms: requested_time_ranges_ms.clone(), - }; - // Compiled once (no reader opened), then instantiated for this run. - let roots = [reference.roots[0] as u64]; - let compiled = compile(&reference, &[], &roots).unwrap(); - let bound = compiled.instantiate(Arc::new(store)).unwrap(); - let context = RunContext::new( - Scope::Query { - evaluation_time_ms: EVALUATION_MS, - }, - Limits::default(), - ) - .unwrap(); - // The answer rows: each service and its count. - let rows = block_on(async { - let mut output = bound.execute(&roots, context).unwrap().remove(0); - let mut rows = Vec::new(); - while let Some(batch) = output.next().await { - rows.extend(batch.unwrap().to_rows()); - } - rows - }); - assert_eq!( - *requested_time_ranges_ms.lock().unwrap(), - vec![Some((EVALUATION_MS - 5 * MINUTE_MS, EVALUATION_MS))] - ); - // The 300 rows of the window, 100 per service. - let expected: BTreeMap = (0..3) - .map(|s| (format!("s{s}"), shown(&Value::Int64(100)))) - .collect(); - assert_eq!(by_key(rows), expected); -} diff --git a/crates/planner/tests/summary_sharing.rs b/crates/planner/tests/summary_sharing.rs index bc7658d28..d4d2dc649 100644 --- a/crates/planner/tests/summary_sharing.rs +++ b/crates/planner/tests/summary_sharing.rs @@ -4,20 +4,18 @@ //! plan selection still selects an exact plan for at least one of their //! queries (#580). -use asap_types::ir::{ASAPOp, OperatorNode, QueryRoot}; +use asap_types::ir::{ASAPOp, OperatorNode}; use std::rc::Rc; -use asap_accuracy::test_support::{default_size_params, DEFAULT_DELTA}; -use asap_frontend_promql::{lower_promql_query_workload, promql_metric_names}; +use asap_frontend_promql::promql_metric_names; use asap_frontend_sql::SqlCatalog; -use asap_plan_selection::reference_pricer::WholePlanSelection; -use asap_plan_selection::RejectionCause; + use asap_planner::{e2e_plan, FrontendInput, UserInput}; use asap_planner::{PlanOutput, PlanningModels, QueryPlan}; -use asap_types::ir::operator::agg_intent::default_quantile; + use asap_types::ir::properties::{AccuracyMetric, AccuracyTarget}; use asap_types::ir::schema::{DataType, Field, Schema}; -use asap_types::ir::schema::{FieldDataType, SketchAlgorithm, SketchParams}; + use asap_types::workload::SqlDialect; use asap_types::workload::{ DataArrival, DataWorkload, DurationMs, PlanningWorkload, Predictability, Query, QueryLanguage, @@ -86,166 +84,6 @@ async fn plan_promql(queries: &[(&str, f64)]) -> PlanOutput { e2e_plan(input).await.expect("workload plans") } -/// Explicit exact, all-KLL, and all-DDS assignments for composition checks. -/// These fixtures test candidate support and rejection independently of which -/// candidates a bounded search happens to visit. -fn composed_summary_candidates( - queries: &[(&str, f64)], -) -> ( - Vec, - WholePlanSelection, -) { - use asap_logical_asap_optimization::test_support::{ - compose_logical_candidate, generate_per_computation_candidates, - }; - use asap_types::stages::HowToComputeAggregateMeasure; - let workload = promql_workload(queries); - let roots = lower_promql_query_workload(&workload).expect("lowers"); - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&roots), - ) - .unwrap(); - let mut assignments = vec![vec![0; inventory.targets.len()]]; - for family in [SketchAlgorithm::Kll, SketchAlgorithm::DDSketch] { - let choices: Vec<_> = inventory - .targets - .iter() - .map(|target| { - target - .ways_to_compute - .iter() - .position(|candidate| { - *candidate == HowToComputeAggregateMeasure::Summary(family) - }) - .unwrap_or(0) - }) - .collect(); - assert_eq!( - choices.iter().filter(|choice| **choice != 0).count(), - 2, - "both quantile references must explicitly use {family:?}" - ); - assignments.push(choices); - } - let entries = workload.query_workload.entries().collect::>(); - let data = workload.data_workload.as_ref().unwrap(); - let models = PlanningModels::builtin(); - let candidates: Vec<_> = assignments - .into_iter() - .enumerate() - .map(|(index, choices)| { - let roots: Vec<_> = compose_logical_candidate(&inventory, &choices).unwrap(); - let mut candidate = - asap_physical_asap_optimization::test_support::optimize_physical_asap_chosen( - "explicit-composition", - &roots, - &entries, - data, - models.deployment_capabilities.query_time_results_retained, - &mut |_, _, _| Some(asap_types::physical::Materialization::NotMaterialized), - ) - .unwrap() - .expect("supported physical computation"); - candidate.id = format!("composition-{index}"); - candidate - }) - .collect(); - let selection = select_plan(&candidates, &entries, data, models) - .expect("the exact candidate meets the requested accuracy"); - (candidates, selection) -} - -/// Construct a legal shared-KLL candidate explicitly, independent of the -/// plan the search selects. -fn shared_kll_candidate(workload: &PlanningWorkload) -> asap_types::physical::PhysicalWorkloadPlan { - use asap_logical_asap_optimization::test_support::{ - compose_logical_candidate, generate_per_computation_candidates, - }; - use asap_types::physical::Materialization; - let roots = lower_promql_query_workload(workload).unwrap(); - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&roots), - ) - .unwrap(); - let mut choices = vec![0; inventory.targets.len()]; - for index in 0..choices.len() { - choices[index] = (1..inventory.targets[index].choice_count()).find(|choice| { - let mut trial = vec![0; choices.len()]; - trial[index] = *choice; - compose_logical_candidate(&inventory, &trial).unwrap().iter().any(|root| { - root.operator_refs().into_iter().flat_map(OperatorNode::reachable).any(|node| { - matches!(node.asap(), Some(ASAPOp::SummaryAgg { family: FieldDataType::Sketch(kind, _), .. }) if matches!(kind.params(), SketchParams::Kll { .. })) - }) - }) - }).expect("KLL implementation exists"); - } - let roots: Vec<_> = compose_logical_candidate(&inventory, &choices).unwrap(); - // Every identical sub-DAG of the two queries made one node. - let roots: Vec<_> = asap_logical_asap_optimization::test_support::share_common_sub_dags( - roots - .into_iter() - .enumerate() - .map(|(id, root)| match root { - QueryRoot::Operator(node) => (id, node), - QueryRoot::Scalar(_) => panic!("operator root"), - }) - .collect(), - ) - .into_iter() - .map(|(_, node)| QueryRoot::Operator(node)) - .collect(); - let mut physical = - asap_physical_asap_optimization::test_support::optimize_physical_asap_chosen( - "shared-kll", - &roots, - &workload.query_workload.entries().collect::>(), - workload.data_workload.as_ref().unwrap(), - false, - &mut |_, _, _| Some(Materialization::NotMaterialized), - ) - .unwrap() - .unwrap(); - physical.id = "shared-kll".into(); - select_plan( - std::slice::from_ref(&physical), - &workload.query_workload.entries().collect::>(), - workload.data_workload.as_ref().unwrap(), - PlanningModels::builtin(), - ) - .expect("shared candidate meets both accuracy targets"); - physical -} - -fn shared_kll_output(workload: &PlanningWorkload) -> PlanOutput { - let candidate = shared_kll_candidate(workload); - PlanOutput::new( - candidate - .roots - .into_iter() - .enumerate() - .map(|(entry_index, root)| { - let QueryRoot::Operator(root) = root else { - panic!("operator fixture") - }; - QueryPlan { entry_index, root } - }) - .collect(), - ) -} - -/// Whether `root` reads a sketch state. -fn reads_sketch(root: &Rc) -> bool { - OperatorNode::reachable(root).into_iter().any(|node| { - matches!( - node.asap(), - Some(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(..), - .. - }) - ) - }) -} - async fn plan_sql(queries: &[&str]) -> PlanOutput { let workload = PlanningWorkload { query_workload: QueryWorkload { @@ -330,153 +168,6 @@ fn unique_deployments(output: &PlanOutput) -> usize { seen.len() } -/// An explicitly connected KLL candidate shares one physical producer and -/// satisfies both outputs' accuracy contracts. -#[test] -fn quantiles_with_equal_params_share_one_producer() { - let workload = promql_workload(&[ - ("quantile_over_time(0.5, lat[5m])", 0.01), - ("quantile_over_time(0.99, lat[5m])", 0.01), - ]); - let cheapest = shared_kll_candidate(&workload); - let states: Vec>> = cheapest - .roots - .iter() - .map(|root| { - root.operator_refs() - .into_iter() - .flat_map(OperatorNode::reachable) - .filter(|node| matches!(node.asap(), Some(ASAPOp::SummaryAgg { .. }))) - .collect() - }) - .collect(); - assert_eq!(states[0].len(), 1, "{}", cheapest.id); - assert!(same_states(&states), "{}", cheapest.id); -} - -/// Shared producer identity survives physicalization for a high-volume source; -/// this checks feasibility, independently of the cost optimizer's winner. -#[tokio::test] -async fn quantiles_with_equal_params_deploy_one_producer() { - let mut workload = promql_workload(&[ - ("quantile_over_time(0.5, lat[5m])", 0.01), - ("quantile_over_time(0.99, lat[5m])", 0.01), - ]); - let data = workload.data_workload.as_mut().unwrap(); - data.arrival = DataArrival::Ingesting { - rows_per_second: 666_666.67, - }; - let lat = data.sources.get_mut("lat").unwrap(); - lat.series_count = Some(1_000_000); - lat.sample_interval = Some(DurationMs(1_500)); - let output = shared_kll_output(&workload); - assert!(same_states(&states(&output))); - assert!(!Rc::ptr_eq(&output.plans[0].root, &output.plans[1].root)); - assert_eq!(unique_deployments(&output), 1); -} - -/// A different window or label selector is a different producer, even when -/// one query asks for a stricter accuracy than the other. -#[tokio::test] -#[ignore = "plan selection still selects an exact plan for at least one query: with the built-in cost model its query-time summary costs more than exact (#580)"] -async fn different_producers_are_not_shared() { - for queries in [ - [ - ("quantile_over_time(0.5, lat[5m])", 0.01), - ("quantile_over_time(0.99, lat[10m])", 0.01), - ], - [ - ("quantile_over_time(0.5, lat[5m])", 0.01), - ("quantile_over_time(0.99, lat[10m])", 0.001), - ], - [ - ("quantile_over_time(0.5, lat{job=\"a\"}[5m])", 0.01), - ("quantile_over_time(0.99, lat{job=\"b\"}[5m])", 0.001), - ], - [ - ("quantile_over_time(0.5, lat{job=\"a\"}[5m])", 0.01), - ("quantile_over_time(0.99, lat{job=\"b\"}[5m])", 0.01), - ], - ] { - let output = plan_promql(&queries).await; - assert!(!same_states(&states(&output)), "{queries:?}"); - assert_eq!(unique_deployments(&output), 2, "{queries:?}"); - for (plan, (_, epsilon)) in output.plans.iter().zip(queries) { - assert_eq!(kll_k(plan), kll_k_for(epsilon), "{queries:?}"); - } - } -} - -/// The KLL `k` of the one state a plan deploys. -fn kll_k(plan: &QueryPlan) -> u32 { - let states = plan_states(plan); - let [deployment] = states.as_slice() else { - panic!("one state: {:?}", states.len()); - }; - let asap_types::ir::Operator::ASAP(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), - .. - }) = &deployment.operator - else { - panic!("sketch state: {:?}", deployment.operator); - }; - let SketchParams::Kll { k, .. } = kind.params() else { - panic!("KLL state: {kind:?}"); - }; - *k -} - -/// The KLL `k` sized for `epsilon`. -fn kll_k_for(epsilon: f64) -> u32 { - let SketchParams::Kll { k, .. } = default_size_params( - SketchAlgorithm::Kll, - &default_quantile(0.5), - epsilon, - DEFAULT_DELTA, - ) else { - unreachable!() - }; - k -} - -/// p50 at ε=0.01 and p99 at ε=0.001 over the same input share one KLL sized -/// for the strictest consumer; each reader's guarantee meets its own target. -#[tokio::test] -#[ignore = "plan selection still selects an exact plan for at least one query: with the built-in cost model its query-time summary costs more than exact (#580)"] -async fn quantiles_share_one_producer_sized_for_the_strictest_consumer() { - let p50 = ("quantile_over_time(0.5, lat[5m])", 0.01); - let p99 = ("quantile_over_time(0.99, lat[5m])", 0.001); - assert!(kll_k_for(0.001) > kll_k_for(0.01)); - - let output = plan_promql(&[p50, p99]).await; - assert!(same_states(&states(&output))); - assert_eq!(unique_deployments(&output), 1); - for (plan, (_, epsilon)) in output.plans.iter().zip([p50, p99]) { - assert_eq!(kll_k(plan), kll_k_for(0.001)); - let guarantee = plan.root.guarantee.as_ref().expect("certified"); - assert!( - guarantee.spec.error.evaluate().unwrap() <= epsilon, - "{guarantee:?}" - ); - } - - // Alone, the looser query keeps its own, smaller KLL. - let alone = plan_promql(&[p50]).await; - assert_eq!(kll_k(&alone.plans[0]), kll_k_for(0.01)); -} - -/// Cross-series quantiles name their KLL state after the input column, not the -/// quantile, so p50 and p99 over one selector share it. -#[tokio::test] -async fn cross_series_p50_and_p99_share_one_producer() { - let output = shared_kll_output(&promql_workload(&[ - ("quantile(0.5, lat)", 0.01), - ("quantile(0.99, lat)", 0.01), - ])); - assert!(same_states(&states(&output))); - assert_eq!(unique_deployments(&output), 1); -} - /// Two identical ungrouped queries: their identical producers (rate, then /// sum) are shared. #[tokio::test] @@ -547,124 +238,6 @@ async fn sql_p50_and_p99_share_one_producer() { } } -/// The operator sub-DAG a standalone scalar query reads is planned like an -/// operator root (#511 §2.3): the scalar root is rebuilt over its selected -/// sub-DAG, and entries keep their order around a literal scalar query. -#[tokio::test] -async fn scalar_queries_read_planned_sub_dags() { - let output = plan_promql(&[ - ("1", 0.01), - ("scalar(quantile(0.99, rate(x[5m])))", 0.01), - ("quantile(0.99, rate(x[5m]))", 0.01), - ]) - .await; - assert_eq!(output.entry_indices(), vec![0, 1, 2]); - let scalar_entries: Vec<_> = output.scalar_roots.iter().map(|(i, _)| *i).collect(); - assert_eq!(scalar_entries, vec![0, 1]); - assert_eq!(output.plans.len(), 1); - assert_eq!(output.plans[0].entry_index, 2); - - assert_eq!(output.scalar_plans.len(), 1); - let read = &output.scalar_plans[0]; - assert_eq!(read.entry_index, 1); - let refs = output.scalar_roots[1].1.operator_refs(); - assert_eq!(refs.len(), 1); - assert!(Rc::ptr_eq(refs[0], &read.root)); - - // The export keys each scalar query by its entry and points the scalar - // read at an exported node. - let dag = output.physical_asap_dag().unwrap(); - let keys: Vec<_> = dag.scalar_roots.iter().map(|(i, _)| *i).collect(); - assert_eq!(keys, vec![0, 1]); - assert_eq!(dag.roots.len(), 1); - let [read] = dag.scalar_roots[1].1.operator_refs()[..] else { - panic!("one read"); - }; - assert!(dag.nodes.iter().any(|node| node.id == *read)); -} - -/// A combination of approximate values keeps its summary candidates: each -/// reference of `scalar(q99) - scalar(q50)` is planned against the entry's -/// target, so a candidate reads a sketch in both. A difference of rank (KLL) -/// or relative (DDSketch) errors composes to no guarantee in the MVP (#669) -/// and the built-in models have no empirical output-level evidence, so plan -/// selection rejects it with that reason. -#[test] -fn combined_scalar_references_keep_summary_candidates() { - let (candidates, selection) = composed_summary_candidates(&[( - "scalar(quantile(0.99, rate(x[5m]))) - scalar(quantile(0.5, rate(x[5m])))", - 0.01, - )]); - for both in candidates.iter().skip(1) { - let refs = both.roots[0].operator_refs(); - assert_eq!(refs.len(), 2); - assert!( - refs.into_iter().all(reads_sketch), - "both references use the explicit sketch family" - ); - let rejection = selection - .rejected - .iter() - .find(|r| r.id == both.id) - .expect("not selected"); - assert_eq!( - rejection.cause, - RejectionCause::NoAccuracyGuarantee, - "{rejection:?}" - ); - assert!( - rejection.reason.contains("no guarantee"), - "{}", - rejection.reason - ); - } - assert_eq!( - selection.selected, candidates[0].id, - "the exact assignment is selected" - ); -} - -/// Candidate generation supports summary implementations of the vector -/// operands, but a difference or a comparison of -/// estimates composes to no guarantee in the MVP (#669; a comparison's band -/// guarantee is later work) and the built-in models have no empirical -/// output-level evidence (plan selection, #576), so it is not selected -/// automatically: the exact plan is. -#[tokio::test] -async fn combined_vector_quantiles_keep_unselected_summary_candidates() { - for query in [ - "quantile(0.99, rate(x[5m])) - quantile(0.5, rate(x[5m]))", - "quantile(0.99, rate(x[5m])) > quantile(0.5, rate(x[5m]))", - ] { - let (candidates, selection) = composed_summary_candidates(&[(query, 0.01)]); - for summary in candidates.iter().skip(1) { - let QueryRoot::Operator(root) = &summary.roots[0] else { - panic!("{query}: operator root"); - }; - assert!(reads_sketch(root), "{query}: explicit summary candidate"); - assert!(root.guarantee.is_none(), "{query}"); - let rejection = selection - .rejected - .iter() - .find(|rejection| rejection.id == summary.id) - .expect("uncertified vector computation is rejected"); - assert_eq!( - rejection.cause, - RejectionCause::NoAccuracyGuarantee, - "{query}: {rejection:?}" - ); - assert!(rejection.reason.contains("no guarantee"), "{rejection:?}"); - } - assert_eq!( - selection.selected, candidates[0].id, - "{query}: exact assignment selected" - ); - - let output = plan_promql(&[(query, 0.01)]).await; - assert!(!reads_sketch(&output.plans[0].root), "{query}"); - } -} - /// Front ends build structurally identical references as separate nodes. /// The selected graph may reuse a producer or choose separate physical /// candidates. Every distinct selected reference must be exported exactly once. @@ -698,71 +271,3 @@ async fn identical_scalar_references_satisfy_the_plan_contract() { } } } - -/// A root whose selected candidate is an exact state is a query result: it -/// is read through `FinalizeExactAccumulator`, so its output is plain values, -/// not accumulator state. -#[tokio::test] -async fn exact_state_roots_are_finalized() { - for query in [ - "sum_over_time(m[1m])", - "sum(rate(x[5m]))", - "sum(sum_over_time(m[1m]) + sum_over_time(n[1m]))", - ] { - let output = plan_promql(&[(query, 0.01)]).await; - let root = &output.plans[0].root; - assert!( - matches!(root.asap(), Some(ASAPOp::FinalizeExactAccumulator { .. })), - "{query}: {root:#?}" - ); - let dag = output.physical_asap_dag().unwrap(); - let root = dag - .nodes - .iter() - .find(|node| node.id == dag.roots[0]) - .unwrap(); - assert!( - root.output_schema - .fields - .iter() - .all(|field| field.is_plain()), - "{query}: {:?}", - root.output_schema - ); - // With nothing materialized, the root and the exact state it reads - // run at query time. - use asap_types::ir::properties::{DataPrimitive, ExecutionDataState, ExecutionTiming}; - assert_eq!(root.output_state, ExecutionDataState::QUERY_ROWS, "{query}"); - let state = dag - .edges - .iter() - .find(|edge| edge.consumer == root.id) - .map(|edge| edge.producer) - .unwrap(); - let state = dag.nodes.iter().find(|node| node.id == state).unwrap(); - assert_eq!( - state.output_state, - ExecutionDataState { - timing: ExecutionTiming::QueryTime, - primitive: DataPrimitive::SummaryState, - }, - "{query}" - ); - } -} - -/// The reference pricer over `candidates`, every candidate kept. -fn select_plan( - candidates: &[asap_types::physical::PhysicalWorkloadPlan], - entries: &[asap_types::workload::QueryWorkloadEntry], - data: &asap_types::workload::DataWorkload, - models: PlanningModels<'_>, -) -> Result { - asap_plan_selection::reference_pricer::select_among_plans( - candidates, - entries, - data, - &models.accuracy_check_inputs(), - models.cost_inputs, - ) -} diff --git a/crates/types/src/ir/flat.rs b/crates/types/src/ir/flat.rs index ac56e5b5f..f5c940069 100644 --- a/crates/types/src/ir/flat.rs +++ b/crates/types/src/ir/flat.rs @@ -15,10 +15,8 @@ use serde::{Deserialize, Serialize}; use crate::ir::operator::node::{Operator, OperatorNode, OperatorResultKind}; use crate::ir::operator::query::QueryRoot; -use crate::ir::properties::execution_data_state::ExecutionTiming; use crate::ir::properties::guarantee::AccuracyGuarantee; use crate::ir::schema::Schema; -use crate::physical::materialization::Materialization; /// Index of a node in [`FlatDag::nodes`]. pub type NodeId = usize; @@ -30,8 +28,6 @@ pub struct FlatNode { pub(crate) result_kind: OperatorResultKind, pub(crate) schema: Schema, pub(crate) guarantee: Option, - pub(crate) timing: Option, - pub(crate) materialization: Materialization, } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] @@ -78,8 +74,6 @@ fn visit( result_kind: node.result_kind, schema: node.schema.clone(), guarantee: node.guarantee.clone(), - timing: node.timing, - materialization: node.materialization, }); originals.push(Rc::clone(node)); ids.insert(Rc::as_ptr(node), id); @@ -165,7 +159,6 @@ mod tests { #[test] fn json_round_trips() { let (dag, _) = flatten(&[QueryRoot::Operator(filter(values()))]); - assert!(dag.nodes.iter().all(|n| n.timing.is_none())); let json = serde_json::to_string(&dag).unwrap(); assert_eq!(serde_json::from_str::(&json).unwrap(), dag); } diff --git a/crates/types/src/ir/materialized_view.rs b/crates/types/src/ir/materialized_view.rs index 6a3c2de29..d9971be0c 100644 --- a/crates/types/src/ir/materialized_view.rs +++ b/crates/types/src/ir/materialized_view.rs @@ -5,12 +5,12 @@ //! `retention_ms` installation sets on its nodes //! ([`PhysicalASAPDAG::set_retention`]). //! -//! A **materialized view** is a node whose output is kept across -//! evaluations: its materialization is `MatAtIngestion` or -//! `MatAtQueryKeptAcrossEvaluations`, and it carries its `retention_ms`. A -//! `MatAtQuery` node is not a view: its output is discarded after one +//! A **materialized view** is a node whose output is materialized across +//! evaluations: its lifetime is `AcrossEvaluations`, at ingestion time or +//! at query time, and it carries its `retention_ms`. A node materialized for +//! `OneEvaluation` is not a view: its output is discarded after one //! evaluation. A query-time `Scan` carries how long its source's rows are -//! kept for it (`None` when it reads them unbounded). +//! retained for it (`None` when it reads them unbounded). //! //! ## Refreshing a view //! @@ -79,14 +79,14 @@ use crate::ir::operator::non_asap::NonASAPOp; use crate::ir::operator::operator_properties::Source; use crate::ir::properties::execution_data_state::ExecutionTiming; use crate::ir::properties::summary_coverage::{CoverageError, SelectionBox, SummaryCoverage}; -use crate::physical::materialization::Materialization; +use crate::physical::Lifetime; /// One materialized view of a DAG: what its stored state is, what it is -/// refreshed from, when, and for how long it is kept. +/// refreshed from, when, and for how long it is retained. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct MaterializedView { - /// The node whose output is kept. + /// The node whose output is materialized. pub node_id: PhysicalASAPNodeId, /// Which run refreshes it. pub(crate) refresh: MaterializedViewRefresh, @@ -235,13 +235,13 @@ impl RawDataWindow { #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum MaterializedViewRefresh { - /// Refreshed at ingestion time (`MatAtIngestion`): the runtime runs the + /// Refreshed at ingestion time (materialized at ingestion time): the runtime runs the /// node in an ingestion run once its inputs are complete (its windows' /// samples have arrived and the views it reads are refreshed), and /// keeps the result. AtIngestion, - /// Refreshed at query time and kept across evaluations - /// (`MatAtQueryKeptAcrossEvaluations`): when an evaluation reads this + /// Refreshed at query time and materialized across evaluations + /// (query-time `AcrossEvaluations`): when an evaluation reads this /// view for a selection it has not been refreshed for yet, the runtime /// runs the node in a query run over its inputs, keeps the result, and /// binds it like a view refreshed at ingestion. @@ -375,18 +375,18 @@ pub enum InstallableDagError { /// Bad input (a plan file): view `node_id` has no `retention_ms`. #[error("materialized view {node_id} has no retention_ms")] ViewWithoutRetention { node_id: PhysicalASAPNodeId }, - /// Bad input (a plan file): view `node_id` is kept for less than the + /// Bad input (a plan file): view `node_id` is retained for less than the /// data it summarizes reaches back, so a later evaluation would miss /// it. - #[error("materialized view {node_id} is kept for less than its data reaches back")] + #[error("materialized view {node_id} is retained for less than its data reaches back")] ViewRetentionTooShort { node_id: PhysicalASAPNodeId }, /// Bad input (a plan file): query-time `Scan` `node_id` reads a bounded /// window but has no `retention_ms`. #[error("query-time scan {node_id} reads a bounded window but has no retention_ms")] ScanWithoutRetention { node_id: PhysicalASAPNodeId }, /// Bad input (a plan file): query-time `Scan` `node_id` keeps its rows - /// for less than its readers' window reaches back. - #[error("query-time scan {node_id} is kept for less than its window reaches back")] + /// for less than its consumers' window reaches back. + #[error("query-time scan {node_id} is retained for less than its window reaches back")] ScanRetentionTooShort { node_id: PhysicalASAPNodeId }, } @@ -398,11 +398,9 @@ impl PhysicalASAPDAG { /// covers how far back its data reaches. O(views · nodes) for the /// coverages, plus O(nodes + edges). pub fn materialized_views(&self) -> Result, InstallableDagError> { - if let Some(node) = self - .nodes - .iter() - .find(|node| node.materialization.is_materialized_view() && node.retention_ms.is_none()) - { + if let Some(node) = self.nodes.iter().find(|node| { + matches!(node.lifetime, Lifetime::AcrossEvaluations(_)) && node.retention_ms.is_none() + }) { return Err(InstallableDagError::ViewWithoutRetention { node_id: node.id }); } self.validate()?; @@ -571,25 +569,25 @@ impl PhysicalASAPDAG { /// /// A view takes `tumbling_window_retention_ms_by_node`'s value (Stage 3's retention /// of a tumbling window's chain: its offset plus its window plus the longest - /// range-evaluation span of its readers); otherwise how far back its - /// data reaches, plus, for a view built from other views, its readers' + /// range-evaluation span of its consumers); otherwise how far back its + /// data reaches, plus, for a view built from other views, its consumers' /// longest range-evaluation span (`reader_span_ms_by_node`). Then each - /// view is kept at least as long as every view built from it needs it: + /// view is retained at least as long as every view built from it needs it: /// as long as that view's own retention when it is refreshed on first /// read (a range request may refresh it for its oldest step), else as /// far back as that view's data reaches. This is each node's own /// requirement: a runtime that stores identical outputs of several /// nodes once keeps them for the largest requirement among those nodes. /// - /// A query-time `Scan` keeps its rows for its readers' window + /// A query-time `Scan` keeps its rows for its consumers' window /// ([`Self::scan_window`]'s `lookback_ms`) plus their longest /// range-evaluation span, and at least the retention of every view /// refreshed on first read from it (which may be refreshed for its /// oldest step); `None` when its window is unbounded. In an installed - /// DAG a chain kept across evaluations is its expanded tumbling windows, so its + /// DAG a chain materialized across evaluations is its expanded tumbling windows, so its /// scan keeps the reach of all of them: reusing a tumbling window refreshed at an /// earlier evaluation is optional, and a restart or an older range step - /// refreshes older tumbling windows from raw data. (Stage 3 prices a kept chain on + /// refreshes older tumbling windows from raw data. (Stage 3 prices a materialized chain on /// the unexpanded DAG as reading its newest tumbling window.) pub fn set_retention( &mut self, @@ -625,8 +623,8 @@ impl PhysicalASAPDAG { MaterializedViewRefresh::AtIngestion => reach_ms_by_node[&view.node_id], }; for input in view.inputs.materialized_views.clone() { - let kept = &mut views[position_of[&input]].retention_ms; - *kept = (*kept).max(needed_ms); + let retained = &mut views[position_of[&input]].retention_ms; + *retained = (*retained).max(needed_ms); } } let mut retention_ms_by_node: HashMap> = views @@ -687,9 +685,13 @@ impl PhysicalASAPDAG { return Err(InstallableDagError::ScanWithoutRetention { node_id: node.id }) } }; - match retention.iter_mut().find(|kept| &kept.source == source) { - Some(kept) => { - kept.lookback_ms = kept.lookback_ms.zip(lookback_ms).map(|(a, b)| a.max(b)) + match retention + .iter_mut() + .find(|retained| &retained.source == source) + { + Some(retained) => { + retained.lookback_ms = + retained.lookback_ms.zip(lookback_ms).map(|(a, b)| a.max(b)) } None => retention.push(RawDataRetention { source: source.clone(), @@ -700,7 +702,7 @@ impl PhysicalASAPDAG { Ok(retention) } - /// [`Self::scan_window`] of each query-time `Scan`, with no chain kept, + /// [`Self::scan_window`] of each query-time `Scan`, with no chain materialized, /// by id. fn query_time_scan_windows(&self) -> HashMap { let scans = self @@ -730,7 +732,7 @@ impl PhysicalASAPDAG { let views: HashSet = self .nodes .iter() - .filter(|node| node.materialization.is_materialized_view()) + .filter(|node| matches!(node.lifetime, Lifetime::AcrossEvaluations(_))) .map(|node| node.id) .collect(); // The IR of each node, which coverage is derived from. @@ -764,7 +766,8 @@ impl PhysicalASAPDAG { { window_by_view.insert(id, window); } - if matches!(node.materialization, Materialization::MatAtIngestion(_)) { + let refreshed_at_ingestion = node.output_state.timing == ExecutionTiming::IngestionTime; + if refreshed_at_ingestion { // An ingestion run reads each view for the window ending at // its end, so a view read must end where the raw windows do. let mut end_offsets_ms: Vec = inputs @@ -796,9 +799,10 @@ impl PhysicalASAPDAG { coverage_by_node.insert(id, coverage.clone()); let view = MaterializedView { node_id: id, - refresh: match node.materialization { - Materialization::MatAtIngestion(_) => MaterializedViewRefresh::AtIngestion, - _ => MaterializedViewRefresh::OnFirstRead, + refresh: if refreshed_at_ingestion { + MaterializedViewRefresh::AtIngestion + } else { + MaterializedViewRefresh::OnFirstRead }, coverage: ViewCoverage { definition_id: coverage.definition_id(), @@ -817,7 +821,7 @@ impl PhysicalASAPDAG { /// `inputs` summarizes reaches: the farthest of its raw-data windows and of /// the views it reads (`reach_ms_by_node`, each already computed). /// Whether `node` is a `Scan` run at query time, which carries how long its -/// source's rows are kept for it. +/// source's rows are retained for it. pub(crate) fn is_query_time_scan(node: &PhysicalASAPDAGNode) -> bool { matches!(node.payload, Operator::NonASAP(NonASAPOp::Scan { .. })) && node.output_state.timing == ExecutionTiming::QueryTime @@ -1083,79 +1087,15 @@ impl<'a> ViewIndex<'a> { } } -/// The coverage of view `id` (IR `ir`), refreshed from `inputs`. A merge's -/// is the union of its inputs' selections under their shared definition -/// ([`SummaryCoverage::derive`]). A summary built from exactly one view and -/// nothing else, with no selection of its own (a rollup to a coarser -/// grouping, or across series), takes that input's selection: its -/// definition is its own sub-DAG with the input replaced by the input's -/// definition, so the rollups of equal inputs share one definition. -/// Anything else keeps its own derived coverage. +/// The coverage of view `id` (IR `ir`), refreshed from `inputs`: the +/// coverage its memo group stores, which the realized plan will carry. +/// Coverage is derived only in the memo, never on node trees. fn coverage_of( id: PhysicalASAPNodeId, inputs: &MaterializedViewInputs, ir: &HashMap>, coverage_by_node: &HashMap, ) -> Result { - let node = &ir[&id]; - let own = SummaryCoverage::derive(node)?; - let [input] = inputs.materialized_views.as_slice() else { - return Ok(own); - }; - if !inputs.raw_data_windows.is_empty() - || matches!(node.asap(), Some(ASAPOp::SummaryMerge { .. })) - || own.selection != [SelectionBox::default()] - { - return Ok(own); - } - let input_coverage = &coverage_by_node[input]; - let over_definition = with_replaced(node, &ir[input], &input_coverage.definition); - Ok(SummaryCoverage { - definition: SummaryCoverage::derive(&over_definition)?.definition, - selection: input_coverage.selection.clone(), - }) -} - -/// `node` with every use of `old` (by address) replaced by `new`; nodes -/// that do not reach `old` are shared, not copied. -fn with_replaced( - node: &Rc, - old: &Rc, - new: &Rc, -) -> Rc { - fn replace( - node: &Rc, - old: &Rc, - new: &Rc, - cache: &mut HashMap<*const OperatorNode, Rc>, - ) -> Rc { - if Rc::ptr_eq(node, old) { - return Rc::clone(new); - } - if let Some(done) = cache.get(&Rc::as_ptr(node)) { - return Rc::clone(done); - } - let children = node.children(); - let replaced: Vec> = children - .iter() - .map(|child| replace(child, old, new, cache)) - .collect(); - let result = if children - .iter() - .zip(&replaced) - .all(|(a, b)| Rc::ptr_eq(a, b)) - { - Rc::clone(node) - } else { - let mut next = replaced.into_iter(); - let mut copy = (**node).clone(); - copy.operator = node - .operator - .map_children(|_| next.next().expect("one per child")); - Rc::new(copy) - }; - cache.insert(Rc::as_ptr(node), Rc::clone(&result)); - result - } - replace(node, old, new, &mut HashMap::new()) + let _ = (id, inputs, ir, coverage_by_node); + todo!("#1207 PR 6: the view's coverage from its memo group's GroupProperties::summary_coverage") } diff --git a/crates/types/src/ir/mod.rs b/crates/types/src/ir/mod.rs index 026dc620e..b424f16de 100644 --- a/crates/types/src/ir/mod.rs +++ b/crates/types/src/ir/mod.rs @@ -14,9 +14,3 @@ pub use schema::error::SchemaDerivationError; pub mod flat; pub mod materialized_view; pub mod physical_export; -pub use properties::timing::{ - apply_materialization_timings, apply_materialization_timings_to_roots, data_state, - is_filtered_build, timing_under_consumer, MaterializationAssignment, NodeToTimedNodeCache, -}; -#[cfg(any(test, feature = "test-support"))] -pub use properties::timing::{planned_data_state, validate_at_ingestion}; diff --git a/crates/types/src/ir/operator/asap.rs b/crates/types/src/ir/operator/asap.rs index 707ab4add..c4490bb2d 100644 --- a/crates/types/src/ir/operator/asap.rs +++ b/crates/types/src/ir/operator/asap.rs @@ -5,7 +5,8 @@ use std::rc::Rc; use serde::{Deserialize, Serialize}; -use super::node::{OperatorNode, OperatorResultKind}; +use super::input::{InputOrTemporaryBuild, OperatorInput}; +use super::node::{Operator, OperatorNode, OperatorResultKind}; use crate::ir::operator::maintained_population::{MaintainedPopulation, PopulationStatistic}; use crate::ir::operator::operator_properties::Reduction; use crate::ir::schema::sketch::{GroupingStrategy, SketchStatistic, SummaryUpdate}; @@ -263,7 +264,7 @@ impl ASAPOp { } } -impl ASAPOp { +impl ASAPOp { fn unimplemented() -> SchemaDerivationError { SchemaDerivationError::InvalidScalarSignature(UNIMPLEMENTED_ASAP_OP.into()) } @@ -276,7 +277,7 @@ impl ASAPOp { | ASAPOp::SummaryJoin { family, .. } => Some(family), ASAPOp::SummaryMerge { children, .. } => children.first().and_then(|child| { child - .schema + .schema() .fields .iter() .find(|field| !field.is_plain()) @@ -304,29 +305,12 @@ impl ASAPOp { family, reduction, .. - } => { - let mut schema = crate::ir::schema::aggregate_schema::aggregate_output_schema( - &child.schema, - reduction, - &[crate::ir::operator::AggIntent::Sum { col: None }], - &[], - )?; - let index = match reduction { - Reduction::PerEntity => crate::ir::scalar::resolve_column_ref( - &crate::ir::scalar::ColumnRef::SampleValue, - &schema, - ) - .map_err(|e| SchemaDerivationError::InvalidScalarSignature(e.to_string()))?, - Reduction::Reduce(_) => schema.fields.len() - 1, - }; - schema.fields[index] = Field::new("state", family.clone(), false); - schema - } + } => summary_agg_schema(child.schema(), family, reduction)?, SummaryEstimate { summary_input, query, } => { - let input = &summary_input.schema; + let input = summary_input.schema(); let (name, mut dtype) = match query { SketchStatistic::Quantile { .. } => ("quantile", DataType::Float64), SketchStatistic::Cardinality => ("cardinality", DataType::Int64), @@ -336,7 +320,7 @@ impl ASAPOp { SketchStatistic::TopK { .. } => ("topk", DataType::Utf8), }; if matches!( - summary_input.summary_build(), + summary_build(summary_input), Some( SummaryAgg { reduction: Reduction::PerEntity, @@ -367,107 +351,31 @@ impl ASAPOp { } FinalizeExactAccumulator { child } => { // A rollup finalizes as the coarse build it stands for. - if let Some(build) = rolled_up_build(child) { - return FinalizeExactAccumulator { child: build }.output_schema(); + if let Some((build, schema)) = rolled_up_build(child) { + return finalized_schema(Some(&build), &schema); } // Merged tumbling windows finalize as one build over their window. if super::tumbling_window_chain::merged_chain(child).is_some() { - return FinalizeExactAccumulator { - child: Rc::new(super::tumbling_window_chain::merged_window_build(child)?), - } - .output_schema(); - } - let value_result = if let Some(ASAPOp::SummaryAgg { - child: source, - family: FieldDataType::ExactAggregate(kind, _), - input, - reduction, - filter, - .. - }) = child.summary_build() - { - { - use crate::ir::operator::AggIntent; - use crate::ir::schema::{ExactKind, SummaryInputExpr}; - let column = match &input.weight { - SummaryInputExpr::Column(col) => Some( - crate::ir::scalar::column_resolution::resolve_column_ref( - col, - &source.schema, - ) - .map_err(|e| { - SchemaDerivationError::InvalidScalarSignature(e.to_string()) - })?, - ), - _ => None, - }; - let measure = match kind { - ExactKind::Sum => Some(AggIntent::Sum { col: column }), - ExactKind::Min => Some(AggIntent::Min { col: column }), - ExactKind::Max => Some(AggIntent::Max { col: column }), - ExactKind::Count => Some(AggIntent::Count { - accuracy: crate::ir::properties::AccuracyTarget::exact(), - }), - ExactKind::Rate => Some(AggIntent::Rate), - ExactKind::IRate => Some(AggIntent::IRate), - ExactKind::Increase => Some(AggIntent::Increase), - }; - measure - .map(|measure| { - crate::ir::NonASAPOp::Aggregate { - child: Rc::clone(source), - reduction: reduction.clone(), - measures: vec![measure], - output_names: vec![], - // A filtered SUM/MIN/MAX is NULL for a - // group without a matching row. - filters: vec![filter.clone()], - having: None, - } - .output_schema() - }) - .transpose()? - .and_then(|s| match reduction { - Reduction::PerEntity => { - s.column_id("value").and_then(|i| s.fields.get(i).cloned()) - } - Reduction::Reduce(_) => s.fields.last().cloned(), - }) - } - } else { - None - }; - let mut out = child.schema.clone(); - for f in &mut out.fields { - if let FieldDataType::ExactAggregate(kind, _) = &f.dtype { - if let Some(result) = &value_result { - // The finalized value replaces the aggregate it - // realizes, so it takes that aggregate's column. - f.name = result.name.clone(); - f.dtype = result.dtype.clone(); - f.nullable = result.nullable; - } else { - f.dtype = FieldDataType::Plain(finalized_data_type(kind)); - } - } + let (build, schema) = super::tumbling_window_chain::merged_window_build(child)?; + return finalized_schema(Some(&build), &schema); } - out + finalized_schema(summary_build(child).as_ref(), child.schema())? } - MaintainPopulation { child, .. } => child.schema.clone(), + MaintainPopulation { child, .. } => child.schema().clone(), EvaluatePopulation { child, evaluation } => { use crate::ir::operator::maintained_population::PopulationInput; use crate::ir::operator::{AggIntent, GroupKeys}; - let Some(MaintainPopulation { + let Operator::ASAP(MaintainPopulation { child: source, population, - }) = child.asap() + }) = child.operator() else { return Err(SchemaDerivationError::InvalidScalarSignature( "population evaluation requires maintained membership".into(), )); }; if matches!(evaluation, PopulationStatistic::TopK { .. }) { - source.schema.clone() + source.schema().clone() } else { let (keys, column) = match &population.input { PopulationInput::Rows { @@ -480,7 +388,7 @@ impl ASAPOp { .grouping .iter() .map(|name| { - source.schema.column_id(name).ok_or_else(|| { + source.schema().column_id(name).ok_or_else(|| { SchemaDerivationError::InvalidScalarSignature( "population grouping column is absent".into(), ) @@ -524,7 +432,7 @@ impl ASAPOp { children, group_by, .. } => { self.validate_inputs()?; - let input = &children[0].schema; + let input = children[0].schema(); match group_by { None => input.clone(), Some(keys) => regrouped_schema(input, keys)?, @@ -558,11 +466,11 @@ impl ASAPOp { /// Local input-contract checks. pub(crate) fn validate_inputs(&self) -> Result<(), SchemaDerivationError> { use ASAPOp::*; - let needs_state = |node: &OperatorNode, what: &str| { - if node.result_kind != OperatorResultKind::State { + let needs_state = |node: &I, what: &str| { + if node.result_kind() != OperatorResultKind::State { Err(SchemaDerivationError::InvalidScalarSignature(format!( "{what} requires summary state as input, got {:?}", - node.result_kind + node.result_kind() ))) } else { Ok(()) @@ -581,11 +489,11 @@ impl ASAPOp { } if let Some(n) = newest_tumbling_windows { let tumbling_window_count = match children.as_slice() { - [chain] => match chain.asap() { - Some(SummaryTumblingWindowChain { + [chain] => match chain.operator() { + Operator::ASAP(SummaryTumblingWindowChain { tumbling_window_count, .. - }) => Some(*tumbling_window_count), + }) => Some(tumbling_window_count), _ => None, }, _ => None, @@ -606,7 +514,7 @@ impl ASAPOp { // Matching state parameters and grouping positions are necessary; // matching names alone cannot prove two states compatible. if first - .schema + .schema() .fields .iter() .filter(|field| !field.is_plain()) @@ -619,7 +527,7 @@ impl ASAPOp { } for child in children { needs_state(child, "SummaryMerge")?; - if child.schema != first.schema { + if child.schema() != first.schema() { return Err(SchemaDerivationError::InvalidScalarSignature( "summary merge inputs must have identical state and grouping schemas" .into(), @@ -639,7 +547,7 @@ impl ASAPOp { // The family-capability table is the one source of which // statistic a state's family answers. let states: Vec<_> = summary_input - .schema + .schema() .fields .iter() .filter(|f| !f.is_plain()) @@ -663,7 +571,7 @@ impl ASAPOp { FinalizeExactAccumulator { child } => { needs_state(child, "FinalizeExactAccumulator")?; if child - .schema + .schema() .fields .iter() .all(|f| !matches!(f.dtype, FieldDataType::ExactAggregate(..))) @@ -676,7 +584,7 @@ impl ASAPOp { } EvaluatePopulation { child, evaluation } => { needs_state(child, "EvaluatePopulation")?; - if !matches!(child.asap(), Some(MaintainPopulation { population, .. }) if population.supports(evaluation)) + if !matches!(child.operator(), Operator::ASAP(MaintainPopulation { population, .. }) if population.supports(evaluation)) { return Err(SchemaDerivationError::InvalidScalarSignature( "population evaluation requires compatible maintained membership".into(), @@ -698,12 +606,12 @@ impl ASAPOp { // The chain is the range selector: it reads the series the // selector would, or a SQL table's rows for a trailing window. if !matches!( - child.result_kind, + child.result_kind(), OperatorResultKind::InstantVector | OperatorResultKind::Relation ) { return Err(SchemaDerivationError::InvalidScalarSignature(format!( "a tumbling-window chain reads an instant vector or a relation, got {:?}", - child.result_kind + child.result_kind() ))); } self.summary_agg_inputs() @@ -740,7 +648,7 @@ impl ASAPOp { else { unreachable!("called on a summary build"); }; - if family.is_plain() || child.result_kind == OperatorResultKind::State { + if family.is_plain() || child.result_kind() == OperatorResultKind::State { return Err(SchemaDerivationError::InvalidScalarSignature( "summary aggregation requires values and produces a state family".into(), )); @@ -765,12 +673,12 @@ impl ASAPOp { } Ok(()) } - check(&input.weight, &child.schema)?; + check(&input.weight, child.schema())?; if let Some(item) = &input.item { - check(item, &child.schema)?; + check(item, child.schema())?; } if let Some(filter) = filter { - if filter.0.scalar_type(&child.schema)?.0 != DataType::Bool { + if filter.0.scalar_type(child.schema())?.0 != DataType::Bool { return Err(SchemaDerivationError::InvalidScalarSignature( "summary filter must be boolean".into(), )); @@ -781,26 +689,32 @@ impl ASAPOp { } /// The build a rollup (a `SummaryMerge` with `group_by` of one input) -/// stands for: the input's build (one build over a merged chain's window) -/// grouped by the input columns the rollup's keys name. `None` for any -/// other node, or a rollup whose build does not derive. -fn rolled_up_build(state: &OperatorNode) -> Option> { - let ASAPOp::SummaryMerge { +/// stands for, with its schema: the input's build (one build over a merged +/// chain's window) grouped by the input columns the rollup's keys name. +/// `None` for any other input, or a rollup whose build does not derive. +fn rolled_up_build( + state: &I, +) -> Option<(ASAPOp>, Schema)> { + let Operator::ASAP(ASAPOp::SummaryMerge { children, group_by: Some(keys), .. - } = state.asap()? + }) = state.operator() else { return None; }; let [input] = children.as_slice() else { return None; }; - let input = selected_from(input).unwrap_or(input); - let build = match input.asap()? { - ASAPOp::SummaryAgg { .. } => (**input).clone(), - _ if super::tumbling_window_chain::merged_chain(input).is_some() => { - super::tumbling_window_chain::merged_window_build(input).ok()? + let input = selected_from(input).unwrap_or_else(|| input.clone()); + let build = match input.operator() { + Operator::ASAP(op @ ASAPOp::SummaryAgg { .. }) => { + op.map_children(|child| InputOrTemporaryBuild::Input(child.clone())) + } + _ if super::tumbling_window_chain::merged_chain(&input).is_some() => { + super::tumbling_window_chain::merged_window_build(&input) + .ok()? + .0 } _ => return None, }; @@ -811,27 +725,138 @@ fn rolled_up_build(state: &OperatorNode) -> Option> { reduction, grouping, filter, - } = build.asap()? + } = build else { return None; }; let mapped = keys .iter() - .map(|&key| match reduction { + .map(|&key| match &reduction { Reduction::Reduce(by) if !by.is_without() => by.keys().get(key).copied(), Reduction::PerEntity => Some(key), Reduction::Reduce(_) => None, }) .collect::>>()?; - OperatorNode::new_shared(super::node::Operator::ASAP(ASAPOp::SummaryAgg { - child: Rc::clone(child), - family: family.clone(), - input: update.clone(), - reduction: Reduction::by(mapped), - grouping: grouping.clone(), - filter: filter.clone(), - })) - .ok() + let reduction = Reduction::by(mapped); + let schema = summary_agg_schema(child.schema(), &family, &reduction).ok()?; + Some(( + ASAPOp::SummaryAgg { + child, + family, + input: update, + reduction, + grouping, + filter, + }, + schema, + )) +} + +/// The schema of a `SummaryAgg` (or `SummaryTumblingWindowChain`) of +/// `family` by `reduction` over rows of schema `child`: the reduction's +/// output columns, its value column replaced by the state. +pub(crate) fn summary_agg_schema( + child: &Schema, + family: &FieldDataType, + reduction: &Reduction, +) -> Result { + let mut schema = crate::ir::schema::aggregate_schema::aggregate_output_schema( + child, + reduction, + &[crate::ir::operator::AggIntent::Sum { col: None }], + &[], + )?; + let index = match reduction { + Reduction::PerEntity => crate::ir::scalar::resolve_column_ref( + &crate::ir::scalar::ColumnRef::SampleValue, + &schema, + ) + .map_err(|e| SchemaDerivationError::InvalidScalarSignature(e.to_string()))?, + Reduction::Reduce(_) => schema.fields.len() - 1, + }; + schema.fields[index] = Field::new("state", family.clone(), false); + Ok(schema) +} + +/// The schema `FinalizeExactAccumulator` derives over state of +/// `state_schema` that `build` (the state's `SummaryAgg`, if any) builds: +/// each exact aggregate replaced by the value it finalizes to. +fn finalized_schema( + build: Option<&ASAPOp>, + state_schema: &Schema, +) -> Result { + let value_result = if let Some(ASAPOp::SummaryAgg { + child: source, + family: FieldDataType::ExactAggregate(kind, _), + input, + reduction, + filter, + .. + }) = build + { + { + use crate::ir::operator::AggIntent; + use crate::ir::schema::{ExactKind, SummaryInputExpr}; + let column = match &input.weight { + SummaryInputExpr::Column(col) => Some( + crate::ir::scalar::column_resolution::resolve_column_ref(col, source.schema()) + .map_err(|e| { + SchemaDerivationError::InvalidScalarSignature(e.to_string()) + })?, + ), + _ => None, + }; + let measure = match kind { + ExactKind::Sum => Some(AggIntent::Sum { col: column }), + ExactKind::Min => Some(AggIntent::Min { col: column }), + ExactKind::Max => Some(AggIntent::Max { col: column }), + ExactKind::Count => Some(AggIntent::Count { + accuracy: crate::ir::properties::AccuracyTarget::exact(), + }), + ExactKind::Rate => Some(AggIntent::Rate), + ExactKind::IRate => Some(AggIntent::IRate), + ExactKind::Increase => Some(AggIntent::Increase), + }; + measure + .map(|measure| { + crate::ir::NonASAPOp::Aggregate { + child: source.clone(), + reduction: reduction.clone(), + measures: vec![measure], + output_names: vec![], + // A filtered SUM/MIN/MAX is NULL for a + // group without a matching row. + filters: vec![filter.clone()], + having: None, + } + .output_schema() + }) + .transpose()? + .and_then(|s| match reduction { + Reduction::PerEntity => { + s.column_id("value").and_then(|i| s.fields.get(i).cloned()) + } + Reduction::Reduce(_) => s.fields.last().cloned(), + }) + } + } else { + None + }; + let mut out = state_schema.clone(); + for f in &mut out.fields { + if let FieldDataType::ExactAggregate(kind, _) = &f.dtype { + if let Some(result) = &value_result { + // The finalized value replaces the aggregate it + // realizes, so it takes that aggregate's column. + f.name = result.name.clone(); + f.dtype = result.dtype.clone(); + f.nullable = result.nullable; + } else { + f.dtype = FieldDataType::Plain(finalized_data_type(kind)); + } + } + } + Ok(out) } /// The schema of a regrouping `SummaryMerge` over states of schema `input`: @@ -877,11 +902,11 @@ fn finalized_data_type(kind: &crate::ir::schema::sketch::ExactKind) -> DataType /// The states `node` selects from when it is a `Filter` that keeps whole /// summary states by their plain columns (its output is state); `None` for -/// any other node. -pub fn selected_from(node: &OperatorNode) -> Option<&Rc> { - match node.non_asap() { - Some(crate::ir::NonASAPOp::Filter { child, .. }) - if node.result_kind == OperatorResultKind::State => +/// any other input. +pub fn selected_from(node: &I) -> Option { + match node.operator() { + Operator::NonASAP(crate::ir::NonASAPOp::Filter { child, .. }) + if node.result_kind() == OperatorResultKind::State => { Some(child) } @@ -889,18 +914,38 @@ pub fn selected_from(node: &OperatorNode) -> Option<&Rc> { } } +/// The build whose state this summary state carries: a `SummaryAgg` or +/// `SummaryTumblingWindowChain` itself, or the first input's build under +/// `SummaryMerge`s (merged inputs have one definition, which summary +/// coverage checks). `None` under a regrouping merge, whose states no build +/// carries (its coverage's definition is the build it stands for), and for +/// any other input. +pub fn summary_build(state: &I) -> Option> { + match state.operator() { + Operator::ASAP( + op @ (ASAPOp::SummaryAgg { .. } | ASAPOp::SummaryTumblingWindowChain { .. }), + ) => Some(op), + Operator::ASAP(ASAPOp::SummaryMerge { + children, + group_by: None, + .. + }) => summary_build(children.first()?), + _ => None, + } +} + /// The category of the values a evaluation of `state` produces: the category /// of the relational input the state was built from. -fn source_kind(node: &OperatorNode) -> OperatorResultKind { +fn source_kind(node: &I) -> OperatorResultKind { if let Some(states) = selected_from(node) { - return source_kind(states); + return source_kind(&states); } - match &node.operator { - super::node::Operator::ASAP(op) => match op.children().first() { - Some(child) => source_kind(child), + match node.operator() { + Operator::ASAP(op) => match op.children().first() { + Some(child) => source_kind(*child), None => OperatorResultKind::Relation, }, - super::node::Operator::NonASAP(_) => match node.result_kind { + Operator::NonASAP(_) => match node.result_kind() { OperatorResultKind::RangeVector => OperatorResultKind::InstantVector, OperatorResultKind::State => OperatorResultKind::Relation, other => other, @@ -914,21 +959,28 @@ fn source_kind(node: &OperatorNode) -> OperatorResultKind { /// series identity), then the estimated weight, `topk` (Float64). `None` /// when `state` is not a keyed build whose item columns resolve in its /// input. -fn topk_estimate_schema(state: &OperatorNode) -> Option { - use crate::ir::schema::SummaryInputExpr; +fn topk_estimate_schema(state: &I) -> Option { // A rollup reads items as the coarse build it stands for. - let rolled_up = rolled_up_build(state); - let build = match &rolled_up { - Some(build) => build.asap(), - None => state.summary_build(), - }; - let (Some(ASAPOp::SummaryAgg { child, input, .. }) - | Some(ASAPOp::SummaryTumblingWindowChain { child, input, .. })) = build + match rolled_up_build(state) { + Some((build, _)) => topk_schema_of_build(&build, state.schema()), + None => topk_schema_of_build(&summary_build(state)?, state.schema()), + } +} + +/// [`topk_estimate_schema`] of state of `state_schema` built by `build`. +fn topk_schema_of_build( + build: &ASAPOp, + state_schema: &Schema, +) -> Option { + use crate::ir::schema::SummaryInputExpr; + let (ASAPOp::SummaryAgg { child, input, .. } + | ASAPOp::SummaryTumblingWindowChain { child, input, .. }) = build else { return None; }; + let child_schema = child.schema(); let resolve = |column: &crate::ir::scalar::ColumnRef| { - crate::ir::scalar::resolve_column_ref(column, &child.schema).ok() + crate::ir::scalar::resolve_column_ref(column, child_schema).ok() }; let columns = match input.item.as_ref()? { SummaryInputExpr::Column(column) => vec![resolve(column)?], @@ -944,9 +996,9 @@ fn topk_estimate_schema(state: &OperatorNode) -> Option { }) => { let excluded: Vec<_> = excluding.iter().filter_map(resolve).collect(); let value = resolve(&crate::ir::scalar::ColumnRef::SampleValue); - (0..child.schema.fields.len()) + (0..child_schema.fields.len()) .filter(|index| { - Some(*index) != child.schema.time_index + Some(*index) != child_schema.time_index && Some(*index) != value && !excluded.contains(index) }) @@ -960,12 +1012,11 @@ fn topk_estimate_schema(state: &OperatorNode) -> Option { let complete_item_identity = matches!( input.item.as_ref()?, SummaryInputExpr::Column(_) | SummaryInputExpr::Tuple(_) - ) || child.schema.closed + ) || child_schema.closed || columns.iter().any(|&index| { - child.schema.fields[index].name == crate::ir::schema::PROMQL_SERIES_IDENTITY + child_schema.fields[index].name == crate::ir::schema::PROMQL_SERIES_IDENTITY }); - let mut fields: Vec = state - .schema + let mut fields: Vec = state_schema .fields .iter() .filter(|field| field.is_plain()) @@ -974,9 +1025,9 @@ fn topk_estimate_schema(state: &OperatorNode) -> Option { for index in columns { if !fields .iter() - .any(|field| field.name == child.schema.fields[index].name) + .any(|field| field.name == child_schema.fields[index].name) { - fields.push(child.schema.fields[index].clone()); + fields.push(child_schema.fields[index].clone()); } } // One row per (partition, item), not one row per partition. Preserve this @@ -990,7 +1041,7 @@ fn topk_estimate_schema(state: &OperatorNode) -> Option { Some(Schema { fields, unique_keys, - ..state.schema.clone() + ..state_schema.clone() }) } @@ -999,18 +1050,18 @@ fn topk_estimate_schema(state: &OperatorNode) -> Option { /// through `SummaryMerge`) is filtered, or, over a relation, is global (zero /// rows give one empty state) or summarizes a nullable column (a group of /// NULLs gives an empty state). -fn quantile_may_be_empty(state: &OperatorNode) -> bool { - match state.asap() { - Some(ASAPOp::SummaryMerge { +fn quantile_may_be_empty(state: &I) -> bool { + match state.operator() { + Operator::ASAP(ASAPOp::SummaryMerge { children, group_by, .. }) => { // A global rollup must also represent the empty input population, // even when every existing grouped state is nonempty. (source_kind(state) == OperatorResultKind::Relation && group_by.as_ref().is_some_and(Vec::is_empty)) - || children.iter().any(|child| quantile_may_be_empty(child)) + || children.iter().any(quantile_may_be_empty) } - Some( + Operator::ASAP( ASAPOp::SummaryAgg { child, input, @@ -1029,15 +1080,15 @@ fn quantile_may_be_empty(state: &OperatorNode) -> bool { if filter.is_some() { return true; } - if child.result_kind != OperatorResultKind::Relation { + if child.result_kind() != OperatorResultKind::Relation { return false; } let global = matches!(reduction, Reduction::Reduce(by) if by.keys().is_empty()); let nullable = match &input.weight { crate::ir::schema::sketch::SummaryInputExpr::Column(column) => { - crate::ir::scalar::resolve_column_ref(column, &child.schema) + crate::ir::scalar::resolve_column_ref(column, child.schema()) .ok() - .and_then(|index| child.schema.fields.get(index)) + .and_then(|index| child.schema().fields.get(index)) .is_some_and(|field| field.nullable) } _ => false, @@ -1045,6 +1096,6 @@ fn quantile_may_be_empty(state: &OperatorNode) -> bool { global || nullable } // Selecting whole states empties none of them. - _ => selected_from(state).is_some_and(|states| quantile_may_be_empty(states)), + _ => selected_from(state).is_some_and(|states| quantile_may_be_empty(&states)), } } diff --git a/crates/types/src/ir/operator/input.rs b/crates/types/src/ir/operator/input.rs new file mode 100644 index 000000000..1a4d034d5 --- /dev/null +++ b/crates/types/src/ir/operator/input.rs @@ -0,0 +1,145 @@ +//! How the derivation rules read an operator's inputs: one trait, so the +//! same rules (output schema and kind, input validation, sharing, summary +//! coverage) derive a front-end node's schema and a memo group's +//! `GroupProperties` (concepts doc `memo-groups-properties-requirements.md`, +//! section 3). + +use std::rc::Rc; + +use super::node::{Operator, OperatorNode, OperatorResultKind}; +use super::non_asap::NonASAPOp; +use crate::ir::properties::summary_coverage::SummaryCoverage; +use crate::ir::schema::Schema; +use crate::ir::SchemaDerivationError; + +/// An input of an operator, as the rules read it. +pub trait OperatorInput: Sized + Clone { + fn schema(&self) -> &Schema; + fn result_kind(&self) -> OperatorResultKind; + /// The input's own operator, its children as inputs again. + fn operator(&self) -> Operator; + /// What the input covers when it is summary state in the memo; `None` + /// for any other input. + fn summary_coverage(&self) -> Option<&SummaryCoverage>; +} + +/// A front-end node tree. Node trees carry no summary coverage. +impl OperatorInput for Rc { + fn schema(&self) -> &Schema { + &self.schema + } + + fn result_kind(&self) -> OperatorResultKind { + self.result_kind + } + + fn operator(&self) -> Operator { + self.operator.clone() + } + + fn summary_coverage(&self) -> Option<&SummaryCoverage> { + None + } +} + +/// An input of a rule: a real input `I`, or a build the rule constructs +/// only while it derives. +#[derive(Clone)] +pub(crate) enum InputOrTemporaryBuild { + Input(I), + TemporaryBuild(Rc>), +} + +/// A build a rule constructs only while it derives (the whole-window build +/// of a chain, the build a rollup stands for, the build a selection of +/// states stands for): an operator over `I`s with its derived schema and +/// kind. Never stored in the memo. A rule never builds one over another +/// rule's temporary build, so the type does not nest. +pub(crate) struct TemporaryBuild { + operator: Operator>, + schema: Schema, + result_kind: OperatorResultKind, +} + +impl InputOrTemporaryBuild { + /// A non-ASAP `operator` with its derived schema and kind. Only non-ASAP + /// rules derive it, which build no input themselves. + pub(crate) fn build( + operator: NonASAPOp>, + ) -> Result { + let schema = operator.output_schema()?; + Ok(Self::with_schema(Operator::NonASAP(operator), schema)) + } + + /// `operator` with `schema` and its derived kind. + pub(crate) fn with_schema( + operator: Operator>, + schema: Schema, + ) -> Self { + let result_kind = operator.output_kind(); + Self::TemporaryBuild(Rc::new(TemporaryBuild { + operator, + schema, + result_kind, + })) + } +} + +impl OperatorInput for InputOrTemporaryBuild { + fn schema(&self) -> &Schema { + match self { + Self::Input(input) => input.schema(), + Self::TemporaryBuild(build) => &build.schema, + } + } + + fn result_kind(&self) -> OperatorResultKind { + match self { + Self::Input(input) => input.result_kind(), + Self::TemporaryBuild(build) => build.result_kind, + } + } + + fn operator(&self) -> Operator { + match self { + Self::Input(input) => input + .operator() + .map_children(|child| Self::Input(child.clone())), + Self::TemporaryBuild(build) => build.operator.clone(), + } + } + + fn summary_coverage(&self) -> Option<&SummaryCoverage> { + match self { + Self::Input(input) => input.summary_coverage(), + Self::TemporaryBuild(_) => None, + } + } +} + +impl InputOrTemporaryBuild> { + /// The node tree of this input: a temporary build as a new node over + /// its inputs' trees. + pub(crate) fn to_node(&self) -> Rc { + match self { + Self::Input(node) => Rc::clone(node), + Self::TemporaryBuild(build) => Rc::new(OperatorNode::with_schema( + build.operator.map_children(Self::to_node), + build.schema.clone(), + )), + } + } +} + +/// A build over node trees (`tumbling_window_chain`'s builds) as a node. +pub(crate) fn node_of_build( + (build, schema): ( + super::asap::ASAPOp>>, + Schema, + ), +) -> OperatorNode { + OperatorNode::with_schema( + Operator::ASAP(build.map_children(InputOrTemporaryBuild::to_node)), + schema, + ) +} diff --git a/crates/types/src/ir/operator/maintained_population.rs b/crates/types/src/ir/operator/maintained_population.rs index 570bebfa6..f11ab8682 100644 --- a/crates/types/src/ir/operator/maintained_population.rs +++ b/crates/types/src/ir/operator/maintained_population.rs @@ -2,6 +2,8 @@ //! Resource limits, ingestion placement and data structures belong to the executor. use serde::{Deserialize, Serialize}; +use crate::ir::operator::input::OperatorInput; + #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct CurrentSeriesInput { pub metric: String, @@ -56,35 +58,35 @@ pub struct MaintainedPopulation { impl CurrentSeriesInput { /// Verify the named contract against the canonical maintenance input. - pub(crate) fn matches_node(&self, input: &crate::ir::OperatorNode) -> bool { + pub(crate) fn matches_node(&self, input: &I) -> bool { use crate::ir::operator::Source; use crate::ir::scalar::{CompareOpKind, ScalarValue}; use crate::ir::schema::DataType; use crate::ir::{NonASAPOp, Operator, ScalarExpr, TimeRangeKind}; - let input = match &input.operator { + let input = match input.operator() { Operator::NonASAP(NonASAPOp::TimeRange { range, kind: TimeRangeKind::Instant, child, }) if self.lookback_ms > 0 - && *range == std::time::Duration::from_millis(self.lookback_ms) => + && range == std::time::Duration::from_millis(self.lookback_ms) => { - child.as_ref() + child } Operator::NonASAP(NonASAPOp::TimeRange { .. }) => return false, - _ if self.lookback_ms == 300_000 => input, + _ if self.lookback_ms == 300_000 => input.clone(), _ => return false, }; let Operator::NonASAP(NonASAPOp::Scan { source: Source::TimeSeries { metric }, predicates, schema, - }) = &input.operator + }) = input.operator() else { return false; }; if self.metric.is_empty() - || *metric != self.metric + || metric != self.metric || (schema.closed && !schema.has_promql_series_identity()) || schema.time_index.is_none() { @@ -99,7 +101,7 @@ impl CurrentSeriesInput { return false; } let mut matchers = Vec::new(); - for predicate in predicates { + for predicate in &predicates { let ScalarExpr::Compare { left, op, right, .. } = &predicate.0 @@ -138,7 +140,7 @@ impl CurrentSeriesInput { impl MaintainedPopulation { /// Whether `input` is the maintenance input this population declares. - pub(crate) fn matches_node(&self, input: &crate::ir::OperatorNode) -> bool { + pub(crate) fn matches_node(&self, input: &I) -> bool { use crate::ir::operator::Source; use crate::ir::schema::DataType; use crate::ir::{NonASAPOp, Operator}; @@ -149,18 +151,21 @@ impl MaintainedPopulation { value_column, grouping, } => { + let operator = input.operator(); let Operator::NonASAP(NonASAPOp::Scan { source: Source::Table { .. }, schema, .. - }) = &input.operator + }) = &operator else { return false; }; // The same computation, whatever accuracy or timing has - // been attached to the node since. - let same_source = - expected.operator == input.operator && expected.schema == input.schema; + // been attached to the node since. A `Scan` has no input, + // so its operator without inputs is all of it. + let same_source = expected.operator.map_children(|_| ()) + == operator.map_children(|_| ()) + && expected.schema == *input.schema(); same_source && schema.closed && schema diff --git a/crates/types/src/ir/operator/mod.rs b/crates/types/src/ir/operator/mod.rs index f7baca7a9..8b0d5ce2e 100644 --- a/crates/types/src/ir/operator/mod.rs +++ b/crates/types/src/ir/operator/mod.rs @@ -4,6 +4,8 @@ //! - [`node`] — [`OperatorNode`](node::OperatorNode): an operator plus the //! planning properties every traversal needs. //! - [`non_asap`] / [`asap`] — ordinary query operators and ASAP operators. +//! - [`input`] — how the derivation rules read an operator's inputs +//! ([`OperatorInput`](input::OperatorInput)). //! - [`operator_properties`] — operator parameter types ([`GroupKeys`], //! [`Reduction`], [`Source`], …). //! - [`agg_intent`] — the aggregation-intent vocabulary ([`AggIntent`]). @@ -12,6 +14,7 @@ //! - [`query`] — a query root: an operator result or a standalone scalar. pub mod agg_intent; pub mod asap; +pub mod input; pub mod maintained_population; pub mod node; pub mod non_asap; diff --git a/crates/types/src/ir/operator/node.rs b/crates/types/src/ir/operator/node.rs index d2ae4f643..956a85dc6 100644 --- a/crates/types/src/ir/operator/node.rs +++ b/crates/types/src/ir/operator/node.rs @@ -1,7 +1,9 @@ //! The unified operator node: one node type before and after ASAP //! optimization. A node is an operator plus the common planning properties -//! every traversal needs (output category and schema, accuracy guarantee, -//! execution timing). +//! every traversal needs (output category and schema, accuracy guarantee). +//! Physical properties (timing, lifetime, schedule) are not on the node: +//! each physical candidate holds its own +//! ([`crate::stages::PhysicalCandidate::properties`]). use std::collections::HashSet; use std::rc::Rc; @@ -9,13 +11,11 @@ use std::rc::Rc; use serde::{Deserialize, Serialize}; use super::asap::ASAPOp; +use super::input::OperatorInput; use super::non_asap::NonASAPOp; -use crate::ir::properties::execution_data_state::ExecutionTiming; use crate::ir::properties::guarantee::AccuracyGuarantee; -use crate::ir::properties::summary_coverage::{NodeToSummaryCoverageCache, SummaryCoverage}; use crate::ir::schema::Schema; use crate::ir::SchemaDerivationError; -use crate::physical::materialization::Materialization; /// The output category of an operator, derived from the operation and its /// inputs. Matching column schemas do not make categories interchangeable. @@ -62,7 +62,9 @@ impl Operator { } } -impl Operator { +/// The derivation rules, one set for every kind of input +/// ([`OperatorInput`]): front-end node trees and memo groups. +impl Operator { pub(crate) fn output_schema(&self) -> Result { match self { Operator::NonASAP(op) => op.output_schema(), @@ -83,6 +85,16 @@ impl Operator { Operator::ASAP(op) => op.validate_inputs(), } } + + /// Whether two equal copies of this operator over the same inputs return + /// the same rows, so they may be one group: an ASAP operator, or a + /// non-ASAP one [`NonASAPOp::is_shareable_when_equal`]. + pub(crate) fn is_shareable_when_equal(&self) -> bool { + match self { + Operator::NonASAP(op) => op.is_shareable_when_equal(), + Operator::ASAP(_) => true, + } + } } /// A node of the logical DAG. Nodes are immutable and shared through `Rc`; @@ -91,19 +103,13 @@ impl Operator { /// /// `schema` and `result_kind` are derived from `operator` and its children /// at construction and retained. `guarantee` is `None` until accuracy -/// assessment establishes one (`None` never means exact). `timing` is `None` -/// until a materialization assignment is applied; export rejects an executable -/// node without one. `materialization` is the option that assignment chose -/// for this node (`NotMaterialized` until then, and for every node whose -/// output is not stored). +/// assessment establishes one (`None` never means exact). #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct OperatorNode { pub operator: Operator, pub result_kind: OperatorResultKind, pub schema: Schema, pub guarantee: Option, - pub timing: Option, - pub materialization: Materialization, } impl OperatorNode { @@ -112,9 +118,7 @@ impl OperatorNode { /// ASAP operator, ...). pub fn new(operator: Operator) -> Result { let schema = operator.output_schema()?; - let node = Self::with_schema(operator, schema); - node.check_merge(&mut NodeToSummaryCoverageCache::new())?; - Ok(node) + Ok(Self::with_schema(operator, schema)) } /// Build a node with caller-supplied output names and qualifiers. For @@ -127,8 +131,6 @@ impl OperatorNode { result_kind, schema, guarantee: None, - timing: None, - materialization: Materialization::NotMaterialized, } } @@ -144,55 +146,10 @@ impl OperatorNode { self } - pub fn with_timing(mut self, timing: Option) -> Self { - self.timing = timing; - self - } - - pub fn with_materialization(mut self, materialization: Materialization) -> Self { - self.materialization = materialization; - self - } - - /// What this summary state covers; `None` for a node that is not a - /// `SummaryAgg` or a valid `SummaryMerge`. Derived on each call, since - /// the public fields can be edited in place. - #[cfg(any(test, feature = "test-support"))] - pub fn coverage(&self) -> Option { - SummaryCoverage::derive(self).ok() - } - - /// The build whose state this summary-state node carries: a - /// `SummaryAgg` or `SummaryTumblingWindowChain` itself, or the first input's build under - /// `SummaryMerge`s (merged inputs have one definition, which - /// `coverage` checks). `None` under a regrouping - /// merge, whose states no build in the DAG carries (its coverage's - /// `definition` is the build it stands for), and for any other node. - pub fn summary_build(&self) -> Option<&ASAPOp> { - match self.asap()? { - op @ (ASAPOp::SummaryAgg { .. } | ASAPOp::SummaryTumblingWindowChain { .. }) => { - Some(op) - } - ASAPOp::SummaryMerge { - children, - group_by: None, - .. - } => children.first()?.summary_build(), - _ => None, - } - } - - /// A `SummaryMerge` is valid only over inputs with the same definition - /// and disjoint selections. `cache` holds the coverage already derived - /// for nodes of the DAG being validated. - fn check_merge( - &self, - cache: &mut NodeToSummaryCoverageCache, - ) -> Result<(), SchemaDerivationError> { - if matches!(self.asap(), Some(ASAPOp::SummaryMerge { .. })) { - SummaryCoverage::derive_cached(self, cache)?; - } - Ok(()) + /// The build whose state this summary-state node carries + /// ([`summary_build`](super::asap::summary_build)). + pub fn summary_build(self: &Rc) -> Option { + super::asap::summary_build(self) } pub fn non_asap(&self) -> Option<&NonASAPOp> { @@ -237,8 +194,7 @@ impl OperatorNode { /// the old derived schema are retained, for either operator category. A /// change in output arity with such overrides needs an explicit new /// naming assignment. - /// `guarantee`, `timing` and `materialization` depend on the inputs and - /// are cleared. + /// `guarantee` depends on the inputs and is cleared. pub fn with_new_children( &self, f: impl FnMut(&Rc) -> Rc, @@ -302,39 +258,12 @@ impl OperatorNode { out } - /// Validate assigned phases without imposing any particular runtime implementation. - #[cfg(any(test, feature = "test-support"))] - pub fn validate_execution_timing(self: &Rc) -> Result<(), SchemaDerivationError> { - self.validate_structure()?; - for node in Self::reachable(self) { - let timing = node.timing.ok_or_else(|| { - SchemaDerivationError::InvalidScalarSignature( - "execution timing is unassigned".into(), - ) - })?; - if timing == crate::ir::properties::ExecutionTiming::IngestionTime - && node.children().iter().any(|child| { - child.timing != Some(crate::ir::properties::ExecutionTiming::IngestionTime) - }) - { - return Err(SchemaDerivationError::InvalidScalarSignature( - "ingestion-time operation depends on a query-time or unassigned input".into(), - )); - } - } - Ok(()) - } - /// Validate the whole DAG reachable from this node: every operator's /// input contract, scalar typing against the owning operator's input /// schema, and agreement between each retained schema and the one /// derived from the operator. Both operator categories may override field /// names and qualifiers; all structural metadata must match derivation. - /// `timing` may be `None`. pub fn validate_structure(self: &Rc) -> Result<(), SchemaDerivationError> { - // Each summary state's coverage is derived once, not once per merge - // above it; `reachable` keeps every node alive while it is used. - let mut coverage = NodeToSummaryCoverageCache::new(); for node in Self::reachable(self) { if node.schema.time_index.is_some_and(|i| { node.schema.fields.get(i).is_none_or(|f| { @@ -352,7 +281,6 @@ impl OperatorNode { )); } node.operator.validate_inputs()?; - node.check_merge(&mut coverage)?; if node.result_kind != node.operator.output_kind() { return Err(SchemaDerivationError::InvalidScalarSignature( "retained result kind disagrees with operation".into(), diff --git a/crates/types/src/ir/operator/non_asap.rs b/crates/types/src/ir/operator/non_asap.rs index 7eed22fda..dd1755456 100644 --- a/crates/types/src/ir/operator/non_asap.rs +++ b/crates/types/src/ir/operator/non_asap.rs @@ -6,7 +6,8 @@ use std::time::Duration; use serde::{Deserialize, Serialize}; -use super::node::{OperatorNode, OperatorResultKind}; +use super::input::OperatorInput; +use super::node::{Operator, OperatorNode, OperatorResultKind}; use crate::ir::operator::agg_intent::AggIntent; use crate::ir::operator::operator_properties::{ BinaryOpKind, GroupKeys, InfoMatcher, JoinKind, Reduction, RelationalSetOpKind, SampleKind, @@ -491,7 +492,7 @@ impl NonASAPOp { } } -impl NonASAPOp { +impl NonASAPOp { /// Whether two structurally equal copies of this node may be one shared /// node: every execution over the same input rows returns the same /// output rows, so one copy over their one input can be computed (or @@ -514,7 +515,7 @@ impl NonASAPOp { /// function the IR accepts is deterministic (no `random()`; unregistered /// SQL functions are rejected) and that a deployment's /// `AggIntent::Extension` returns the same rows for the same input. - pub fn is_shareable_when_equal(&self) -> bool { + pub(crate) fn is_shareable_when_equal(&self) -> bool { use NonASAPOp::*; match self { Limit { @@ -524,14 +525,14 @@ impl NonASAPOp { child, } => { (n.is_none() && *offset == 0) - || matches!(child.non_asap(), Some(Sort { partition_by: sorted_by, .. }) - if sorted_by == partition_by) + || matches!(child.operator(), Operator::NonASAP(Sort { partition_by: sorted_by, .. }) + if sorted_by == *partition_by) } // Empty `cols` means every column. Dedup { cols, child } => { cols.is_empty() || cols.iter().collect::>().len() - == child.schema.fields.len() + == child.schema().fields.len() } SQLWindowFunc { func, @@ -610,7 +611,7 @@ impl NonASAPOp { .output_schema(); } let mut output = - aggregate_output_schema(&child.schema, reduction, measures, output_names)?; + aggregate_output_schema(child.schema(), reduction, measures, output_names)?; let offset = reduction.group_keys().map_or(0, |keys| keys.len()); // A filtered quantile is NULL for a group without a matching // row, as its summary estimate is. @@ -625,14 +626,14 @@ impl NonASAPOp { } // Over a relation, a quantile is also NULL for a global // aggregate of no row and a group whose values are all NULL. - if child.result_kind == OperatorResultKind::Relation { + if child.result_kind() == OperatorResultKind::Relation { for (index, measure) in measures.iter().enumerate() { if matches!(measure, AggIntent::Quantile { .. }) && (offset == 0 || measure .input_cols() .iter() - .any(|i| child.schema.fields[*i].nullable)) + .any(|i| child.schema().fields[*i].nullable)) { if let Some(field) = output.fields.get_mut(offset + index) { field.nullable = true; @@ -640,7 +641,7 @@ impl NonASAPOp { } } } - if child.result_kind == OperatorResultKind::Relation { + if child.result_kind() == OperatorResultKind::Relation { for (index, measure) in measures.iter().enumerate() { if matches!( measure, @@ -654,7 +655,7 @@ impl NonASAPOp { || measure .input_cols() .iter() - .any(|i| child.schema.fields[*i].nullable); + .any(|i| child.schema().fields[*i].nullable); if let Some(field) = output.fields.get_mut(offset + index) { field.nullable = nullable; } @@ -671,13 +672,13 @@ impl NonASAPOp { | PromqlSeriesSample { child, .. } | PromqlInfoEnrich { child, .. } | TimeRange { child, .. } - | TimeShift { child, .. } => child.schema.clone(), + | TimeShift { child, .. } => child.schema().clone(), // ρ — relabel preserves every input column and writes one label // `dst` (Utf8): overwritten in place if it already exists, else // appended (nullable). Row-uniqueness is no longer provable. PromqlRelabel { dst, child, .. } => { - let mut out = child.schema.clone(); + let mut out = child.schema().clone(); if let Some(existing) = out.fields.iter_mut().find(|c| c.name == *dst) { existing.dtype = FieldDataType::Plain(DataType::Utf8); existing.nullable = true; @@ -698,7 +699,7 @@ impl NonASAPOp { qualifier, child, } => { - let in_schema = &child.schema; + let in_schema = child.schema(); let fields: Vec = cols .iter() .enumerate() @@ -747,7 +748,7 @@ impl NonASAPOp { } Dedup { cols, child } => { - let mut out = child.schema.clone(); + let mut out = child.schema().clone(); if !cols.is_empty() { out.add_unique_key(cols.clone()); } @@ -758,12 +759,12 @@ impl NonASAPOp { let mut s = children .first() .ok_or(SchemaDerivationError::EmptyConcat)? - .schema + .schema() .clone(); // A column is nullable if any child's is: a rolled-up // grouping key is NULL only in the coarser levels. for child in &children[1..] { - for (field, other) in s.fields.iter_mut().zip(&child.schema.fields) { + for (field, other) in s.fields.iter_mut().zip(&child.schema().fields) { field.nullable |= other.nullable; } } @@ -771,7 +772,7 @@ impl NonASAPOp { s } SetOp { left, .. } => { - let mut s = left.schema.clone(); + let mut s = left.schema().clone(); s.unique_keys.clear(); s } @@ -781,8 +782,8 @@ impl NonASAPOp { left, right, } => { - let l = &left.schema; - let r = &right.schema; + let l = left.schema(); + let r = right.schema(); if matches!(kind, JoinKind::Semi | JoinKind::Anti) { return Ok(Schema { unique_keys: Vec::new(), @@ -825,7 +826,7 @@ impl NonASAPOp { child, .. } => { - let mut out = child.schema.clone(); + let mut out = child.schema().clone(); let arg = args.first().map(|a| a.scalar_type(&out)).transpose()?; let arg_dtype = || { arg.as_ref() @@ -871,7 +872,7 @@ impl NonASAPOp { BinaryOp { lhs, rhs, operator, .. } => { - let mut output = lhs.schema.clone(); + let mut output = lhs.schema().clone(); let grouping = operator .vector_match .as_ref() @@ -883,7 +884,7 @@ impl NonASAPOp { let mut additions = Vec::new(); if right_rows { additions.extend( - rhs.schema + rhs.schema() .fields .iter() .filter(|c| c.plain_dtype() == Some(&DataType::Utf8)) @@ -917,7 +918,7 @@ impl NonASAPOp { Source::Table { .. } => OperatorResultKind::Relation, }, Values { .. } | SQLWindowFunc { .. } => OperatorResultKind::Relation, - TimeRange { child, .. } if child.result_kind == OperatorResultKind::Relation => { + TimeRange { child, .. } if child.result_kind() == OperatorResultKind::Relation => { OperatorResultKind::Relation } TimeRange { kind, .. } => match kind { @@ -931,12 +932,12 @@ impl NonASAPOp { // category (a SQL GROUP BY stays a relation). Aggregate { reduction, child, .. - } => match (reduction, child.result_kind) { + } => match (reduction, child.result_kind()) { (Reduction::PerEntity, OperatorResultKind::RangeVector) | (_, OperatorResultKind::State) => OperatorResultKind::InstantVector, (_, kind) => kind, }, - Project { child, cols, .. } if cols.iter().any(|item| matches!(item.expr, ScalarExpr::Column(i) if child.schema.fields.get(i).is_some_and(|f| !f.is_plain()))) => OperatorResultKind::State, + Project { child, cols, .. } if cols.iter().any(|item| matches!(item.expr, ScalarExpr::Column(i) if child.schema().fields.get(i).is_some_and(|f| !f.is_plain()))) => OperatorResultKind::State, // Selecting whole states by their plain (key) columns keeps them states. Filter { child, pred } if selects_states(child, pred) => OperatorResultKind::State, Filter { child, .. } @@ -947,12 +948,12 @@ impl NonASAPOp { | TimeShift { child, .. } | PromqlRelabel { child, .. } | PromqlInfoEnrich { child, .. } - | PromqlSeriesSample { child, .. } => readable(child.result_kind), - Join { left, .. } | SetOp { left, .. } => readable(left.result_kind), + | PromqlSeriesSample { child, .. } => readable(child.result_kind()), + Join { left, .. } | SetOp { left, .. } => readable(left.result_kind()), Concat { children, .. } => children .first() - .map_or(OperatorResultKind::Relation, |c| readable(c.result_kind)), - BinaryOp { lhs, .. } => readable(lhs.result_kind), + .map_or(OperatorResultKind::Relation, |c| readable(c.result_kind())), + BinaryOp { lhs, .. } => readable(lhs.result_kind()), } } @@ -960,8 +961,8 @@ impl NonASAPOp { /// and its children's output categories. pub(crate) fn validate_inputs(&self) -> Result<(), SchemaDerivationError> { use NonASAPOp::*; - let no_state = |node: &OperatorNode, what: &str| { - if node.result_kind == OperatorResultKind::State { + let no_state = |node: &I, what: &str| { + if node.result_kind() == OperatorResultKind::State { Err(SchemaDerivationError::InvalidScalarSignature(format!( "{what} consumes summary state; read it out first" ))) @@ -970,18 +971,19 @@ impl NonASAPOp { } }; let invalid = |message: &str| SchemaDerivationError::InvalidScalarSignature(message.into()); - let predicate = |pred: &Predicate, scope: &Schema| -> Result<(), SchemaDerivationError> { - if matches!( - pred.0.scalar_type(scope)?.0, - DataType::Bool | DataType::Null - ) { - Ok(()) - } else { - Err(invalid("predicate must be boolean")) - } - }; - let instant = |child: &OperatorNode| -> Result<(), SchemaDerivationError> { - if child.result_kind == OperatorResultKind::InstantVector { + let predicate = + |pred: &Predicate, scope: &Schema| -> Result<(), SchemaDerivationError> { + if matches!( + pred.0.scalar_type(scope)?.0, + DataType::Bool | DataType::Null + ) { + Ok(()) + } else { + Err(invalid("predicate must be boolean")) + } + }; + let instant = |child: &I| -> Result<(), SchemaDerivationError> { + if child.result_kind() == OperatorResultKind::InstantVector { Ok(()) } else { Err(invalid("operation requires an instant vector")) @@ -1025,14 +1027,14 @@ impl NonASAPOp { if !selects_states(child, pred) { no_state(child, "Filter")?; } - predicate(pred, &child.schema)?; + predicate(pred, child.schema())?; } Project { child, cols, .. } => { for col in cols { if let ScalarExpr::Column(index) = col.expr { - columns(&[index], &child.schema)?; + columns(&[index], child.schema())?; } else { - col.expr.scalar_type(&child.schema)?; + col.expr.scalar_type(child.schema())?; } } } @@ -1044,8 +1046,8 @@ impl NonASAPOp { } => { no_state(lhs, "BinaryOp")?; no_state(rhs, "BinaryOp")?; - if lhs.result_kind != rhs.result_kind - || lhs.result_kind == OperatorResultKind::RangeVector + if lhs.result_kind() != rhs.result_kind() + || lhs.result_kind() == OperatorResultKind::RangeVector { return Err(invalid("binary operands have incompatible result kinds")); } @@ -1058,16 +1060,16 @@ impl NonASAPOp { } => { no_state(left, "Join")?; no_state(right, "Join")?; - if left.result_kind != OperatorResultKind::Relation - || right.result_kind != OperatorResultKind::Relation + if left.result_kind() != OperatorResultKind::Relation + || right.result_kind() != OperatorResultKind::Relation { return Err(invalid("SQL join requires relations")); } let scope = Schema::new( - left.schema + left.schema() .fields .iter() - .chain(&right.schema.fields) + .chain(&right.schema().fields) .cloned() .collect(), ); @@ -1076,8 +1078,8 @@ impl NonASAPOp { SetOp { left, right, .. } => { no_state(left, "SetOp")?; no_state(right, "SetOp")?; - if left.result_kind != OperatorResultKind::Relation - || right.result_kind != OperatorResultKind::Relation + if left.result_kind() != OperatorResultKind::Relation + || right.result_kind() != OperatorResultKind::Relation { return Err(invalid("SQL set operation requires relations")); } @@ -1097,16 +1099,16 @@ impl NonASAPOp { } => { no_state(child, "Aggregate")?; if let Reduction::Reduce(keys) = reduction { - columns(keys.keys(), &child.schema)?; + columns(keys.keys(), child.schema())?; } for measure in measures { - columns(&measure.input_cols(), &child.schema)?; + columns(&measure.input_cols(), child.schema())?; } if !filters.is_empty() && filters.len() != measures.len() { return Err(invalid("aggregate filter count differs from measure count")); } for pred in filters.iter().flatten() { - predicate(pred, &child.schema)?; + predicate(pred, child.schema())?; } if let Some(pred) = having { predicate(pred, &self.output_schema()?)?; @@ -1114,23 +1116,23 @@ impl NonASAPOp { } Dedup { child, cols } => { no_state(child, "Dedup")?; - columns(cols, &child.schema)?; + columns(cols, child.schema())?; } Sort { child, keys, partition_by, } => { - columns(partition_by.keys(), &child.schema)?; + columns(partition_by.keys(), child.schema())?; for key in keys { - key.expr.scalar_type(&child.schema)?; + key.expr.scalar_type(child.schema())?; } } Limit { child, partition_by, .. - } => columns(partition_by.keys(), &child.schema)?, + } => columns(partition_by.keys(), child.schema())?, SQLWindowFunc { child, args, @@ -1138,12 +1140,12 @@ impl NonASAPOp { partition_by, .. } => { - if child.result_kind != OperatorResultKind::Relation { + if child.result_kind() != OperatorResultKind::Relation { return Err(invalid("SQL window requires a relation")); } - columns(partition_by.keys(), &child.schema)?; + columns(partition_by.keys(), child.schema())?; for expr in args.iter().chain(order_by.iter().map(|k| &k.expr)) { - expr.scalar_type(&child.schema)?; + expr.scalar_type(child.schema())?; } } PromqlVectorFromScalar(expr) => { @@ -1156,12 +1158,12 @@ impl NonASAPOp { | PromqlSeriesSample { child, .. } => instant(child)?, PromqlRelabel { child, value, .. } => { instant(child)?; - if value.scalar_type(&child.schema)?.0 != DataType::Utf8 { + if value.scalar_type(child.schema())?.0 != DataType::Utf8 { return Err(invalid("label expression requires a string")); } } TimeRange { child, .. } => { - if child.result_kind != OperatorResultKind::Relation { + if child.result_kind() != OperatorResultKind::Relation { instant(child)?; } } @@ -1169,7 +1171,7 @@ impl NonASAPOp { // A relation is a SQL table's rows, shifted under a trailing // window's tumbling window. if !matches!( - child.result_kind, + child.result_kind(), OperatorResultKind::InstantVector | OperatorResultKind::RangeVector | OperatorResultKind::Relation @@ -1187,13 +1189,13 @@ impl NonASAPOp { /// Each input retains its keys only when the equality predicates constrain /// the other input to at most one matching row. In particular this preserves /// grouped sum/count keys through the SQL-average implementation. -fn inner_join_unique_keys( - predicate: &ScalarExpr, +fn inner_join_unique_keys( + predicate: &ScalarExpr, left: &Schema, right: &Schema, ) -> Vec> { use crate::ir::scalar::CompareOpKind; - fn columns(expr: &ScalarExpr) -> Option<(ColumnId, ColumnId)> { + fn columns(expr: &ScalarExpr) -> Option<(ColumnId, ColumnId)> { if let ScalarExpr::Compare { left, op: CompareOpKind::Eq, @@ -1208,7 +1210,7 @@ fn inner_join_unique_keys( } None } - fn equality(expr: &ScalarExpr) -> Option<(ColumnId, ColumnId)> { + fn equality(expr: &ScalarExpr) -> Option<(ColumnId, ColumnId)> { if let Some(pair) = columns(expr) { return Some(pair); } @@ -1241,7 +1243,7 @@ fn inner_join_unique_keys( } None } - fn collect(expr: &ScalarExpr, pairs: &mut Vec<(ColumnId, ColumnId)>) { + fn collect(expr: &ScalarExpr, pairs: &mut Vec<(ColumnId, ColumnId)>) { if let ScalarExpr::BoolAnd(terms) = expr { for term in terms { collect(term, pairs); @@ -1289,11 +1291,11 @@ fn inner_join_unique_keys( /// Whether a `Filter` of `pred` over `child` selects whole summary states: /// `child` is state and `pred` reads only its plain columns, such as group keys. -fn selects_states(child: &OperatorNode, pred: &Predicate) -> bool { - child.result_kind == OperatorResultKind::State +fn selects_states(child: &I, pred: &Predicate) -> bool { + child.result_kind() == OperatorResultKind::State && pred.0.columns_referenced().iter().all(|&column| { child - .schema + .schema() .fields .get(column) .is_some_and(|field| field.is_plain()) @@ -1309,7 +1311,11 @@ fn readable(kind: OperatorResultKind) -> OperatorResultKind { /// Default output-column name for a projection item with no explicit alias: /// a bare column keeps its (schema) name; anything else gets `col_{i}`. -fn default_proj_name(expr: &ScalarExpr, idx: usize, schema: &Schema) -> String { +fn default_proj_name( + expr: &ScalarExpr, + idx: usize, + schema: &Schema, +) -> String { match expr { ScalarExpr::Column(id) => schema .fields @@ -1793,7 +1799,7 @@ mod tests { unique_keys: vec![vec![0, 1]], closed: true, }; - let equal = |left, right| ScalarExpr::Compare { + let equal = |left, right| ScalarExpr::>::Compare { left: Box::new(ScalarExpr::Column(left)), op: CompareOpKind::Eq, right: Box::new(ScalarExpr::Column(right)), diff --git a/crates/types/src/ir/operator/tumbling_window_chain.rs b/crates/types/src/ir/operator/tumbling_window_chain.rs index 8416fa701..add4528bf 100644 --- a/crates/types/src/ir/operator/tumbling_window_chain.rs +++ b/crates/types/src/ir/operator/tumbling_window_chain.rs @@ -8,44 +8,47 @@ //! the family merges (the family-capability table), and the executor runs //! the chain as its tumbling windows (`expand_tumbling_window_chains`). A merge of only the //! newest `m < n` tumbling windows (`SummaryMerge::newest_tumbling_windows`) is the state of one -//! build over `TimeRange(m·w)` ([`merged_window_build`]): one chain kept for +//! build over `TimeRange(m·w)` (`merged_window_build`): one chain materialized for //! the longest window serves every query whose window, on the same tumbling windows, //! ends at the same time and is shorter. +#[cfg(any(test, feature = "test-support"))] use std::collections::HashMap; use std::rc::Rc; use std::time::Duration; -use super::asap::ASAPOp; +use super::asap::{summary_agg_schema, ASAPOp}; +use super::input::node_of_build; +use super::input::{InputOrTemporaryBuild, OperatorInput}; use super::node::{Operator, OperatorNode}; use super::non_asap::{NonASAPOp, TimeRangeKind}; +#[cfg(any(test, feature = "test-support"))] use super::operator_properties::TimeShift; -use super::query::QueryRoot; -use crate::ir::properties::ExecutionTiming; +use crate::ir::schema::Schema; use crate::ir::SchemaDerivationError; fn invalid(message: &str) -> SchemaDerivationError { SchemaDerivationError::InvalidScalarSignature(message.into()) } -/// The `SummaryTumblingWindowChain` a summary-state node carries merged: the chain itself, +/// The `SummaryTumblingWindowChain` a summary state carries merged: the chain itself, /// or the one chain a `SummaryMerge` that keeps its groups reads, in full -/// or its newest tumbling windows (`merged_tumbling_window_count`). `None` for any other node, +/// or its newest tumbling windows (`merged_tumbling_window_count`). `None` for any other input, /// a rollup (regrouping merge) included. -pub fn merged_chain(state: &OperatorNode) -> Option<&OperatorNode> { - match state.asap()? { - ASAPOp::SummaryTumblingWindowChain { .. } => Some(state), - ASAPOp::SummaryMerge { +pub fn merged_chain(state: &I) -> Option { + match state.operator() { + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) => Some(state.clone()), + Operator::ASAP(ASAPOp::SummaryMerge { children, group_by: None, .. - } => match children.as_slice() { + }) => match children.as_slice() { [child] if matches!( - child.asap(), - Some(ASAPOp::SummaryTumblingWindowChain { .. }) + child.operator(), + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) ) => { - Some(child) + Some(child.clone()) } _ => None, }, @@ -53,45 +56,51 @@ pub fn merged_chain(state: &OperatorNode) -> Option<&OperatorNode> { } } -/// How many of its chain's tumbling windows, newest first, a summary-state node with +/// How many of its chain's tumbling windows, newest first, a summary state with /// a [`merged_chain`] merges: the merge's `newest_tumbling_windows`, otherwise the -/// chain's `tumbling_window_count`. `None` for any other node. -fn merged_tumbling_window_count(state: &OperatorNode) -> Option { - let Some(ASAPOp::SummaryTumblingWindowChain { +/// chain's `tumbling_window_count`. `None` for any other input. +fn merged_tumbling_window_count(state: &I) -> Option { + let Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { tumbling_window_count, .. - }) = merged_chain(state)?.asap() + }) = merged_chain(state)?.operator() else { return None; }; - match state.asap()? { - ASAPOp::SummaryMerge { + match state.operator() { + Operator::ASAP(ASAPOp::SummaryMerge { newest_tumbling_windows: Some(newest), .. - } => Some(*newest), - _ => Some(*tumbling_window_count), + }) => Some(newest), + _ => Some(tumbling_window_count), } } +/// A build over the inputs of a chain over `I`, with its schema. +pub(crate) type BuildWithSchema = (ASAPOp>, Schema); + /// The `SummaryAgg` over `TimeRange(tumbling_window_count · tumbling_window_width_ms)` over the /// chain's input whose state the merged tumbling windows of `chain` equal. -pub(crate) fn whole_window_build( - chain: &OperatorNode, -) -> Result { - let Some(ASAPOp::SummaryTumblingWindowChain { +#[cfg(any(test, feature = "test-support"))] +pub(crate) fn whole_window_build( + chain: &I, +) -> Result, SchemaDerivationError> { + let Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { tumbling_window_count, .. - }) = chain.asap() + }) = chain.operator() else { return Err(invalid("expected a tumbling-window chain")); }; - newest_tumbling_windows_build(chain, *tumbling_window_count) + newest_tumbling_windows_build(chain, tumbling_window_count) } -/// The `SummaryAgg` whose state a summary-state node with a +/// The `SummaryAgg` whose state a summary state with a /// [`merged_chain`] carries: one build over the window of the tumbling windows it /// merges (`merged_tumbling_window_count`). -pub fn merged_window_build(state: &OperatorNode) -> Result { +pub(crate) fn merged_window_build( + state: &I, +) -> Result, SchemaDerivationError> { let (Some(chain), Some(tumbling_window_count)) = (merged_chain(state), merged_tumbling_window_count(state)) else { @@ -99,63 +108,89 @@ pub fn merged_window_build(state: &OperatorNode) -> Result( + chain: &I, + tumbling_window_count: u32, +) -> Result, SchemaDerivationError> { + let Operator::ASAP(chain_op) = chain.operator() else { + return Err(invalid("expected a tumbling-window chain")); + }; + named_as_chain( + window_build(&chain_op, tumbling_window_count)?, + chain.schema(), + ) +} + +/// [`newest_tumbling_windows_build`] of the chain operator `chain`, without +/// its schema. +pub(crate) fn window_build( + chain: &ASAPOp, tumbling_window_count: u32, -) -> Result { - let Some(ASAPOp::SummaryTumblingWindowChain { +) -> Result>, SchemaDerivationError> { + let ASAPOp::SummaryTumblingWindowChain { child, tumbling_window_width_ms, .. - }) = chain.asap() + } = chain else { return Err(invalid("expected a tumbling-window chain")); }; let window = tumbling_window_width_ms .checked_mul(u64::from(tumbling_window_count)) .ok_or_else(|| invalid("tumbling-window chain window overflows"))?; - build_over(chain, range(Rc::clone(child), window)?) + build_over( + chain, + range(InputOrTemporaryBuild::Input(child.clone()), window)?, + ) } +#[cfg(any(test, feature = "test-support"))] /// Tumbling window `index` (0 = newest) of `chain`: its `SummaryAgg` over the range /// `TimeRange(tumbling_window_width_ms)` ending `index · tumbling_window_width_ms` before the /// chain's input. -pub(crate) fn tumbling_window( - chain: &OperatorNode, +pub(crate) fn tumbling_window( + chain: &I, index: u32, -) -> Result { - let Some(ASAPOp::SummaryTumblingWindowChain { +) -> Result, SchemaDerivationError> { + let Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { child, tumbling_window_width_ms, .. - }) = chain.asap() + }) = chain.operator() else { return Err(invalid("expected a tumbling-window chain")); }; let back = i64::try_from(tumbling_window_width_ms.saturating_mul(u64::from(index))) .map_err(|_| invalid("tumbling window offset overflows"))?; - build_over( - chain, - range(shifted(child, back)?, *tumbling_window_width_ms)?, + let Operator::ASAP(chain_op) = chain.operator() else { + return Err(invalid("expected a tumbling-window chain")); + }; + named_as_chain( + build_over( + &chain_op, + range(shifted(child, back)?, tumbling_window_width_ms)?, + )?, + chain.schema(), ) } +#[cfg(any(test, feature = "test-support"))] /// `child` evaluated `back_ms` earlier: one `TimeShift`, folded into a /// `TimeShift` directly below. -fn shifted( - child: &Rc, +fn shifted( + child: I, back_ms: i64, -) -> Result, SchemaDerivationError> { +) -> Result, SchemaDerivationError> { if back_ms == 0 { - return Ok(Rc::clone(child)); + return Ok(InputOrTemporaryBuild::Input(child)); } - let (shift, input) = match child.non_asap() { - Some(NonASAPOp::TimeShift { + let (shift, input) = match child.operator() { + Operator::NonASAP(NonASAPOp::TimeShift { shift, child: input, }) => ( @@ -176,64 +211,77 @@ fn shifted( child, ), }; - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeShift { + InputOrTemporaryBuild::build(NonASAPOp::TimeShift { shift, - child: Rc::clone(input), - })) + child: InputOrTemporaryBuild::Input(input), + }) } -fn range( - child: Rc, +/// A range `TimeRange(width_ms)` over `child`. +fn range( + child: InputOrTemporaryBuild, width_ms: u64, -) -> Result, SchemaDerivationError> { - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeRange { +) -> Result, SchemaDerivationError> { + InputOrTemporaryBuild::build(NonASAPOp::TimeRange { range: Duration::from_millis(width_ms), kind: TimeRangeKind::Range, child, - })) + }) } -/// The chain's `SummaryAgg` over `input`, with the chain's column names. -fn build_over( - chain: &OperatorNode, - input: Rc, -) -> Result { - let Some(ASAPOp::SummaryTumblingWindowChain { +/// The chain's `SummaryAgg` over `input`. +fn build_over( + chain: &ASAPOp, + input: InputOrTemporaryBuild, +) -> Result>, SchemaDerivationError> { + let ASAPOp::SummaryTumblingWindowChain { family, input: update, reduction, grouping, filter, .. - }) = chain.asap() + } = chain.map_children(|child| InputOrTemporaryBuild::Input(child.clone())) else { return Err(invalid("expected a tumbling-window chain")); }; - let mut build = OperatorNode::new(Operator::ASAP(ASAPOp::SummaryAgg { + Ok(ASAPOp::SummaryAgg { child: input, - family: family.clone(), - input: update.clone(), - reduction: reduction.clone(), - grouping: grouping.clone(), - filter: filter.clone(), - }))?; - for (field, kept) in build.schema.fields.iter_mut().zip(&chain.schema.fields) { + family, + input: update, + reduction, + grouping, + filter, + }) +} + +/// `build` with its schema, named as the chain of schema `chain`. +fn named_as_chain( + build: ASAPOp>, + chain: &Schema, +) -> Result, SchemaDerivationError> { + let ASAPOp::SummaryAgg { + child, + family, + reduction, + .. + } = &build + else { + return Err(invalid("expected a summary build")); + }; + let mut schema = summary_agg_schema(child.schema(), family, reduction)?; + for (field, kept) in schema.fields.iter_mut().zip(&chain.fields) { field.name = kept.name.clone(); field.table = kept.table.clone(); } - Ok(build) + Ok((build, schema)) } /// `root` with every `SummaryTumblingWindowChain` expanded into its tumbling windows: a `SummaryMerge` /// reads each tumbling window of each chain it read, newest first, as plain /// `SummaryAgg` nodes the executor runs; a merge of the newest `n` tumbling windows /// reads the first `n`, the same nodes a merge of the whole chain reads. Every other node keeps its -/// guarantee, timing and materialization. Each tumbling window takes its chain's -/// timing and materialization, and the range and shift it reads take the -/// chain's timing: a chain materialized at ingestion time expands into -/// tumbling windows each built and stored at ingestion time. Expanding before timings -/// are applied leaves the tumbling windows untimed. A chain read by anything but a -/// `SummaryMerge` is an error. +/// guarantee. A chain read by anything but a `SummaryMerge` is an error. #[cfg(any(test, feature = "test-support"))] pub fn expand_tumbling_window_chains( root: &Rc, @@ -241,110 +289,7 @@ pub fn expand_tumbling_window_chains( expand(root, &mut NodeToExpandedNodeCache::default()) } -/// A workload with its tumbling-window chains expanded -/// ([`expand_tumbling_window_chains_in_workload`]). -#[derive(Debug, Clone)] -pub struct TumblingWindowChainExpansion { - /// The workload's roots, in order, with every chain expanded. - pub roots: Vec, - /// Each expanded chain with its tumbling windows some merge reads, newest first, - /// in the order the chains were first reached. - pub chains: Vec, - /// Every other node of the input workload the expansion reached, with - /// the node that replaces it (itself when nothing below it changed), - /// so a caller can carry what it knows about a node to the expanded - /// workload. - pub expanded_nodes: Vec<(Rc, Rc)>, -} - -/// One expanded chain: the `SummaryTumblingWindowChain` node of the input workload and the -/// `SummaryAgg` tumbling windows that replace it. -#[derive(Debug, Clone)] -pub struct ExpandedTumblingWindowChain { - /// The chain node, as the input workload holds it. - pub chain: Rc, - /// Its tumbling windows in the expanded workload that some merge reads, newest - /// first. - pub tumbling_windows: Vec>, -} - -/// `expand_tumbling_window_chains` over every root of one workload, with one cache, so -/// a node several roots share stays one node and a chain several merges -/// read expands into one set of tumbling windows. A standalone scalar root has the -/// operators it reads expanded. Also returns each chain's tumbling windows, so a -/// caller can carry what it knows about a chain (e.g. its retention) to -/// them. -pub fn expand_tumbling_window_chains_in_workload( - roots: &[QueryRoot], -) -> Result { - let mut cache = NodeToExpandedNodeCache::default(); - let roots = roots - .iter() - .map(|root| match root { - QueryRoot::Operator(node) => expand(node, &mut cache).map(QueryRoot::Operator), - QueryRoot::Scalar(expr) => { - let mut error = None; - let expr = expr.map_operator_refs(&mut |node| { - expand(node, &mut cache).unwrap_or_else(|e| { - error.get_or_insert(e); - Rc::clone(node) - }) - }); - match error { - Some(e) => Err(e), - None => Ok(QueryRoot::Scalar(expr)), - } - } - }) - .collect::, _>>()?; - let chains = cache - .chains - .into_iter() - .map(|chain| { - let tumbling_window_count = match chain.asap() { - Some(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_count, - .. - }) => *tumbling_window_count, - _ => unreachable!("only chains are recorded"), - }; - let tumbling_windows = (0..tumbling_window_count) - .map_while(|index| { - cache - .tumbling_window_of - .get(&(Rc::as_ptr(&chain), index)) - .cloned() - }) - .collect(); - ExpandedTumblingWindowChain { - chain, - tumbling_windows, - } - }) - .collect(); - Ok(TumblingWindowChainExpansion { - roots, - chains, - expanded_nodes: cache.expanded, - }) -} - -/// The untimed `node` with `timing` written into it and into every untimed -/// node below it: the nodes [`tumbling_window`] created, over the chain's input, which -/// the timing pass already timed. -fn timed_like(node: &Rc, timing: ExecutionTiming) -> OperatorNode { - OperatorNode::with_schema( - node.operator.map_children(|child| match child.timing { - Some(_) => Rc::clone(child), - None => Rc::new(timed_like(child, timing)), - }), - node.schema.clone(), - ) - .with_guarantee(node.guarantee.clone()) - .with_materialization(node.materialization) - .with_timing(Some(timing)) -} - +#[cfg(any(test, feature = "test-support"))] /// What one expansion has built: each expanded node by its input's /// address, and each tumbling window by its chain's address and index, so a chain two /// merges read expands into one set of tumbling windows. @@ -358,6 +303,7 @@ struct NodeToExpandedNodeCache { expanded: Vec<(Rc, Rc)>, } +#[cfg(any(test, feature = "test-support"))] fn expand( node: &Rc, cache: &mut NodeToExpandedNodeCache, @@ -389,14 +335,8 @@ fn expand( inputs.push(Rc::clone(done)); continue; } - let tumbling_window = Rc::new(tumbling_window(child, index)?); - let tumbling_window = match child.timing { - Some(timing) => Rc::new( - timed_like(&tumbling_window, timing) - .with_materialization(child.materialization), - ), - None => tumbling_window, - }; + let tumbling_window = + Rc::new(node_of_build(tumbling_window(child, index)?)); if index == 0 { cache.chains.push(Rc::clone(child)); } @@ -416,9 +356,7 @@ fn expand( }))?; Rc::new( OperatorNode::with_schema(merge.operator, node.schema.clone()) - .with_guarantee(node.guarantee.clone()) - .with_timing(node.timing) - .with_materialization(node.materialization), + .with_guarantee(node.guarantee.clone()), ) } _ => { @@ -436,9 +374,7 @@ fn expand( .map_children(|child| Rc::clone(&children[&Rc::as_ptr(child)])), node.schema.clone(), ) - .with_guarantee(node.guarantee.clone()) - .with_timing(node.timing) - .with_materialization(node.materialization), + .with_guarantee(node.guarantee.clone()), ) } else { Rc::clone(node) diff --git a/crates/types/src/ir/physical_export.rs b/crates/types/src/ir/physical_export.rs index ccb07e256..f6d383b28 100644 --- a/crates/types/src/ir/physical_export.rs +++ b/crates/types/src/ir/physical_export.rs @@ -1,11 +1,11 @@ //! Physical ASAP DAG (planner-layering stage 2 output). //! -//! The flattened operators of [`super::flat`], plus the materialization and -//! execution timing (data state) of every node and edge. The input must -//! already be timed ([`crate::ir::properties::timing::apply_materialization_timings`]); export -//! reads each node's timing and materialization and does not re-run -//! data-state validation. The deployment executes the plan as given -//! (#509 Assumption 8): it does not change materialization or timing. +//! The flattened operators of [`super::flat`], plus the lifetime and +//! execution timing (data state) of every node and edge. The realizer +//! (`AssignmentRealizer::realize`) builds it from a `DagOfSelectedCandidates`, +//! each node's timing and lifetime from the `PhysicalCandidate` it realizes. +//! The deployment executes the plan as given +//! (#509 Assumption 8): it does not change lifetime or timing. use std::collections::{HashMap, HashSet}; use std::rc::Rc; @@ -13,24 +13,16 @@ use std::rc::Rc; use serde::{Deserialize, Serialize}; use thiserror::Error; -use super::flat::{flatten, NodeId}; +use super::flat::NodeId; use super::scalar::ScalarExpr; use crate::ir::operator::asap::ASAPOp; use crate::ir::operator::node::{Operator, OperatorNode}; use crate::ir::operator::non_asap::NonASAPOp; -use crate::ir::operator::operator_properties::Reduction; -use crate::ir::operator::query::QueryRoot; -use crate::ir::properties::execution_data_state::{ - ExecutionDataState, ExecutionDataStateError, ExecutionTiming, -}; + +use crate::ir::properties::execution_data_state::{ExecutionDataState, ExecutionTiming}; use crate::ir::properties::guarantee::AccuracyGuarantee; -#[cfg(any(test, feature = "test-support"))] -use crate::ir::properties::timing::{ - apply_materialization_timings_to_roots, MaterializationAssignment, -}; -use crate::ir::properties::timing::{data_state, is_materialization_point}; -use crate::ir::schema::{FieldDataType, Schema}; -use crate::physical::materialization::Materialization; +use crate::ir::schema::{FieldDataType, Schema, SketchParams}; +use crate::physical::{Lifetime, PhysicalPropertiesError}; /// A node's operator, with each child replaced by its node id. pub type PhysicalASAPOperatorPayload = Operator; @@ -71,22 +63,28 @@ pub struct PhysicalASAPDAGNode { pub payload: PhysicalASAPOperatorPayload, /// Phase is a placement choice for every operator, independent of payload kind. pub output_state: ExecutionDataState, - /// Whether and where the deployment stores this node's output: the - /// option the materialization assignment chose for the sub-DAG this node - /// produces. Any node may be stored (#509 §Materialization); a node with - /// no assignment is `NotMaterialized`. - pub materialization: Materialization, + /// How long and where the deployment materializes this node's output: the + /// lifetime the physical properties assignment chose for the sub-DAG this + /// node produces. Any node may be materialized (#509 §Materialization); a node + /// with no assignment has lifetime `None`. With `output_state.timing` it + /// is the node's [`PhysicalProperties`]. + pub lifetime: Lifetime, + /// θ of a summary build (`SummaryAgg`, `SummaryTumblingWindowChain`): its + /// candidate's `LogicalProperties::parameters`, which the executor builds + /// the sketch from. `None` on every other node and for an exact build. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub parameters: Option, pub output_schema: Schema, pub guarantee: Option, - /// For a materialized view (a node whose materialization keeps its - /// output across evaluations), how far back from the newest evaluation + /// For a materialized view (a node whose lifetime materializes its output + /// across evaluations), how far back from the newest evaluation /// time its output must stay available once refreshed; for a query-time /// `Scan`, how far back its source's rows must stay available (`None` /// when it reads them unbounded). `None` on every other node, and on /// every node of a DAG exported before installation sets it /// ([`PhysicalASAPDAG::set_retention`]). A - /// node field rather than part of [`Materialization`], because Stage 2 - /// enumerates materializations before Stage 3 knows retention. + /// node field rather than part of [`Lifetime`], because Stage 2 + /// enumerates lifetimes before Stage 3 knows retention. #[serde(default, skip_serializing_if = "Option::is_none")] pub retention_ms: Option, } @@ -171,14 +169,19 @@ pub struct PhysicalASAPDAG { #[derive(Debug, Clone, PartialEq, Eq, Error)] pub enum PhysicalASAPDAGValidationError { - #[error( - "node {node:?} runs at {timing:?}, which its materialization {materialization:?} does not" - )] - MaterializationTimingMismatch { + /// A node whose timing and lifetime cannot go together: a planner bug. + #[error("node {node:?}: {reason}")] + PhysicalProperties { node: PhysicalASAPNodeId, - timing: ExecutionTiming, - materialization: Materialization, + #[source] + reason: PhysicalPropertiesError, }, + /// A summary build (or maintained population) that is not materialized but does + /// not run at query time: a planner bug. + #[error( + "node {node:?} is a summary build that is not materialized but does not run at query time" + )] + UnmaterializedSummaryBuildNotAtQueryTime { node: PhysicalASAPNodeId }, #[error("ingestion node {consumer:?} depends on query node {producer:?}")] QueryDependencyInIngestion { producer: PhysicalASAPNodeId, @@ -222,7 +225,7 @@ pub enum PhysicalASAPDAGValidationError { SummaryGroupingMismatch { node: PhysicalASAPNodeId }, } -/// Which rows of a `Scan` its readers read, back from an evaluation time `T` +/// Which rows of a `Scan` its consumers read, back from an evaluation time `T` /// ([`PhysicalASAPDAG::scan_window`]). #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] pub enum ScanWindow { @@ -337,39 +340,9 @@ pub(crate) fn offset_below<'a>( Some(end_offset_ms) } -/// Why `PhysicalASAPDAG::with_every_node_at_query_time` failed. -#[derive(Debug, Error)] -pub enum ReferencePlanError { - /// Bad input: the DAG is not a valid physical ASAP DAG. - #[error("invalid physical ASAP DAG: {0}")] - InvalidDag(#[from] PhysicalASAPDAGValidationError), - /// Bad input: the plan cannot run at query time (a data-state rule of - /// the timing pass fails). - #[error("the plan does not run at query time: {0}")] - Timing(#[from] ExecutionDataStateError), -} - impl PhysicalASAPDAG { - /// This DAG's plan with nothing materialized: the same operators, - /// summaries and summary parameters, with every sub-DAG - /// `NotMaterialized` and so every node at query time (the default - /// [`MaterializationAssignment`]). Node ids are renumbered, and `roots` - /// and `scalar_roots` keep their order. - /// - /// The correctness reference of a selected plan: a deployment runtime - /// compares the answers of the plan it installed with the executor - /// running this one over raw data, which needs the raw data the - /// installed plan may not keep. - #[cfg(any(test, feature = "test-support"))] - pub fn with_every_node_at_query_time(&self) -> Result { - let roots = self.untimed_query_roots()?; - let timed = - apply_materialization_timings_to_roots(&roots, &MaterializationAssignment::default())?; - Ok(compile_physical_asap_workload(&timed)?) - } - - /// Every node of this DAG as IR, by node id, untimed and - /// `NotMaterialized` with its exported guarantee: what a caller derives + /// Every node of this DAG as IR, by node id, with its exported + /// guarantee (an operator node has no physical properties): what a caller derives /// IR properties of a plan file's node from, such as its /// [`SummaryCoverage`](crate::ir::properties::summary_coverage::SummaryCoverage). /// Shared inputs stay shared. Validates the DAG first. @@ -404,32 +377,32 @@ impl PhysicalASAPDAG { Ok(rebuilt) } - /// The rows of the `Scan` `scan` its readers read, back from the - /// evaluation time `T`: the union of each reader's window. A reader + /// The rows of the `Scan` `scan` its consumers read, back from the + /// evaluation time `T`: the union of each consumer's window. A consumer /// reaches the `Scan` through `TimeShift`s without an `@` anchor /// (offsets summing to `s`) and is /// - a `TimeRange(w)`, range or instant: `(T - s - w, T - s]`; /// - a `SummaryTumblingWindowChain` of `n` tumbling windows of width `w`: its whole window `(T - s - /// n·w, T - s]`, or only its newest tumbling window `(T - s - w, T - s]` for a - /// chain in `kept_chains` (kept across evaluations, so each + /// chain in `materialized_chains` (materialized across evaluations, so each /// evaluation builds one tumbling window). /// - /// `Unbounded` when a root, any other reader, or a shift with an `@` + /// `Unbounded` when a root, any other consumer, or a shift with an `@` /// anchor reads the `Scan` or one of those shifts, or nothing reads it. pub fn scan_window( &self, scan: PhysicalASAPNodeId, - kept_chains: &HashSet, + materialized_chains: &HashSet, ) -> ScanWindow { - self.scan_windows([scan], kept_chains)[&scan] + self.scan_windows([scan], materialized_chains)[&scan] } /// [`Self::scan_window`] of each of `scans`, by id, with the DAG - /// indexed once for all of them: O(nodes + edges + the readers walked). + /// indexed once for all of them: O(nodes + edges + the consumers walked). pub fn scan_windows( &self, scans: impl IntoIterator, - kept_chains: &HashSet, + materialized_chains: &HashSet, ) -> HashMap { let payloads = self.payloads_by_id(); let mut consumers: HashMap> = HashMap::new(); @@ -461,7 +434,7 @@ impl PhysicalASAPDAG { Some(Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { tumbling_window_width_ms, .. - })) if kept_chains.contains(&consumer) => { + })) if materialized_chains.contains(&consumer) => { u128::from(*tumbling_window_width_ms) } Some(Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { @@ -567,54 +540,35 @@ impl PhysicalASAPDAG { .collect() } - /// The IR this DAG exports, one root per query in batch order, every - /// node untimed and `NotMaterialized`: the inverse of - /// [`compile_physical_asap_workload`] up to `Rc` identity and timing. - #[cfg(any(test, feature = "test-support"))] - fn untimed_query_roots(&self) -> Result, PhysicalASAPDAGValidationError> { - let rebuilt = self.operator_nodes()?; - let mut operator_roots = self.roots.iter(); - let mut scalar_roots = self.scalar_roots.iter().peekable(); - let mut roots = Vec::with_capacity(self.roots.len() + self.scalar_roots.len()); - for position in 0..self.roots.len() + self.scalar_roots.len() { - match scalar_roots.peek() { - Some((at, expr)) if *at == position => { - roots.push(QueryRoot::Scalar( - expr.map_operator_refs(&mut |id| Rc::clone(&rebuilt[id])), - )); - scalar_roots.next(); - } - _ => { - let id = operator_roots.next().expect("validated root positions"); - roots.push(QueryRoot::Operator(Rc::clone(&rebuilt[id]))); - } - } - } - Ok(roots) - } - pub fn validate(&self) -> Result<(), PhysicalASAPDAGValidationError> { let mut nodes = HashMap::new(); for node in &self.nodes { if nodes.insert(node.id, node).is_some() { return Err(PhysicalASAPDAGValidationError::DuplicateNodeId(node.id)); } - // A sub-DAG output runs when its materialization says; any other - // node is not stored and runs when its consumer runs. - if (is_materialization_point(&node.payload) - || node.materialization != Materialization::NotMaterialized) - && node.materialization.timing() != node.output_state.timing + // A node's timing and lifetime must go together; a summary + // build that is not materialized runs at query time. Any other node that + // is not materialized runs when its consumer runs. + if node.output_state.timing == ExecutionTiming::IngestionTime + && matches!(node.lifetime, Lifetime::OneEvaluation(_)) + { + return Err(PhysicalASAPDAGValidationError::PhysicalProperties { + node: node.id, + reason: PhysicalPropertiesError::OneEvaluationAtIngestion, + }); + } + if is_materializable_state_output(&node.payload) + && node.output_state.timing == ExecutionTiming::IngestionTime + && node.lifetime == Lifetime::None { return Err( - PhysicalASAPDAGValidationError::MaterializationTimingMismatch { + PhysicalASAPDAGValidationError::UnmaterializedSummaryBuildNotAtQueryTime { node: node.id, - timing: node.output_state.timing, - materialization: node.materialization, }, ); } if node.retention_ms.is_some() - && !node.materialization.is_materialized_view() + && !matches!(node.lifetime, Lifetime::AcrossEvaluations(_)) && !super::materialized_view::is_query_time_scan(node) { return Err(PhysicalASAPDAGValidationError::UnexpectedRetention(node.id)); @@ -800,65 +754,66 @@ impl PhysicalASAPDAG { } } -// ── Compilation from the IR ────────────────────────────────────────────── - -/// Compiler-local identity assignment. It deliberately retains `Rc` handles -/// and is not serialized; deployed artifacts persist the physical ASAP node ID -/// together with their physical materialization/query IDs. -#[derive(Debug, Clone)] -pub struct PhysicalASAPNodeIdentityMap { - nodes_by_id: Vec>, -} - -impl PhysicalASAPNodeIdentityMap { - #[cfg(any(test, feature = "test-support"))] - pub fn node_id(&self, node: &Rc) -> Option { - self.nodes_by_id - .iter() - .position(|candidate| Rc::ptr_eq(candidate, node)) - } - - pub fn operator_node(&self, id: PhysicalASAPNodeId) -> Option<&Rc> { - self.nodes_by_id.get(id) +/// Each input of `operator` with its edge role: the operator inputs, then the +/// nodes read by its scalar expressions (the order of [`Operator::children`]). +/// A node referenced several times from the scalar expressions is one +/// `ScalarRef` input, as cost lowering reads it once; ids are per node +/// pointer, so this dedupes by pointer. +fn edge_roles(operator: &Operator) -> Vec<(C, EdgeRole)> { + use EdgeRole::*; + let inputs: Vec = match operator { + Operator::NonASAP(op) => match op { + NonASAPOp::Join { .. } | NonASAPOp::SetOp { .. } | NonASAPOp::BinaryOp { .. } => { + vec![Left, Right] + } + NonASAPOp::Concat { children, .. } => vec![Input; children.len()], + NonASAPOp::Scan { .. } + | NonASAPOp::Values { .. } + | NonASAPOp::PromqlVectorFromScalar(_) => vec![], + _ => vec![Input], + }, + Operator::ASAP(op) => match op { + ASAPOp::SummarySubtract { .. } | ASAPOp::SummaryJoin { .. } => vec![Left, Right], + ASAPOp::SummaryMerge { children, .. } => vec![Input; children.len()], + _ => vec![Input], + }, + }; + let children = operator.children(); + let (direct, scalar_refs) = children.split_at(inputs.len()); + let mut edges: Vec<(C, EdgeRole)> = direct.iter().map(|c| **c).zip(inputs).collect(); + for producer in scalar_refs { + if !edges.contains(&(**producer, ScalarRef)) { + edges.push((**producer, ScalarRef)); + } } + edges } -#[derive(Debug, Clone)] -pub struct PhysicalASAPDAGCompilation { - pub dag: PhysicalASAPDAG, - pub node_ids: PhysicalASAPNodeIdentityMap, -} - -#[cfg(any(test, feature = "test-support"))] -pub fn compile_physical_asap_dag( - root: &Rc, -) -> Result { - Ok(compile_physical_asap_dag_with_node_ids(root)?.dag) +/// Whether `operator`'s output is a sub-DAG output that materialization may +/// store: a `SummaryAgg`, a `SummaryTumblingWindowChain` or a `MaintainPopulation`. +pub(crate) fn is_materializable_state_output(operator: &Operator) -> bool { + matches!( + operator, + Operator::ASAP( + ASAPOp::SummaryAgg { .. } + | ASAPOp::SummaryTumblingWindowChain { .. } + | ASAPOp::MaintainPopulation { .. } + ) + ) } -/// Export the timed DAG below `root`. Every reachable node must carry a -/// timing (see [`crate::ir::properties::timing::apply_materialization_timings`]); the data-state -/// rules were checked by that pass and are not re-run here. +/// Test-only: `roots` exported as one DAG with every node at query time and +/// not materialized, and no θ (every `parameters` is `None`), with each +/// exported node's operator node by node id. Executor tests hand-build DAGs +/// this way; planning exports only through the realizer +/// (`AssignmentRealizer::realize`). #[cfg(any(test, feature = "test-support"))] -pub(crate) fn compile_physical_asap_dag_with_node_ids( - root: &Rc, -) -> Result { - compile_physical_asap_workload_with_node_ids(&[QueryRoot::Operator(Rc::clone(root))]) -} - -/// Export a timed batch as one DAG: operator queries become `roots` and -/// standalone scalar queries `scalar_roots`, keyed by their position in the -/// batch. -pub fn compile_physical_asap_workload( - roots: &[QueryRoot], -) -> Result { - Ok(compile_physical_asap_workload_with_node_ids(roots)?.dag) -} - -pub fn compile_physical_asap_workload_with_node_ids( - roots: &[QueryRoot], -) -> Result { - let (flat, nodes_by_id) = flatten(roots); +pub(crate) fn physical_asap_dag_at_query_time_with_node_ids( + roots: &[crate::ir::QueryRoot], +) -> Result<(PhysicalASAPDAG, Vec>), PhysicalASAPDAGValidationError> { + use crate::ir::properties::execution_data_state::DataPrimitive; + use crate::ir::QueryRoot; + let (flat, nodes_by_id) = super::flat::flatten(roots); let mut scalar_roots = Vec::new(); let roots = flat .roots @@ -876,33 +831,36 @@ pub fn compile_physical_asap_workload_with_node_ids( let mut edges = Vec::new(); for (id, flat_node) in flat.nodes.into_iter().enumerate() { let node = &nodes_by_id[id]; - let output_state = data_state(node).ok_or(ExecutionDataStateError::UntimedNode { - operator: node.operator.kind_name(), - })?; + let primitive = match &node.operator { + Operator::ASAP(op) if op.produced_state().is_some() => DataPrimitive::SummaryState, + Operator::ASAP(ASAPOp::MaintainPopulation { .. }) => DataPrimitive::SummaryState, + _ if node.result_kind == crate::ir::OperatorResultKind::State => { + DataPrimitive::SummaryState + } + _ => DataPrimitive::Raw, + }; + let output_state = ExecutionDataState { + timing: ExecutionTiming::QueryTime, + primitive, + }; // Children come before their parents, so every producer is already in `nodes`. for (producer, role) in edge_roles(&flat_node.operator) { - let producer_state = nodes[producer].output_state; - let ingestion_dependency = producer_state.timing == ExecutionTiming::IngestionTime - && output_state.timing == ExecutionTiming::IngestionTime; edges.push(PhysicalASAPDAGEdge { producer, consumer: id, role, intermediate_schema: nodes[producer].output_schema.clone(), - data_state: producer_state, + data_state: nodes[producer].output_state, grouping: grouping_compatibility(&nodes[producer].payload, &flat_node.operator), - window: if ingestion_dependency { - WindowEdgeCompatibility::RequiresAlignedTumblingWindowPhaseOrExactWindowEdgeResidual - } else { - WindowEdgeCompatibility::NotApplicable - }, + window: WindowEdgeCompatibility::NotApplicable, }); } nodes.push(PhysicalASAPDAGNode { id, payload: flat_node.operator, output_state, - materialization: flat_node.materialization, + lifetime: Lifetime::None, + parameters: None, output_schema: flat_node.schema, guarantee: flat_node.guarantee, retention_ms: None, @@ -914,52 +872,16 @@ pub fn compile_physical_asap_workload_with_node_ids( roots, scalar_roots, }; - dag.validate().map_err(ExecutionDataStateError::from)?; - Ok(PhysicalASAPDAGCompilation { - dag, - node_ids: PhysicalASAPNodeIdentityMap { nodes_by_id }, - }) -} - -/// Each input of `operator` with its edge role: the operator inputs, then the -/// nodes read by its scalar expressions (the order of [`Operator::children`]). -/// A node referenced several times from the scalar expressions is one -/// `ScalarRef` input, as cost lowering reads it once; ids are per node -/// pointer, so this dedupes by pointer. -fn edge_roles(operator: &Operator) -> Vec<(C, EdgeRole)> { - use EdgeRole::*; - let inputs: Vec = match operator { - Operator::NonASAP(op) => match op { - NonASAPOp::Join { .. } | NonASAPOp::SetOp { .. } | NonASAPOp::BinaryOp { .. } => { - vec![Left, Right] - } - NonASAPOp::Concat { children, .. } => vec![Input; children.len()], - NonASAPOp::Scan { .. } - | NonASAPOp::Values { .. } - | NonASAPOp::PromqlVectorFromScalar(_) => vec![], - _ => vec![Input], - }, - Operator::ASAP(op) => match op { - ASAPOp::SummarySubtract { .. } | ASAPOp::SummaryJoin { .. } => vec![Left, Right], - ASAPOp::SummaryMerge { children, .. } => vec![Input; children.len()], - _ => vec![Input], - }, - }; - let children = operator.children(); - let (direct, scalar_refs) = children.split_at(inputs.len()); - let mut edges: Vec<(C, EdgeRole)> = direct.iter().map(|c| **c).zip(inputs).collect(); - for producer in scalar_refs { - if !edges.contains(&(**producer, ScalarRef)) { - edges.push((**producer, ScalarRef)); - } - } - edges + dag.validate()?; + Ok((dag, nodes_by_id)) } +#[cfg(any(test, feature = "test-support"))] fn grouping_compatibility( producer: &Operator, consumer: &Operator, ) -> GroupingEdgeCompatibility { + use crate::ir::operator::operator_properties::Reduction; let ( Operator::ASAP(ASAPOp::SummaryAgg { reduction: producer, diff --git a/crates/types/src/ir/properties/execution_data_state.rs b/crates/types/src/ir/properties/execution_data_state.rs index e3a2cda8e..c4c22f04c 100644 --- a/crates/types/src/ir/properties/execution_data_state.rs +++ b/crates/types/src/ir/properties/execution_data_state.rs @@ -2,14 +2,9 @@ //! //! [`ExecutionTiming`] says when a node's value is produced (ingestion vs. //! query time); [`ExecutionDataState`] pairs it with the [`DataPrimitive`] -//! the edge carries (raw values vs. summary state). [`Materialization`] is -//! the per-sub-DAG choice a node's timing is derived from. The rules that assign -//! and check them over a DAG live in [`crate::ir::properties::timing`], which reports -//! violations as [`ExecutionDataStateError`]. - -use thiserror::Error; - -use crate::physical::Materialization; +//! the edge carries (raw values vs. summary state). A node's timing comes +//! from the [`crate::physical::PhysicalProperties`] of the physical candidate +//! it realizes. /// When a post-ASAP value is produced. #[derive( @@ -78,121 +73,6 @@ impl std::fmt::Display for ExecutionDataState { } } -/// A plan-construction-time data_state violation. Typed (not a string) so a -/// strategy can degrade to a conservative fallback on the specific variant -/// it expects, and so tests can assert the *reason* a plan was rejected. -#[derive(Debug, Clone, PartialEq, Eq, Error)] -pub enum ExecutionDataStateError { - #[error("invalid maintained-population maintenance/evaluation contract")] - InvalidMaintainedPopulation, - /// A query-time value (a `SummaryEstimate` or query-time operator output) - /// placed beneath a summary built at ingestion time — the one shape issue - /// #171's data_state split exists to make unrepresentable. - #[error( - "query-time value under an ingestion-time summary: {edge} received a {child} input, but a \ - summary built at ingestion time can only consume ingestion-time rows (or exact \ - accumulator state)" - )] - QueryTimeValueUnderIngestionSummary { - edge: &'static str, - child: ExecutionDataState, - }, - /// A summary build at query time over an input (rows or exact-accumulator - /// state) produced at ingestion time and not stored there (the input node - /// is not `MatAtIngestion`): at query time that input no longer exists. - /// Bad input (a hand-built or candidate-fixed plan), not an internal bug. - #[error( - "a query-time {edge} reads {operator} output produced at ingestion time that is not \ - stored (not MatAtIngestion)" - )] - UnstoredIngestionInputUnderQueryTimeSummary { - edge: &'static str, - operator: &'static str, - }, - /// Any other edge whose child data_state the parent does not accept - /// (e.g. plain update rows fed straight into a `SummaryEstimate`, or a - /// sketch's opaque state fed into a query-time operator). - #[error("{edge} does not accept a {child} input")] - IllegalChildDataState { - edge: &'static str, - child: ExecutionDataState, - }, - /// A `SummaryAgg` whose child is summary state of a family other than an - /// exact accumulator — re-accumulating opaque sketch/sample/… state on - /// the update path has no defined semantics here. - #[error( - "SummaryAgg.child carries {family} summary state; only exact accumulator state can be \ - composed into another maintained summary" - )] - UnsupportedStateComposition { family: String }, - /// One shared node assigned two different execution timings by its - /// consumers; no single execution of it can serve both. - #[error("shared node is assigned conflicting timings: {first} and {second}")] - ConflictingTiming { - first: ExecutionDataState, - second: ExecutionDataState, - }, - /// A plan root that runs at ingestion time without being stored there - /// (a timing fixed by a candidate builder): no query reads its rows. A - /// root assigned `MatAtIngestion` is valid; the query reads the stored - /// output. - #[error( - "{operator} root runs at ingestion time but is not MatAtIngestion: no query reads its \ - output" - )] - UnstoredIngestionRoot { operator: &'static str }, - /// An explicit materialization on a node whose timing a candidate builder - /// fixed (e.g. a query-time `FinalizeExactAccumulator` read boundary) - /// that needs the other timing. - #[error( - "{operator} is fixed at {} by its candidate; materialization {materialization:?} needs \ - the other timing", - fixed.as_str() - )] - MaterializationConflictsWithFixedTiming { - operator: &'static str, - fixed: ExecutionTiming, - materialization: Materialization, - }, - /// A summary build with a filter placed at ingestion time. An - /// ingestion-time build reads raw data samples, which carry none of the - /// query's columns the filter reads, so only a query-time build can - /// apply it. - #[error("{operator} has a filter, which only a query-time build can apply")] - FilteredSummaryAtIngestion { operator: &'static str }, - /// An ingestion-time `BinaryOp` the update path cannot evaluate: not - /// arithmetic without vector matching, operands that are not the same - /// per-series rows, or a schema other than non-null timestamps, at most - /// one series identity and one Float64 value. - #[error("unsupported ingestion-time binary schema or operator")] - InvalidIngestionTimeBinary, - #[error("checked division requires one valid guard on a read-time division operator")] - InvalidCheckedDivision, - /// An exact operator whose input columns are not all `Plain` at its - /// declared data_state. - #[error("exact operator consumes non-plain column {column:?} ({dtype})")] - NonPlainOperand { column: String, dtype: String }, - /// An ASAP operator with no execution timing contract yet in an - /// executable plan: the reserved `SummarySubtract`, `SummaryDelete`, - /// `SummaryJoin` and `Extension`. - #[error("{operator} has no execution timing contract yet")] - UnimplementedOperator { operator: &'static str }, - /// A node assigned `MatAtIngestion` over a sub-DAG that can only run at - /// query time (#509 Algorithm 3, line 13). - #[error("{operator}: upstream of ingestion-time node not at ingestion time")] - UpstreamNotAtIngestion { operator: &'static str }, - /// A node reached by export without a timing: the materialization timing pass - /// was not applied to the DAG first. - #[error( - "{operator} node has no execution timing; apply materialization timings before export" - )] - UntimedNode { operator: &'static str }, - /// The exported physical DAG failed its own validation, e.g. an empty - /// batch or a hand-timed plan the timing pass would have rejected. - #[error("invalid physical ASAP DAG: {0}")] - InvalidPhysicalDAG(#[from] crate::ir::physical_export::PhysicalASAPDAGValidationError), -} - #[cfg(test)] mod tests { use super::*; diff --git a/crates/types/src/ir/properties/mod.rs b/crates/types/src/ir/properties/mod.rs index 455a489f2..c7730ba8f 100644 --- a/crates/types/src/ir/properties/mod.rs +++ b/crates/types/src/ir/properties/mod.rs @@ -7,21 +7,16 @@ //! ([`AccuracyGuarantee`]). //! - [`execution_data_state`] — when a node's value is produced //! ([`ExecutionTiming`]) and what its edge carries ([`ExecutionDataState`]). -//! - [`timing`] — execution timing written into every node from a -//! materialization assignment, and its validation. //! - [`summary_coverage`] — what a summary state covers. pub mod accuracy; pub mod execution_data_state; pub mod guarantee; pub mod summary_coverage; -pub mod timing; pub use accuracy::{ default_metric_for_root, AccuracyMetric, AccuracyScope, AccuracySpec, AccuracyTarget, NormScale, }; -pub use execution_data_state::{ - DataPrimitive, ExecutionDataState, ExecutionDataStateError, ExecutionTiming, -}; +pub use execution_data_state::{DataPrimitive, ExecutionDataState, ExecutionTiming}; pub use guarantee::{ AccuracyError, AccuracyGuarantee, BoundExpr, CompositionOperator, GuaranteeSource, ProbabilityExpr, VALUES_PER_RESULT, diff --git a/crates/types/src/ir/properties/summary_coverage.rs b/crates/types/src/ir/properties/summary_coverage.rs index 5cf1812ed..0e2dec55e 100644 --- a/crates/types/src/ir/properties/summary_coverage.rs +++ b/crates/types/src/ir/properties/summary_coverage.rs @@ -3,47 +3,69 @@ //! `docs/design_docs/proposals/asap-primitive-schema.md` §4.2, after //! Goldstein & Larson's view matching. //! -//! Coverage is derived from the node, never declared. Walking down from a -//! `SummaryAgg`, a predicate conjunct moves into `selection` when it can be -//! lifted to the `SummaryAgg` (through `Filter`, a range `TimeRange`, a -//! `TimeShift` without `@` and direct-column `Project` items) and it is a -//! value set or an interval on one column. Everything else stays in -//! `definition` as a residual, so two states are merged only when they -//! compute the same thing over disjoint rows. +//! Coverage is derived once for each summary-state memo group, from its +//! input groups, never declared, and stored in its `GroupProperties`. +//! Walking down from a `SummaryAgg` through the input groups' expressions, a +//! predicate conjunct moves into `selection` when it can be lifted to the +//! `SummaryAgg` (through `Filter`, a range `TimeRange`, a `TimeShift` without +//! `@` and direct-column `Project` items) and it is a value set or an +//! interval on one column. Everything else stays in `definition` as a +//! residual, so two states are merged only when they compute the same thing +//! over disjoint rows. Node trees carry no coverage. use std::cmp::Ordering; -use std::collections::{BTreeMap, HashMap, HashSet}; +use std::collections::BTreeMap; use std::ops::Bound; -use std::rc::Rc; use serde::{Deserialize, Serialize}; use thiserror::Error; -use crate::ir::operator::asap::ASAPOp; -use crate::ir::operator::node::{Operator, OperatorNode}; +use crate::ir::operator::asap::{selected_from, summary_agg_schema, ASAPOp}; +use crate::ir::operator::input::{InputOrTemporaryBuild, OperatorInput}; +use crate::ir::operator::node::{Operator, OperatorResultKind}; use crate::ir::operator::non_asap::{NonASAPOp, TimeRangeKind}; use crate::ir::operator::operator_properties::Reduction; +use crate::ir::operator::tumbling_window_chain::window_build; use crate::ir::scalar::expr_ir::{CompareOpKind, ScalarValue}; use crate::ir::scalar::{Predicate, ScalarExpr}; use crate::ir::schema::family_capability::{FamilyCapability, MergeInputRule}; use crate::ir::schema::sketch::GroupingStrategy; use crate::ir::schema::{ColumnId, Schema}; +use crate::ir::SchemaDerivationError; +use crate::stages::{MemoGroupAsInput, MemoGroupId}; + +/// What a summary state computes, with every lifted selection removed. +#[derive(Debug, Clone, PartialEq)] +pub struct SummaryStateDefinition { + /// The group the unlifted operators read: the first input under the + /// SummaryAgg that is not Filter / range TimeRange / TimeShift without @ + /// / direct-column Project. `None` when that input is a `Scan` with a + /// liftable predicate (a PromQL label matcher): it is lifted too, and the + /// `Scan` with the rest of its predicates is the last unlifted operator, + /// so states over one source with different matchers may merge. + pub input_group: Option, + /// The operators not lifted into the selection, listed top down from the + /// SummaryAgg, each with only its residual arguments. `None` marks an + /// operator's main input (the next unlifted operator, or input_group); + /// `Some(g)` is another input group it reads, such as the group a scalar + /// subquery in a Filter reads. Two states may merge only if input_group + /// and these operators are equal. + pub unlifted_operators: Vec>>, +} #[derive(Debug, Clone, PartialEq)] pub struct SummaryCoverage { - /// The `SummaryAgg` with the selection removed from its sub-DAG. - pub(crate) definition: Rc, - /// Union of boxes over the output rows of the definition's child. + pub(crate) definition: SummaryStateDefinition, + /// Union of boxes over the output rows of the unlifted operators' input. pub(crate) selection: Vec, } #[cfg(any(test, feature = "test-support"))] impl SummaryCoverage { - /// The `SummaryAgg` with the selection removed from its sub-DAG. - pub fn definition(&self) -> &Rc { + pub fn definition(&self) -> &SummaryStateDefinition { &self.definition } - /// Union of boxes over the output rows of the definition's child. + /// Union of boxes over the output rows of the unlifted operators' input. pub fn selection(&self) -> &[SelectionBox] { &self.selection } @@ -262,115 +284,124 @@ pub enum CoverageError { RegroupedBuild { reason: String }, } -/// Coverage already derived during one validation, by node address. The -/// nodes must stay alive while it is in use. -pub(crate) type NodeToSummaryCoverageCache = HashMap<*const OperatorNode, SummaryCoverage>; +/// A memo group read as an input, and the inputs a rule builds over such +/// groups. +type GroupInput<'m> = InputOrTemporaryBuild>; impl SummaryCoverage { - /// Coverage of a `SummaryAgg`, `SummaryTumblingWindowChain` or `SummaryMerge`. A chain - /// covers what one build over its whole window does, and a merge of its - /// newest tumbling windows what one build over their window does. A merge fails unless - /// every input has the same definition and their selections are + /// Coverage of a summary state a memo group computes with `operator` + /// over its input groups: a `SummaryAgg`, a `SummaryTumblingWindowChain`, + /// a `SummaryMerge`, or a `Filter` that keeps whole states of a grouped + /// `SummaryAgg`. A chain covers what one build over its whole window + /// does, and a merge of its newest tumbling windows what one build over + /// their window does. A merge reads its inputs' stored coverage and fails + /// unless every input has the same definition and their selections are /// pairwise disjoint; a family whose merge is idempotent - /// ([`MergeInputRule::MayOverlap`] in the family-capability - /// table) may also merge overlapping selections, and a counter family + /// ([`MergeInputRule::MayOverlap`] in the family-capability table) may + /// also merge overlapping selections, and a counter family /// ([`MergeInputRule::DisjointInTime`]) needs disjoint time windows. - pub(crate) fn derive(node: &OperatorNode) -> Result { - Self::derive_cached(node, &mut NodeToSummaryCoverageCache::new()) - } - - /// A stable identity of `definition`: the 128-bit FNV-1a hash, in hex, - /// of its sub-DAG's operators, output kinds and schemas, children - /// first, without planning metadata (`timing`, `guarantee`, - /// `materialization`). Two equal definitions (the equality a merge - /// requires) get equal ids on any node and in any process, so a summary - /// store can key states by this id and a concrete selection. - pub(crate) fn definition_id(&self) -> String { - format!( - "{:032x}", - definition_hash(&self.definition, &mut HashMap::new()) - ) - } - - /// [`derive`](Self::derive), reusing and recording the coverage of - /// every summary state in `cache`, so validating a DAG derives each - /// state once. - pub(crate) fn derive_cached( - node: &OperatorNode, - cache: &mut NodeToSummaryCoverageCache, + pub(crate) fn derive_from_inputs<'m>( + operator: &Operator>, ) -> Result { - let key = node as *const OperatorNode; - if let Some(coverage) = cache.get(&key) { - return Ok(coverage.clone()); - } - if let Some(build) = selected_build(node)? { - let coverage = of_summary_agg(&build); - cache.insert(key, coverage.clone()); - return Ok(coverage); - } - let coverage = match node.asap() { - Some(ASAPOp::SummaryAgg { .. }) => of_summary_agg(node), - // Back-to-back tumbling windows cover the whole window one build over it - // would. - Some(ASAPOp::SummaryTumblingWindowChain { .. }) => of_summary_agg( - &crate::ir::operator::tumbling_window_chain::whole_window_build(node).map_err( - |error| CoverageError::TumblingWindowChainExpansion { - reason: error.to_string(), - }, - )?, - ), - // The newest tumbling windows of one chain cover what one build over their - // window does. - Some(ASAPOp::SummaryMerge { - newest_tumbling_windows: Some(_), + let expansion = + |error: SchemaDerivationError| CoverageError::TumblingWindowChainExpansion { + reason: error.to_string(), + }; + let as_built = |op: &ASAPOp>| -> ASAPOp> { + op.map_children(|input| InputOrTemporaryBuild::Input(*input)) + }; + match operator { + Operator::NonASAP(_) => match selected_build(operator)? { + Some(build) => of_summary_agg(&build), + None => Err(CoverageError::NotSummary), + }, + Operator::ASAP(op @ ASAPOp::SummaryAgg { .. }) => of_summary_agg(&as_built(op)), + // Back-to-back tumbling windows cover the whole window one build + // over it would. + Operator::ASAP( + op @ ASAPOp::SummaryTumblingWindowChain { + tumbling_window_count, + .. + }, + ) => of_summary_agg(&window_build(op, *tumbling_window_count).map_err(expansion)?), + // The newest tumbling windows of one chain cover what one build + // over their window does. + Operator::ASAP(ASAPOp::SummaryMerge { + children, + newest_tumbling_windows: Some(newest), group_by: None, - .. - }) => of_summary_agg( - &crate::ir::operator::tumbling_window_chain::merged_window_build(node).map_err( - |error| CoverageError::TumblingWindowChainExpansion { - reason: error.to_string(), - }, - )?, - ), - Some(ASAPOp::SummaryMerge { + }) => { + let [chain] = children.as_slice() else { + return Err(CoverageError::NotSummary); + }; + let Operator::ASAP(chain) = chain.operator() else { + return Err(CoverageError::NotSummary); + }; + of_summary_agg(&window_build(&chain, *newest).map_err(expansion)?) + } + Operator::ASAP(ASAPOp::SummaryMerge { children, group_by, .. }) => { let inputs = children .iter() - .map(|child| Self::derive_cached(child, cache)) + .map(|child| { + child + .summary_coverage() + .cloned() + .ok_or(CoverageError::NotSummary) + }) .collect::, _>>()?; let merged = of_merge(&inputs)?; match group_by { - None => merged, - Some(keys) => regrouped(merged, keys)?, + None => Ok(merged), + Some(keys) => regrouped(merged, &children[0], keys), } } - _ => return Err(CoverageError::NotSummary), - }; - cache.insert(key, coverage.clone()); - Ok(coverage) + Operator::ASAP(_) => Err(CoverageError::NotSummary), + } + } + + /// A stable identity of `definition`: the 128-bit FNV-1a hash, in hex, + /// of its input group and unlifted operators. Two equal definitions + /// (the equality a merge requires) of one memo get equal ids, so a + /// summary store can key states by this id and a concrete selection. + pub(crate) fn definition_id(&self) -> String { + const OFFSET: u128 = 0x6c62272e07bb014262b821756295c58d; + const PRIME: u128 = 0x0000000001000000000000000000013b; + let mut hash = OFFSET; + for byte in format!("{:?}", self.definition).bytes() { + hash ^= u128::from(byte); + hash = hash.wrapping_mul(PRIME); + } + format!("{hash:032x}") } } /// For a `Filter` that keeps whole states of a grouped `SummaryAgg` by /// their key columns, the build that computes exactly the kept states: the /// same build over its input rows filtered by the same predicate on the -/// input columns those keys name. `None` for any other node. -fn selected_build(node: &OperatorNode) -> Result>, CoverageError> { - let Some(states) = crate::ir::operator::asap::selected_from(node) else { +/// input columns those keys name. `None` for any other operator. +fn selected_build<'m>( + operator: &Operator>, +) -> Result>>, CoverageError> { + let Operator::NonASAP(NonASAPOp::Filter { + pred, + child: states, + }) = operator + else { return Ok(None); }; - let Some(NonASAPOp::Filter { pred, .. }) = node.non_asap() else { + if operator.output_kind() != OperatorResultKind::State { return Ok(None); - }; - let Some(ASAPOp::SummaryAgg { + } + let Operator::ASAP(ASAPOp::SummaryAgg { child, family, input, reduction: Reduction::Reduce(by), grouping, filter, - }) = states.asap() + }) = states.operator() else { return Err(CoverageError::NotSummary); }; @@ -381,7 +412,7 @@ fn selected_build(node: &OperatorNode) -> Result>, Cover let mut unmapped = None; let rows_pred = pred.0.map_columns(&mut |column| { let mapped = by.keys().get(column).copied().filter(|&input_column| { - matches!((states.schema.fields.get(column), child.schema.fields.get(input_column)), + matches!((states.schema().fields.get(column), child.schema().fields.get(input_column)), (Some(at_state), Some(at_input)) if at_state.name == at_input.name) }); mapped.unwrap_or_else(|| { @@ -392,41 +423,42 @@ fn selected_build(node: &OperatorNode) -> Result>, Cover if let Some(key) = unmapped { return Err(CoverageError::NotAGroupKey { key }); } - let rows = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Filter { - pred: crate::ir::scalar::Predicate(rows_pred), - child: Rc::clone(child), - })) - .map_err(|error| CoverageError::RegroupedBuild { + let mut built = |input: &MemoGroupAsInput<'m>| InputOrTemporaryBuild::Input(*input); + let regrouped_build = |error: SchemaDerivationError| CoverageError::RegroupedBuild { reason: error.to_string(), - })?; - OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryAgg { + }; + let rows = InputOrTemporaryBuild::build(NonASAPOp::Filter { + pred: Predicate(rows_pred.map_operator_refs(&mut built)), + child: InputOrTemporaryBuild::Input(child), + }) + .map_err(regrouped_build)?; + let reduction = Reduction::Reduce(by); + summary_agg_schema(rows.schema(), &family, &reduction).map_err(regrouped_build)?; + Ok(Some(ASAPOp::SummaryAgg { child: rows, - family: family.clone(), - input: input.clone(), - reduction: Reduction::Reduce(by.clone()), - grouping: grouping.clone(), - filter: filter.clone(), + family, + input, + reduction, + grouping, + filter: filter.map(|pred| Predicate(pred.0.map_operator_refs(&mut built))), })) - .map(Some) - .map_err(|error| CoverageError::RegroupedBuild { - reason: error.to_string(), - }) } /// The coverage of a merge of states covering `inputs`. fn of_merge(inputs: &[SummaryCoverage]) -> Result { let first = inputs.first().ok_or(CoverageError::EmptyMerge)?; - let mut proven = HashSet::new(); - if !inputs + if inputs .iter() - .all(|input| same_definition(&first.definition, &input.definition, &mut proven)) + .any(|input| input.definition != first.definition) { return Err(CoverageError::DefinitionMismatch); } // A family without a merge is checked as the strictest rule; the // executor rejects the merge itself. - let required = match first.definition.asap() { - Some(ASAPOp::SummaryAgg { family, .. }) => FamilyCapability::of(family).merge_inputs, + let required = match first.definition.unlifted_operators.first() { + Some(Operator::ASAP(ASAPOp::SummaryAgg { family, .. })) => { + FamilyCapability::of(family).merge_inputs + } _ => None, } .unwrap_or(MergeInputRule::Disjoint); @@ -452,26 +484,30 @@ fn of_merge(inputs: &[SummaryCoverage]) -> Result Result { - let Some(ASAPOp::SummaryAgg { - child, +/// The coverage of a regrouping merge, keyed by `keys`, of states whose +/// merge covers `merged` (`first` is its first input): the same rows +/// (`selection`), computed by the input's build grouped by the columns +/// `keys` name. The input states of one output group are different groups +/// of one build, so their rows are disjoint. A counter family merges only +/// time-disjoint windows, never different groups at one time. +fn regrouped( + merged: SummaryCoverage, + first: &MemoGroupAsInput<'_>, + keys: &[ColumnId], +) -> Result { + let Some(Operator::ASAP(ASAPOp::SummaryAgg { family, input, reduction, grouping, filter, - }) = merged.definition.asap() + .. + })) = merged.definition.unlifted_operators.first() else { return Err(CoverageError::NotSummary); }; @@ -480,7 +516,8 @@ fn regrouped(merged: SummaryCoverage, keys: &[ColumnId]) -> Result Result Result, _>>()?; - // Derived as a direct build's schema, so its definition id is the same. - let definition = OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryAgg { - child: Rc::clone(child), + let reduction = Reduction::by(mapped); + summary_agg_schema(&rows, family, &reduction).map_err(|error| { + CoverageError::RegroupedBuild { + reason: error.to_string(), + } + })?; + // A direct build's definition, so its definition id is the same. + let mut definition = merged.definition.clone(); + definition.unlifted_operators[0] = Operator::ASAP(ASAPOp::SummaryAgg { + child: None, family: family.clone(), input: input.clone(), - reduction: Reduction::by(mapped), + reduction, grouping: grouping.clone(), filter: filter.clone(), - })) - .map_err(|error| CoverageError::RegroupedBuild { - reason: error.to_string(), - })?; + }); Ok(SummaryCoverage { definition, selection: merged.selection, }) } +/// The schema of the rows the build of summary state `state` reads (its +/// `SummaryAgg`'s input; one build over a chain's window reads its range of +/// the chain's input, which has the input's schema). `None` when `state` +/// is no summary state. +fn build_input_schema(state: &I) -> Option { + if let Some(states) = selected_from(state) { + return build_input_schema(&states); + } + match state.operator() { + Operator::ASAP( + ASAPOp::SummaryAgg { child, .. } | ASAPOp::SummaryTumblingWindowChain { child, .. }, + ) => Some(child.schema().clone()), + Operator::ASAP(ASAPOp::SummaryMerge { children, .. }) => { + build_input_schema(children.first()?) + } + _ => None, + } +} + /// Whether the time windows of every box of `inputs` are pairwise apart: /// sorted by start, each ends before the next starts. `false` when a box /// has no time window or two windows overlap, boxes of one input included. @@ -548,216 +608,126 @@ fn windows_ordered_apart(inputs: &[SummaryCoverage]) -> bool { }) } -/// The hash [`SummaryCoverage::definition_id`] reports: each node's -/// operator without its children, output kind and schema (in their `Debug` -/// form, which every IR type derives and which is fixed for a given IR), -/// then its children's hashes in order. `cache` hashes a shared node once. -fn definition_hash( - node: &Rc, - cache: &mut HashMap<*const OperatorNode, u128>, -) -> u128 { - if let Some(hash) = cache.get(&Rc::as_ptr(node)) { - return *hash; - } - const OFFSET: u128 = 0x6c62272e07bb014262b821756295c58d; - const PRIME: u128 = 0x0000000001000000000000000000013b; - let mut hash = OFFSET; - let mut write = |bytes: &[u8]| { - for byte in bytes { - hash ^= u128::from(*byte); - hash = hash.wrapping_mul(PRIME); - } - }; - let own = format!( - "{:?}|{:?}|{:?}", - node.operator.map_children(|_| ()), - node.result_kind, - node.schema - ); - write(own.as_bytes()); - for child in node.children() { - let child_hash = definition_hash(child, cache); - write(&child_hash.to_le_bytes()); - } - cache.insert(Rc::as_ptr(node), hash); - hash -} - -/// Structural equality ignoring planning metadata (`timing`, `guarantee`). -/// `proven` memoizes node pairs already found equal. -fn same_definition( - a: &Rc, - b: &Rc, - proven: &mut HashSet<(*const OperatorNode, *const OperatorNode)>, -) -> bool { - let key = (Rc::as_ptr(a), Rc::as_ptr(b)); - if Rc::ptr_eq(a, b) || proven.contains(&key) { - return true; - } - let (ac, bc) = (a.children(), b.children()); - let equal = a.operator.map_children(|_| ()) == b.operator.map_children(|_| ()) - && a.result_kind == b.result_kind - && a.schema == b.schema - && ac.len() == bc.len() - && ac - .iter() - .zip(&bc) - .all(|(x, y)| same_definition(x, y, proven)); - if equal { - proven.insert(key); - } - equal -} - -fn of_summary_agg(node: &OperatorNode) -> SummaryCoverage { - let Some(ASAPOp::SummaryAgg { +/// The coverage of `build`, a `SummaryAgg` over memo groups (or over an +/// input a rule built over them): its operators walked down through the +/// liftable operators of its input groups' expressions. +fn of_summary_agg(build: &ASAPOp>) -> Result { + let ASAPOp::SummaryAgg { child, family, input, reduction, grouping, filter, - }) = node.asap() + } = build else { unreachable!("called on a SummaryAgg"); }; - - // The chain a predicate can be lifted through, top to bottom, and the - // first node below it. - let mut chain: Vec<&Rc> = Vec::new(); - let mut base = child; - while let Some(op) = base.non_asap() { - let passes = match op { - // A rename outside a `Project` would be dropped with the node - // when everything is lifted out of it, so such a node stays. - NonASAPOp::Project { .. } => true, - _ if base.operator.output_schema().ok().as_ref() != Some(&base.schema) => false, - NonASAPOp::Filter { .. } => true, - NonASAPOp::TimeRange { kind, .. } => *kind == TimeRangeKind::Range, - NonASAPOp::TimeShift { shift, .. } => shift.at.is_none(), + // The operators a predicate can be lifted through, top to bottom, and + // the first input below them. A subquery's group stays named in the + // residual of the operator that reads it. + let mut chain: Vec>> = Vec::new(); + let mut base = child.clone(); + loop { + let operator = base.operator(); + let passes = match &operator { + Operator::NonASAP(NonASAPOp::Project { .. } | NonASAPOp::Filter { .. }) => true, + Operator::NonASAP(NonASAPOp::TimeRange { kind, .. }) => *kind == TimeRangeKind::Range, + Operator::NonASAP(NonASAPOp::TimeShift { shift, .. }) => shift.at.is_none(), _ => false, }; if !passes { break; } - chain.push(base); - base = base.children()[0]; + // The main input is an operator's first child. + base = operator.children()[0].clone(); + chain.push(operator); } + let InputOrTemporaryBuild::Input(input_group) = &base else { + unreachable!("a rule builds only liftable operators over memo groups"); + }; // Time is lifted only from exactly one range window. let lift_time = chain .iter() - .filter(|link| matches!(link.non_asap(), Some(NonASAPOp::TimeRange { .. }))) + .filter(|link| matches!(link, Operator::NonASAP(NonASAPOp::TimeRange { .. }))) .count() == 1; - let top = &child.schema; + let top = child.schema(); // Top-down: decide which conjuncts are lifted. `kept[d]` is the residual // of `chain[d]` when it is a `Filter`. let mut lifted = BTreeMap::new(); let agg_filter = filter .as_ref() .and_then(|pred| residual(pred, top, &[], &mut lifted)); - let kept: Vec> = chain + let kept: Vec>>> = chain .iter() .enumerate() - .map(|(depth, link)| match link.non_asap() { - Some(NonASAPOp::Filter { pred, .. }) => { + .map(|(depth, link)| match link { + Operator::NonASAP(NonASAPOp::Filter { pred, .. }) => { residual(pred, top, &chain[..depth], &mut lifted) } _ => None, }) .collect(); - let scan_kept = match base.non_asap() { - Some(NonASAPOp::Scan { predicates, .. }) => Some( - predicates + // A `Scan`'s predicates (PromQL label matchers) are lifted too; when + // one is, the `Scan` with the rest of its predicates is the last + // unlifted operator, and there is no input group. + let scan = match base.operator() { + Operator::NonASAP(NonASAPOp::Scan { + source, + predicates, + schema, + }) => { + let lifted_above = lifted.clone(); + let kept = predicates .iter() .filter_map(|pred| residual(pred, top, &chain, &mut lifted)) - .collect::>(), - ), + .collect(); + (lifted != lifted_above).then_some(NonASAPOp::Scan { + source, + predicates: kept, + schema, + }) + } _ => None, }; - // Bottom-up: rebuild without what was lifted. A node whose input and - // operator are unchanged keeps its `Rc`. - let mut rebuilt = Rc::clone(base); - if let ( - Some(kept), - Some(NonASAPOp::Scan { - source, - predicates, - schema, - }), - ) = (scan_kept, base.non_asap()) - { - if kept != *predicates { - rebuilt = rebuild( - base, - NonASAPOp::Scan { - source: source.clone(), - predicates: kept, - schema: schema.clone(), - }, - ); - } - } + let mut unlifted_operators = vec![Operator::ASAP(ASAPOp::SummaryAgg { + child: None, + family: family.clone(), + input: input.clone(), + reduction: reduction.clone(), + grouping: grouping.clone(), + filter: agg_filter, + })]; let (mut window_ms, mut shift_ms) = (0i64, 0i64); - for (depth, link) in chain.iter().enumerate().rev() { - let child = Rc::clone(&rebuilt); - let op = match link.non_asap().expect("chain nodes are non-ASAP") { - NonASAPOp::Filter { pred, .. } => match &kept[depth] { - None => continue, - Some(rest) if rest == pred && Rc::ptr_eq(&child, link.children()[0]) => { - rebuilt = Rc::clone(link); - continue; + for (depth, link) in chain.iter().enumerate() { + match link { + Operator::NonASAP(NonASAPOp::Filter { .. }) => { + if let Some(rest) = kept[depth].clone() { + unlifted_operators.push(Operator::NonASAP(NonASAPOp::Filter { + pred: rest, + child: None, + })); } - Some(rest) => NonASAPOp::Filter { - pred: rest.clone(), - child, - }, - }, - NonASAPOp::TimeRange { range, .. } if lift_time => { + } + Operator::NonASAP(NonASAPOp::TimeRange { range, .. }) if lift_time => { window_ms = range.as_millis() as i64; - continue; } - NonASAPOp::TimeShift { shift, .. } if lift_time => { + Operator::NonASAP(NonASAPOp::TimeShift { shift, .. }) if lift_time => { shift_ms += shift.offset_ms; - continue; } - _ if Rc::ptr_eq(&child, link.children()[0]) => { - rebuilt = Rc::clone(link); - continue; - } - NonASAPOp::TimeRange { range, kind, .. } => NonASAPOp::TimeRange { - range: *range, - kind: *kind, - child, - }, - NonASAPOp::TimeShift { shift, .. } => NonASAPOp::TimeShift { - shift: *shift, - child, - }, - NonASAPOp::Project { - cols, qualifier, .. - } => NonASAPOp::Project { - cols: cols.clone(), - qualifier: qualifier.clone(), - child, - }, - _ => unreachable!("only liftable operators are on the chain"), - }; - rebuilt = rebuild(link, op); + other => unlifted_operators.push(unlifted(other)), + } } - let definition = rebuild_asap( - node, - ASAPOp::SummaryAgg { - child: rebuilt, - family: family.clone(), - input: input.clone(), - reduction: reduction.clone(), - grouping: grouping.clone(), - filter: agg_filter, - }, - ); + let input_group = match scan { + Some(scan) => { + unlifted_operators.push(Operator::NonASAP(scan)); + None + } + None => Some(input_group.group_id), + }; let selection = SelectionBox { columns: lifted @@ -776,40 +746,28 @@ fn of_summary_agg(node: &OperatorNode) -> SummaryCoverage { Bound::Included(-shift_ms), )), }; - SummaryCoverage { - definition, + Ok(SummaryCoverage { + definition: SummaryStateDefinition { + input_group, + unlifted_operators, + }, selection: vec![selection], - } -} - -/// A node with the same output schema over a new operator. The definition -/// describes what was computed; it is not re-validated as a plan. -fn rebuild(original: &OperatorNode, op: NonASAPOp) -> Rc { - Rc::new(OperatorNode::with_schema( - Operator::NonASAP(op), - original.schema.clone(), - )) -} - -fn rebuild_asap(original: &OperatorNode, op: ASAPOp) -> Rc { - Rc::new(OperatorNode::with_schema( - Operator::ASAP(op), - original.schema.clone(), - )) + }) } /// Lift what `pred`'s conjuncts can into `lifted` (keyed by the column of -/// `top`, the agg child's schema) and return the rest. `above` are the chain -/// nodes between the predicate and the `SummaryAgg`, top to bottom. A column -/// whose `(table, name)` is not unique in `top` cannot be named in a -/// selection, so its conjuncts stay. +/// `top`, the agg child's schema) and return the rest, each scalar subquery +/// naming the group it reads. `above` +/// are the operators between the predicate and the `SummaryAgg`, top to +/// bottom. A column whose `(table, name)` is not unique in `top` cannot be +/// named in a selection, so its conjuncts stay. fn residual( - pred: &Predicate, + pred: &Predicate>, top: &Schema, - above: &[&Rc], + above: &[Operator>], lifted: &mut BTreeMap, -) -> Option { - let mut rest: Vec = Vec::new(); +) -> Option>> { + let mut rest: Vec>> = Vec::new(); for conjunct in pred.0.conjuncts() { let lift = constraint_of(conjunct).and_then(|(column, constraint)| { let column = column_at_top(column, above)?; @@ -832,7 +790,7 @@ fn residual( Some((column, constraint)) => { lifted.insert(column, constraint); } - None => rest.push(conjunct.clone()), + None => rest.push(conjunct.map_operator_refs(&mut group_id)), } } match rest.len() { @@ -842,14 +800,38 @@ fn residual( } } +/// The memo group `input` is; `None` for a rule's temporary build. +fn group_id(input: &GroupInput<'_>) -> Option { + match input { + InputOrTemporaryBuild::Input(group) => Some(group.group_id), + InputOrTemporaryBuild::TemporaryBuild(_) => None, + } +} + +/// `link`, a liftable operator kept whole in a definition: its main input +/// `None`, every other input the group it is. +fn unlifted(link: &Operator>) -> Operator> { + let mut operator = link.map_children(group_id); + match &mut operator { + Operator::NonASAP( + NonASAPOp::Filter { child, .. } + | NonASAPOp::Project { child, .. } + | NonASAPOp::TimeRange { child, .. } + | NonASAPOp::TimeShift { child, .. }, + ) => *child = None, + _ => unreachable!("only liftable operators are walked"), + } + operator +} + /// Map a column of the input of `above.last()` up to the agg child's output: /// a `Project` passes it only as a direct column item. -fn column_at_top(mut column: ColumnId, above: &[&Rc]) -> Option { +fn column_at_top(mut column: ColumnId, above: &[Operator]) -> Option { for link in above.iter().rev() { - if let Some(NonASAPOp::Project { cols, .. }) = link.non_asap() { + if let Operator::NonASAP(NonASAPOp::Project { cols, .. }) = link { column = cols .iter() - .position(|item| item.expr == ScalarExpr::Column(column))?; + .position(|item| matches!(item.expr, ScalarExpr::Column(c) if c == column))?; } } Some(column) @@ -857,8 +839,8 @@ fn column_at_top(mut column: ColumnId, above: &[&Rc]) -> Option`, `>=`, `[NOT] IN (...)` and `OR` of /// equalities on one column, with non-null literals. -fn constraint_of(expr: &ScalarExpr) -> Option<(ColumnId, Constraint)> { - let literal = |e: &ScalarExpr| match e { +fn constraint_of(expr: &ScalarExpr) -> Option<(ColumnId, Constraint)> { + let literal = |e: &ScalarExpr| match e { ScalarExpr::Literal(v) if *v != ScalarValue::Null => Some(v.clone()), _ => None, }; diff --git a/crates/types/src/ir/properties/timing.rs b/crates/types/src/ir/properties/timing.rs deleted file mode 100644 index 07684c9fc..000000000 --- a/crates/types/src/ir/properties/timing.rs +++ /dev/null @@ -1,1640 +0,0 @@ -//! Execution timing: written into every node from a materialization -//! assignment, then validated against each operator's kind and its consuming -//! edges. -//! -//! The logical DAG carries no timing. Materialization (#509 -//! §Materialization) gives each sub-DAG one of its options: -//! `NotMaterialized`, `MatAtIngestion(medium)`, `MatAtQuery(medium)` or -//! `MatAtQueryKeptAcrossEvaluations(medium)`. -//! [`MaterializationAssignment`] records that option per sub-DAG, named by -//! the node whose output would be stored (any node: a summary state, a -//! population, or e.g. the tumbling windows of #509 Example 4), and -//! [`apply_materialization_timings`] expands it into a timing on every node: -//! -//! - `SummaryEstimate` and `EvaluatePopulation` run at query time; -//! - a node assigned `MatAtIngestion` runs at ingestion time, and one -//! assigned `MatAtQuery` or `MatAtQueryKeptAcrossEvaluations` at query -//! time. A `SummaryAgg` or -//! `MaintainPopulation` without an assignment is `NotMaterialized` and runs -//! at query time. Every node upstream of an ingestion-time node also runs -//! at ingestion time, so `MatAtIngestion` (or an ingestion timing fixed by -//! a candidate builder) over something that can only exist at query time -//! is an error ([`ExecutionDataStateError::UpstreamNotAtIngestion`]); -//! - a PromQL `BinaryOp` over per-series inputs that are not the same series -//! rows runs at query time, since maintenance cannot align them; -//! - every other node is `NotMaterialized` and runs when its consumer runs: -//! everything that feeds an ingestion-time node runs at ingestion time, -//! everything above an evaluation at query time. -//! -//! A node whose timing a candidate builder fixed (e.g. the query-time -//! `FinalizeExactAccumulator` of `finalize_query_candidate`) keeps that -//! timing. An explicit assignment on it that agrees is kept; one that needs -//! the other timing is an error -//! ([`ExecutionDataStateError::MaterializationConflictsWithFixedTiming`]). -//! A root may be `MatAtIngestion`: the query reads its stored output. A root -//! of rows that runs at ingestion time without being stored there is an -//! error ([`ExecutionDataStateError::UnstoredIngestionRoot`]). -//! -//! A node reached from two consumers that need different timings cannot be -//! executed once for both; Stage 2's `split_shared_by_timing` copies such a -//! sub-DAG for one side (by [`timing_under_consumer`]) before the assignment -//! is applied, and the pass itself -//! rejects a conflict it still finds. -//! -//! ## Edge rules (checked after the write) -//! -//! | Consumer | Accepts from an input | -//! |---|---| -//! | `SummaryAgg.child`, `SummaryTumblingWindowChain.child` | Rows, or exact-accumulator state, never a query-time value when the state is maintained; a query-time build reads an ingestion-time input (rows or exact-accumulator state) only as a `MatAtIngestion` input's stored output | -//! | `SummaryEstimate.summary_input` | Summary state at either timing | -//! | `SummaryMerge.children` | Summary state at either timing; at ingestion time only for a merge at ingestion time | -//! | `FinalizeExactAccumulator.child` | Exact-accumulator state | -//! | `EvaluatePopulation.child` | A timed `MaintainPopulation` | -//! | `MaintainPopulation.child` | Rows at its own timing matching the population's input; unlike a `NonASAP` consumer, a query-time population does not read a `MatAtIngestion` input's stored output | -//! | any `NonASAP` consumer | Rows (or exact-accumulator state for a projection-like operator) at the consumer's own timing, or a `MatAtIngestion` output read at query time; ingestion work never reads a query-time value | - -use std::collections::HashMap; -use std::rc::Rc; - -use crate::ir::operator::asap::ASAPOp; -use crate::ir::operator::node::{Operator, OperatorNode}; -use crate::ir::operator::non_asap::NonASAPOp; -use crate::ir::operator::operator_properties::BinaryOpKind; -use crate::ir::operator::query::QueryRoot; -use crate::ir::properties::execution_data_state::{ - DataPrimitive, ExecutionDataState, ExecutionDataStateError, ExecutionTiming, -}; -use crate::ir::schema::{DataType, FieldDataType, Schema}; -use crate::physical::materialization::Materialization; -#[cfg(any(test, feature = "test-support"))] -use crate::physical::materialization::Medium; - -/// The materialization option of each sub-DAG (#509 §Materialization: -/// "Materialization decides, for each sub-DAG, whether its output is -/// stored"). A sub-DAG is named by the node whose output would be stored (by -/// identity); any node may be assigned. A node without an assignment is not -/// stored: a `SummaryAgg` or `MaintainPopulation` takes the default, every -/// other node runs when its consumer runs. `Default` assigns -/// `NotMaterialized` to every sub-DAG. -/// -/// The assignment keeps every assigned node alive, so a later allocation -/// cannot reuse its address and match a stale entry. -#[derive(Clone, Default)] -pub struct MaterializationAssignment { - assigned: HashMap<*const OperatorNode, (Rc, Materialization)>, - unassigned: Materialization, -} - -/// Each assigned node as its operator kind and address, not its whole -/// sub-DAG. -impl std::fmt::Debug for MaterializationAssignment { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let assigned: Vec<_> = self - .assigned - .iter() - .map(|(ptr, (node, materialization))| { - ( - format!("{}@{ptr:p}", node.operator.kind_name()), - materialization, - ) - }) - .collect(); - f.debug_struct("MaterializationAssignment") - .field("assigned", &assigned) - .field("unassigned", &self.unassigned) - .finish() - } -} - -impl MaterializationAssignment { - /// Every `SummaryAgg` or `MaintainPopulation` not [`set`](Self::set) - /// explicitly takes `materialization` where the DAG allows it: a - /// `MatAtIngestion` default over a sub-DAG that can only run at query - /// time leaves that sub-DAG `NotMaterialized`. - pub fn with_default(materialization: Materialization) -> Self { - Self { - assigned: HashMap::new(), - unassigned: materialization, - } - } - - /// Assign `materialization` to the sub-DAG whose output `node` produces. - pub fn set(&mut self, node: &Rc, materialization: Materialization) { - self.assigned - .insert(Rc::as_ptr(node), (Rc::clone(node), materialization)); - } - - /// The explicit assignment of `node`, if any. - pub fn get(&self, node: &OperatorNode) -> Option { - self.assigned - .get(&(node as *const OperatorNode)) - .map(|(_, m)| *m) - } - - /// The option `node` gets when `upstream_at_query_time` says whether - /// something below it can only run at query time. - fn resolve( - &self, - node: &OperatorNode, - upstream_at_query_time: bool, - ) -> Result { - let at_ingestion = |m: Materialization| m.timing() == ExecutionTiming::IngestionTime; - match self.get(node) { - Some(m) if at_ingestion(m) && is_filtered_build(node) => { - Err(ExecutionDataStateError::FilteredSummaryAtIngestion { - operator: node.operator.kind_name(), - }) - } - Some(m) if at_ingestion(m) && upstream_at_query_time => { - Err(ExecutionDataStateError::UpstreamNotAtIngestion { - operator: node.operator.kind_name(), - }) - } - Some(m) => Ok(m), - None if at_ingestion(self.unassigned) - && (upstream_at_query_time || is_filtered_build(node)) => - { - Ok(Materialization::NotMaterialized) - } - None => Ok(self.unassigned), - } - } -} - -/// Whether `operator`'s output is a sub-DAG output that materialization may -/// store: a `SummaryAgg`, a `SummaryTumblingWindowChain` or a `MaintainPopulation`. -pub(crate) fn is_materialization_point(operator: &Operator) -> bool { - matches!( - operator, - Operator::ASAP( - ASAPOp::SummaryAgg { .. } - | ASAPOp::SummaryTumblingWindowChain { .. } - | ASAPOp::MaintainPopulation { .. } - ) - ) -} - -/// Whether `node` is a summary build (`SummaryAgg` or `SummaryTumblingWindowChain`) with a -/// filter, which never runs at ingestion time -/// ([`ExecutionDataStateError::FilteredSummaryAtIngestion`]). -pub fn is_filtered_build(node: &OperatorNode) -> bool { - matches!( - node.asap(), - Some( - ASAPOp::SummaryAgg { - filter: Some(_), - .. - } | ASAPOp::SummaryTumblingWindowChain { - filter: Some(_), - .. - } - ) - ) -} - -/// Cache of one [`apply_materialization_timings`] pass: `input node → timed node`, -/// shared by every root of a workload so a node shared by two roots stays -/// one `Rc`. Re-reaching a node with a different timing is a conflict. -#[derive(Default)] -pub struct NodeToTimedNodeCache { - timed_node_of: HashMap<*const OperatorNode, Rc>, -} - -impl NodeToTimedNodeCache { - pub fn new() -> Self { - Self::default() - } - - /// The timed node produced for `input`, if the pass has reached it. - pub(crate) fn timed(&self, input: &Rc) -> Option<&Rc> { - self.timed_node_of.get(&Rc::as_ptr(input)) - } -} - -/// What `node`'s output carries: summary state from a build, a merge or a -/// node that keeps whole states (a key filter, a projection of the state -/// column), raw rows otherwise. -fn primitive(node: &OperatorNode) -> DataPrimitive { - match &node.operator { - Operator::ASAP(op) if op.produced_state().is_some() => DataPrimitive::SummaryState, - Operator::ASAP(ASAPOp::MaintainPopulation { .. }) => DataPrimitive::SummaryState, - _ if node.result_kind == crate::ir::OperatorResultKind::State => { - DataPrimitive::SummaryState - } - _ => DataPrimitive::Raw, - } -} - -/// The data state a timed node's output carries. -pub fn data_state(node: &OperatorNode) -> Option { - Some(ExecutionDataState { - timing: node.timing?, - primitive: primitive(node), - }) -} - -/// Whether the sub-DAG below `node` contains a node that can only run at -/// query time (an evaluation, or a sub-DAG `assignment` does not put at -/// ingestion time), which forces every consumer above it to query time as well. -/// A PromQL `BinaryOp` whose inputs are not the same per-series rows also runs -/// at query time. -fn forces_query_time( - node: &OperatorNode, - assignment: &MaterializationAssignment, - seen: &mut HashMap<*const OperatorNode, bool>, -) -> bool { - let key = node as *const OperatorNode; - if let Some(&cached) = seen.get(&key) { - return cached; - } - let upstream = |seen: &mut HashMap<*const OperatorNode, bool>| { - node.children() - .iter() - .any(|child| forces_query_time(child, assignment, seen)) - }; - let forced = match &node.operator { - _ if node.timing == Some(ExecutionTiming::QueryTime) => true, - Operator::ASAP(ASAPOp::SummaryEstimate { .. }) - | Operator::ASAP(ASAPOp::EvaluatePopulation { .. }) => true, - _ if node.timing.is_none() - && (is_materialization_point(&node.operator) || assignment.get(node).is_some()) => - { - let upstream = upstream(seen); - match assignment.resolve(node, upstream) { - Ok(Materialization::NotMaterialized) - if !is_materialization_point(&node.operator) => - { - upstream - } - Ok(m) => m.timing() == ExecutionTiming::QueryTime, - Err(_) => true, - } - } - Operator::NonASAP(NonASAPOp::BinaryOp { lhs, rhs, .. }) - if node - .schema - .fields - .iter() - .any(|f| f.name == crate::ir::schema::PROMQL_SERIES_IDENTITY) - && per_series_rows(lhs).is_none_or(|rows| per_series_rows(rhs) != Some(rows)) => - { - true - } - _ => upstream(seen), - }; - seen.insert(key, forced); - forced -} - -/// Write the timings of `assignment` into every node reachable from `root`, -/// top-down, then validate every edge. Returns the timed copy of `root`; -/// `cache` carries the sharing across the roots of one workload. -pub fn apply_materialization_timings( - root: &Rc, - assignment: &MaterializationAssignment, - cache: &mut NodeToTimedNodeCache, -) -> Result, ExecutionDataStateError> { - let mut forced = HashMap::new(); - let timed = write( - root, - ExecutionTiming::QueryTime, - assignment, - cache, - &mut forced, - )?; - // A root stored at ingestion time is read by the query; one that runs at - // ingestion time without being stored there produces rows nothing reads. - if timed.timing == Some(ExecutionTiming::IngestionTime) - && !matches!(timed.materialization, Materialization::MatAtIngestion(_)) - && data_state(&timed).map(|s| s.primitive) == Some(DataPrimitive::Raw) - { - return Err(ExecutionDataStateError::UnstoredIngestionRoot { - operator: timed.operator.kind_name(), - }); - } - validate(&timed, &mut HashMap::new())?; - Ok(timed) -} - -/// [`apply_materialization_timings`] over every root of one workload with -/// one cache, so a node several roots share is timed once and stays one -/// node. A standalone scalar root has each operator it reads timed, and is -/// rebuilt over the timed nodes. -pub fn apply_materialization_timings_to_roots( - roots: &[QueryRoot], - assignment: &MaterializationAssignment, -) -> Result, ExecutionDataStateError> { - let mut cache = NodeToTimedNodeCache::new(); - roots - .iter() - .map(|root| { - Ok(match root { - QueryRoot::Operator(node) => QueryRoot::Operator(apply_materialization_timings( - node, assignment, &mut cache, - )?), - QueryRoot::Scalar(expr) => { - for node in expr.operator_refs() { - apply_materialization_timings(node, assignment, &mut cache)?; - } - QueryRoot::Scalar(expr.map_operator_refs(&mut |node| { - Rc::clone(cache.timed(node).expect("scalar reference was timed")) - })) - } - }) - }) - .collect() -} - -/// Validate the sub-DAG below `root` with every sub-DAG `MatAtIngestion` -/// where the DAG allows it ([`MaterializationAssignment::with_default`]) and -/// `root` consumed at `root_timing`. For planning-time legality checks of a -/// candidate before it is assembled into a workload DAG: a candidate must stay -/// executable if materialization later puts its states at ingestion time. -/// Nothing is kept. -#[cfg(any(test, feature = "test-support"))] -pub fn validate_at_ingestion( - root: &Rc, - root_timing: ExecutionTiming, -) -> Result<(), ExecutionDataStateError> { - let assignment = all_at_ingestion(); - let mut cache = NodeToTimedNodeCache::new(); - let mut forced = HashMap::new(); - let timed = write(root, root_timing, &assignment, &mut cache, &mut forced)?; - validate(&timed, &mut HashMap::new()) -} - -/// The data state `node` produces with every sub-DAG `MatAtIngestion` where -/// the DAG allows it, when its consumer runs at `consumer` — the planning-time answer to -/// "what does this candidate's output look like", consistent with -/// [`validate_at_ingestion`]. -#[cfg(any(test, feature = "test-support"))] -pub fn planned_data_state( - node: &Rc, - consumer: ExecutionTiming, -) -> ExecutionDataState { - let mut forced = HashMap::new(); - let (timing, _) = placement(node, consumer, &all_at_ingestion(), &mut forced) - .expect("a default assignment never fails"); - ExecutionDataState { - timing, - primitive: primitive(node), - } -} - -/// The assignment of [`validate_at_ingestion`] and [`planned_data_state`]. -#[cfg(any(test, feature = "test-support"))] -fn all_at_ingestion() -> MaterializationAssignment { - MaterializationAssignment::with_default(Materialization::MatAtIngestion(Medium::Memory)) -} - -/// The timing and materialization `node` takes when its consumer runs at -/// `consumer`. -fn placement( - node: &Rc, - consumer: ExecutionTiming, - assignment: &MaterializationAssignment, - forced: &mut HashMap<*const OperatorNode, bool>, -) -> Result<(ExecutionTiming, Materialization), ExecutionDataStateError> { - // A placement fixed when the candidate was built (an exact-state read - // boundary that must run at query time, or one that feeds maintenance) - // is honored; a conflicting consumer is rejected by validation. - let upstream_at_query_time = |forced: &mut HashMap<*const OperatorNode, bool>| { - node.children() - .iter() - .any(|child| forces_query_time(child, assignment, forced)) - }; - if let Some(placed) = node.timing { - if placed == ExecutionTiming::IngestionTime && is_filtered_build(node) { - return Err(ExecutionDataStateError::FilteredSummaryAtIngestion { - operator: node.operator.kind_name(), - }); - } - if placed == ExecutionTiming::IngestionTime && upstream_at_query_time(forced) { - return Err(ExecutionDataStateError::UpstreamNotAtIngestion { - operator: node.operator.kind_name(), - }); - } - // An explicit assignment is never silently dropped: one that agrees - // with the fixed timing is kept, one that needs the other timing is - // an error. - return match assignment.get(node) { - Some(m) if m != Materialization::NotMaterialized && m.timing() != placed => Err( - ExecutionDataStateError::MaterializationConflictsWithFixedTiming { - operator: node.operator.kind_name(), - fixed: placed, - materialization: m, - }, - ), - Some(m) => Ok((placed, m)), - None => Ok((placed, node.materialization)), - }; - } - Ok(match &node.operator { - Operator::ASAP(ASAPOp::SummaryEstimate { .. }) - | Operator::ASAP(ASAPOp::EvaluatePopulation { .. }) => match assignment.get(node) { - Some(m) if m.timing() == ExecutionTiming::IngestionTime => { - return Err(ExecutionDataStateError::UpstreamNotAtIngestion { - operator: node.operator.kind_name(), - }) - } - Some(m) => (ExecutionTiming::QueryTime, m), - None => (ExecutionTiming::QueryTime, Materialization::NotMaterialized), - }, - _ if is_materialization_point(&node.operator) || assignment.get(node).is_some() => { - let upstream = upstream_at_query_time(forced); - match assignment.resolve(node, upstream)? { - Materialization::NotMaterialized if !is_materialization_point(&node.operator) => { - (consumer, Materialization::NotMaterialized) - } - materialization => (materialization.timing(), materialization), - } - } - _ => (consumer, Materialization::NotMaterialized), - }) -} - -/// The timing `node` takes when its consumer runs at `consumer`, by the -/// placement rules [`apply_materialization_timings`] applies, for Stage 2's -/// `split_shared_by_timing`: an assignment that cannot be honored is -/// reported later by [`apply_materialization_timings`]. -/// `query_time_forced` caches, per node, whether something at or below it -/// can only run at query time: pass an empty map and reuse it across the -/// calls of one assignment. -pub fn timing_under_consumer( - node: &Rc, - consumer: ExecutionTiming, - assignment: &MaterializationAssignment, - query_time_forced: &mut HashMap<*const OperatorNode, bool>, -) -> ExecutionTiming { - placement(node, consumer, assignment, query_time_forced) - .map_or(ExecutionTiming::QueryTime, |(timing, _)| timing) -} - -fn write( - node: &Rc, - consumer: ExecutionTiming, - assignment: &MaterializationAssignment, - cache: &mut NodeToTimedNodeCache, - forced: &mut HashMap<*const OperatorNode, bool>, -) -> Result, ExecutionDataStateError> { - let (timing, materialization) = placement(node, consumer, assignment, forced)?; - if let Some(done) = cache.timed_node_of.get(&Rc::as_ptr(node)) { - let previous = done.timing.expect("memoized node is timed"); - if previous != timing { - return Err(ExecutionDataStateError::ConflictingTiming { - first: ExecutionDataState { - timing: previous, - primitive: data_state(done).map_or(DataPrimitive::Raw, |s| s.primitive), - }, - second: ExecutionDataState { - timing, - primitive: data_state(done).map_or(DataPrimitive::Raw, |s| s.primitive), - }, - }); - } - return Ok(Rc::clone(done)); - } - let mut error = None; - let operator = - node.operator.map_children( - |child| match write(child, timing, assignment, cache, forced) { - Ok(timed) => timed, - Err(e) => { - error.get_or_insert(e); - Rc::clone(child) - } - }, - ); - if let Some(e) = error { - return Err(e); - } - let timed = Rc::new( - OperatorNode::with_schema(operator, node.schema.clone()) - .with_guarantee(node.guarantee.clone()) - .with_timing(Some(timing)) - .with_materialization(materialization), - ); - cache - .timed_node_of - .insert(Rc::as_ptr(node), Rc::clone(&timed)); - Ok(timed) -} - -fn state_of(node: &OperatorNode) -> ExecutionDataState { - data_state(node).expect("timed node") -} - -/// Check every edge below `node` against the module-level rules. -fn validate( - node: &Rc, - seen: &mut HashMap<*const OperatorNode, ()>, -) -> Result<(), ExecutionDataStateError> { - if seen.insert(Rc::as_ptr(node), ()).is_some() { - return Ok(()); - } - let timing = node.timing.expect("timed node"); - match &node.operator { - Operator::ASAP(op) => validate_asap(node, op, timing)?, - Operator::NonASAP(op) => validate_non_asap(node, op, timing)?, - } - for child in node.children() { - validate(child, seen)?; - } - Ok(()) -} - -fn is_exact_accumulator_state(schema: &Schema) -> Result<(), ExecutionDataStateError> { - for field in &schema.fields { - match &field.dtype { - FieldDataType::Plain(_) | FieldDataType::ExactAggregate(..) => {} - other => { - return Err(ExecutionDataStateError::UnsupportedStateComposition { - family: format!("{other:?}"), - }) - } - } - } - Ok(()) -} - -fn validate_asap( - node: &OperatorNode, - op: &ASAPOp, - timing: ExecutionTiming, -) -> Result<(), ExecutionDataStateError> { - match op { - ASAPOp::SummaryAgg { child, .. } | ASAPOp::SummaryTumblingWindowChain { child, .. } => { - let edge = match op { - ASAPOp::SummaryTumblingWindowChain { .. } => "SummaryTumblingWindowChain.child", - _ => "SummaryAgg.child", - }; - let avail = state_of(child); - match avail { - ExecutionDataState::INGESTION_ROWS | ExecutionDataState::QUERY_ROWS => {} - s if s.primitive == DataPrimitive::SummaryState => { - is_exact_accumulator_state(&child.schema)? - } - other => { - return Err( - ExecutionDataStateError::QueryTimeValueUnderIngestionSummary { - edge, - child: other, - }, - ) - } - } - // A query-time build reads an ingestion-time input (rows or - // exact-accumulator state) only from where it is stored, as a - // `NonASAP` consumer does. - if timing == ExecutionTiming::QueryTime - && avail.timing == ExecutionTiming::IngestionTime - && child.materialization.timing() != ExecutionTiming::IngestionTime - { - return Err( - ExecutionDataStateError::UnstoredIngestionInputUnderQueryTimeSummary { - edge, - operator: child.operator.kind_name(), - }, - ); - } - if timing == ExecutionTiming::IngestionTime - && avail.timing == ExecutionTiming::QueryTime - { - return Err( - ExecutionDataStateError::QueryTimeValueUnderIngestionSummary { - edge, - child: avail, - }, - ); - } - Ok(()) - } - ASAPOp::SummaryEstimate { summary_input, .. } => { - let s = state_of(summary_input); - if s.primitive != DataPrimitive::SummaryState { - return Err(ExecutionDataStateError::IllegalChildDataState { - edge: "SummaryEstimate.summary_input", - child: s, - }); - } - if timing != ExecutionTiming::QueryTime { - return Err(ExecutionDataStateError::IllegalChildDataState { - edge: "SummaryEstimate", - child: state_of(node), - }); - } - Ok(()) - } - ASAPOp::FinalizeExactAccumulator { child } => { - let s = state_of(child); - if s.primitive != DataPrimitive::SummaryState - || is_exact_accumulator_state(&child.schema).is_err() - || (timing == ExecutionTiming::IngestionTime && s.timing != timing) - { - return Err(ExecutionDataStateError::IllegalChildDataState { - edge: "FinalizeExactAccumulator.child", - child: s, - }); - } - Ok(()) - } - ASAPOp::MaintainPopulation { child, population } => { - let valid = population.matches_node(child) - && state_of(child) - == ExecutionDataState { - timing, - primitive: DataPrimitive::Raw, - }; - if !valid { - return Err(ExecutionDataStateError::InvalidMaintainedPopulation); - } - Ok(()) - } - ASAPOp::EvaluatePopulation { child, evaluation } => { - let valid = timing == ExecutionTiming::QueryTime - && matches!( - &child.operator, - Operator::ASAP(ASAPOp::MaintainPopulation { population, .. }) - if population.supports(evaluation) - && child.timing.is_some() - ); - if !valid { - return Err(ExecutionDataStateError::InvalidMaintainedPopulation); - } - Ok(()) - } - ASAPOp::SummaryMerge { children, .. } => { - for child in children { - let s = state_of(child); - if s.primitive != DataPrimitive::SummaryState - || (timing == ExecutionTiming::IngestionTime && s.timing != timing) - { - return Err(ExecutionDataStateError::IllegalChildDataState { - edge: "SummaryMerge.children", - child: s, - }); - } - } - Ok(()) - } - ASAPOp::SummarySubtract { .. } - | ASAPOp::SummaryDelete { .. } - | ASAPOp::SummaryJoin { .. } - | ASAPOp::Extension { .. } => Err(ExecutionDataStateError::UnimplementedOperator { - operator: op.kind_name(), - }), - } -} - -fn check_plain_or_exact_values(input: &Schema) -> Result<(), ExecutionDataStateError> { - for field in &input.fields { - if !matches!( - field.dtype, - FieldDataType::Plain(_) | FieldDataType::ExactAggregate(..) - ) { - return Err(ExecutionDataStateError::NonPlainOperand { - column: field.name.clone(), - dtype: format!("{:?}", field.dtype), - }); - } - } - Ok(()) -} - -fn check_all_plain(input: &Schema) -> Result<(), ExecutionDataStateError> { - for field in &input.fields { - if !field.is_plain() { - return Err(ExecutionDataStateError::NonPlainOperand { - column: field.name.clone(), - dtype: format!("{:?}", field.dtype), - }); - } - } - Ok(()) -} - -/// Whether `node` is a filter that keeps whole summary states by their key -/// columns ([`selected_from`](crate::ir::operator::asap::selected_from)). -fn selects_states(node: &OperatorNode) -> bool { - crate::ir::operator::asap::selected_from(node).is_some() -} - -fn validate_non_asap( - node: &OperatorNode, - op: &NonASAPOp, - timing: ExecutionTiming, -) -> Result<(), ExecutionDataStateError> { - // Every input is rows at this node's own timing. Ingestion work never - // reads a query-time value; exact-accumulator state may pass through - // the projection-like operators unchanged. - for child in op.children() { - let s = state_of(child); - let passes_state = matches!( - op, - NonASAPOp::Project { .. } - | NonASAPOp::Filter { .. } - | NonASAPOp::Sort { .. } - | NonASAPOp::Limit { .. } - ) && s.primitive == DataPrimitive::SummaryState - && (is_exact_accumulator_state(&child.schema).is_ok() || selects_states(node)); - // A query-time consumer may read the stored output of a sub-DAG - // materialized at ingestion time. - let reads_stored = timing == ExecutionTiming::QueryTime - && child.materialization.timing() == ExecutionTiming::IngestionTime; - if (s.timing != timing && !reads_stored) - || (s.primitive != DataPrimitive::Raw && !passes_state) - { - return Err(ExecutionDataStateError::IllegalChildDataState { - edge: op.kind_name(), - child: s, - }); - } - } - match op { - // A key filter keeps whole summary states of any family. - NonASAPOp::Filter { .. } if selects_states(node) => {} - NonASAPOp::Project { child, .. } - | NonASAPOp::Filter { child, .. } - | NonASAPOp::Sort { child, .. } - | NonASAPOp::Limit { child, .. } => check_plain_or_exact_values(&child.schema)?, - NonASAPOp::Aggregate { - reduction, - measures, - child, - .. - } => { - let mut referenced: Vec = reduction - .group_keys() - .map(|keys| keys.keys().to_vec()) - .unwrap_or_default(); - for m in measures { - referenced.extend(m.input_cols()); - } - let implicit = measures.iter().any(|m| m.input_cols().is_empty()); - for (i, field) in child.schema.fields.iter().enumerate() { - if (implicit || referenced.contains(&i)) && !field.is_plain() { - return Err(ExecutionDataStateError::NonPlainOperand { - column: field.name.clone(), - dtype: format!("{:?}", field.dtype), - }); - } - } - } - NonASAPOp::BinaryOp { - operator, lhs, rhs, .. - } => { - let is_div = matches!( - operator.kind, - BinaryOpKind::Arithmetic(crate::ir::scalar::ArithmeticOpKind::Div) - ); - if (operator.checked_relative_division && operator.checked_finite_division) - || ((operator.checked_relative_division || operator.checked_finite_division) - && (timing != ExecutionTiming::QueryTime || !is_div)) - { - return Err(ExecutionDataStateError::InvalidCheckedDivision); - } - if timing == ExecutionTiming::IngestionTime { - // The opaque series identity is a non-null Utf8 key; every - // other column is a non-null Float64 value or timestamp. - let plain_float_or_ts = |schema: &Schema| { - schema.fields.iter().all(|field| { - !field.nullable - && if field.name == crate::ir::schema::PROMQL_SERIES_IDENTITY { - field.dtype == FieldDataType::Plain(DataType::Utf8) - } else { - matches!( - field.dtype, - FieldDataType::Plain(DataType::Float64 | DataType::Timestamp) - ) - } - }) - }; - let float_count = node - .schema - .fields - .iter() - .filter(|f| matches!(f.dtype, FieldDataType::Plain(DataType::Float64))) - .count(); - if operator.vector_match.is_some() - || !matches!(operator.kind, BinaryOpKind::Arithmetic(_)) - || lhs.schema != rhs.schema - || lhs.schema != node.schema - || node - .schema - .fields - .iter() - .filter(|f| f.name == crate::ir::schema::PROMQL_SERIES_IDENTITY) - .count() - > 1 - || (node - .schema - .fields - .iter() - .any(|f| f.name == crate::ir::schema::PROMQL_SERIES_IDENTITY) - && per_series_rows(lhs) - .is_none_or(|rows| per_series_rows(rhs) != Some(rows))) - || !plain_float_or_ts(&node.schema) - || float_count != 1 - { - return Err(ExecutionDataStateError::InvalidIngestionTimeBinary); - } - } - } - _ => { - for child in op.children() { - check_all_plain(&child.schema)?; - } - } - } - Ok(()) -} - -/// Ingestion-time arithmetic needs the same per-series population on both sides. -fn per_series_rows(node: &OperatorNode) -> Option<&OperatorNode> { - use crate::ir::schema::ExactKind; - match &node.operator { - Operator::ASAP(ASAPOp::FinalizeExactAccumulator { child }) => match &child.operator { - Operator::ASAP(ASAPOp::SummaryAgg { - child, - family: FieldDataType::ExactAggregate(ExactKind::Sum | ExactKind::Count, _), - reduction: crate::ir::operator::Reduction::PerEntity, - filter: None, - .. - }) => Some(child), - _ => None, - }, - Operator::NonASAP(NonASAPOp::BinaryOp { lhs, rhs, .. }) => { - let rows = per_series_rows(lhs)?; - (per_series_rows(rhs) == Some(rows)).then_some(rows) - } - _ => None, - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::ir::operator::agg_intent::AggIntent; - use crate::ir::operator::non_asap::NonASAPOp; - use crate::ir::operator::operator_properties::{Reduction, Source}; - use crate::ir::scalar::expr_ir::ColumnRef; - use crate::ir::schema::sketch::{ - ExactKind, ExactParams, GroupingStrategy, SketchAlgorithm, SketchKind, SketchParams, - SketchStatistic, SummaryUpdate, - }; - use crate::ir::schema::Field; - - fn scan_with(fields: Vec) -> Rc { - OperatorNode::new_shared(crate::ir::Operator::NonASAP(NonASAPOp::Scan { - source: Source::TimeSeries { metric: "m".into() }, - predicates: vec![], - schema: Schema::with_time_index(fields, 0, vec![]), - })) - .unwrap() - } - - fn scan() -> Rc { - scan_with(vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("value", DataType::Float64, false), - Field::plain("zone", DataType::Utf8, true), - ]) - } - - fn kll() -> FieldDataType { - FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: 200, - failure_probability: None, - }, - ), - GroupingStrategy::default(), - ) - } - - fn exact_sum() -> FieldDataType { - FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum) - } - - fn agg(child: Rc, family: FieldDataType) -> Rc { - std::rc::Rc::new( - OperatorNode::with_schema( - crate::ir::Operator::ASAP(ASAPOp::SummaryAgg { - child, - family: family.clone(), - input: SummaryUpdate::column(ColumnRef::SampleValue), - reduction: Reduction::by(vec![]), - grouping: GroupingStrategy::default(), - filter: None, - }), - Schema::lifted(vec![Field::new("state", family, false)], None), - ) - .with_guarantee(None), - ) - } - - fn estimate(child: Rc) -> Rc { - std::rc::Rc::new( - OperatorNode::with_schema( - crate::ir::Operator::ASAP(ASAPOp::SummaryEstimate { - summary_input: child, - query: SketchStatistic::Quantile { q: 0.99 }, - }), - Schema::lifted( - vec![Field::plain("quantile_0_99", DataType::Float64, false)], - None, - ), - ) - .with_guarantee(None), - ) - } - - fn aggregate(measure: AggIntent, child: Rc) -> Rc { - OperatorNode::new_shared(crate::ir::Operator::NonASAP(NonASAPOp::Aggregate { - reduction: Reduction::by(vec![]), - measures: vec![measure], - output_names: vec![], - filters: vec![], - having: None, - child, - })) - .unwrap() - } - - fn max(child: Rc) -> Rc { - aggregate(AggIntent::Max { col: None }, child) - } - - /// `node` with its timing fixed in advance, as a candidate builder does - /// for an operator that must feed maintenance. - fn placed_at_ingestion(node: Rc) -> Rc { - Rc::new( - (*node) - .clone() - .with_timing(Some(ExecutionTiming::IngestionTime)), - ) - } - - /// Apply with every sub-DAG `MatAtIngestion` where the DAG allows it, the - /// placement whose edge rules these tests exercise. - fn apply(root: &Rc) -> Result, ExecutionDataStateError> { - apply_materialization_timings(root, &all_at_ingestion(), &mut NodeToTimedNodeCache::new()) - } - - const AT_INGESTION: Materialization = Materialization::MatAtIngestion(Medium::Memory); - - fn child(node: &Rc) -> Rc { - Rc::clone(node.children()[0]) - } - - /// Without a materialization decision, a summary is `NotMaterialized` and - /// it and its input run at query time; an explicit choice overrides the - /// default and is recorded on the timed node. - #[test] - fn default_assignment_materializes_nothing() { - let summary = agg(scan(), kll()); - let root = apply_materialization_timings( - &summary, - &MaterializationAssignment::default(), - &mut NodeToTimedNodeCache::new(), - ) - .unwrap(); - assert_eq!( - data_state(&root), - Some(ExecutionDataState { - timing: ExecutionTiming::QueryTime, - primitive: DataPrimitive::SummaryState, - }) - ); - assert_eq!(root.materialization, Materialization::NotMaterialized); - assert_eq!( - data_state(&child(&root)), - Some(ExecutionDataState::QUERY_ROWS) - ); - for (materialization, timing) in [ - (AT_INGESTION, ExecutionTiming::IngestionTime), - ( - Materialization::MatAtQuery(Medium::Disk), - ExecutionTiming::QueryTime, - ), - ] { - let mut assignment = MaterializationAssignment::default(); - assignment.set(&summary, materialization); - let root = apply_materialization_timings( - &summary, - &assignment, - &mut NodeToTimedNodeCache::new(), - ) - .unwrap(); - assert_eq!(root.timing, Some(timing)); - assert_eq!(root.materialization, materialization); - assert_eq!(child(&root).timing, Some(timing)); - assert_eq!( - child(&root).materialization, - Materialization::NotMaterialized - ); - } - } - - /// #509 Algorithm 3, line 13: an explicit `MatAtIngestion` over a state - /// that only exists at query time is an error, not a silent move to - /// query time. - #[test] - fn ingestion_time_node_over_query_time_upstream_is_rejected() { - let inner = agg(scan(), exact_sum()); - let outer = agg(Rc::clone(&inner), kll()); - let mut assignment = MaterializationAssignment::default(); - assignment.set(&outer, AT_INGESTION); - assert_eq!( - apply_materialization_timings( - &estimate(outer), - &assignment, - &mut NodeToTimedNodeCache::new() - ) - .err(), - Some(ExecutionDataStateError::UpstreamNotAtIngestion { - operator: "SummaryAgg" - }) - ); - assignment.set(&inner, AT_INGESTION); - let outer = agg(Rc::clone(&inner), kll()); - assignment.set(&outer, AT_INGESTION); - let root = apply_materialization_timings( - &estimate(outer), - &assignment, - &mut NodeToTimedNodeCache::new(), - ) - .unwrap(); - assert_eq!(child(&root).materialization, AT_INGESTION); - assert_eq!(child(&child(&root)).materialization, AT_INGESTION); - } - - #[test] - fn summary_agg_input_runs_at_ingestion_time() { - let root = apply(&agg(scan(), kll())).unwrap(); - assert_eq!( - data_state(&root), - Some(ExecutionDataState::INGESTION_SUMMARY) - ); - assert_eq!( - data_state(&child(&root)), - Some(ExecutionDataState::INGESTION_ROWS) - ); - } - - #[test] - fn exact_accumulator_state_may_feed_another_summary_agg() { - let inner = agg(scan(), exact_sum()); - assert!(apply(&estimate(agg(inner, kll()))).is_ok()); - } - - /// A state not at ingestion time leaves the summary that reads it, when - /// that summary has no explicit assignment, `NotMaterialized` at query - /// time, as an evaluation below it would. - #[test] - fn query_time_state_moves_its_consuming_summary_to_query_time() { - let inner = agg(scan(), exact_sum()); - let outer = agg(Rc::clone(&inner), kll()); - let mut assignment = all_at_ingestion(); - assignment.set(&inner, Materialization::NotMaterialized); - let root = apply_materialization_timings( - &estimate(outer), - &assignment, - &mut NodeToTimedNodeCache::new(), - ) - .unwrap(); - let outer = child(&root); - assert_eq!(outer.timing, Some(ExecutionTiming::QueryTime)); - assert_eq!(outer.materialization, Materialization::NotMaterialized); - assert_eq!(child(&outer).timing, Some(ExecutionTiming::QueryTime)); - } - - #[test] - fn evaluation_can_feed_summary_construction_at_query_time() { - let inner = estimate(agg(scan(), kll())); - let root = apply(&estimate(agg(inner, kll()))).unwrap(); - assert_eq!(child(&root).timing, Some(ExecutionTiming::QueryTime)); - } - - /// Any non-ASAP operator over an evaluation runs at query time. - #[test] - fn query_time_operation_over_evaluation_is_legal_and_root_is_evaluation() { - let evaluation = || estimate(agg(scan(), kll())); - let sorted = OperatorNode::new_shared(crate::ir::Operator::NonASAP(NonASAPOp::Sort { - keys: vec![], - partition_by: Default::default(), - child: evaluation(), - })) - .unwrap(); - for root in [max(evaluation()), sorted] { - let root = apply(&root).unwrap(); - assert_eq!(data_state(&root), Some(ExecutionDataState::QUERY_ROWS)); - } - } - - #[test] - fn query_time_values_can_feed_query_time_summary_construction() { - let post = max(estimate(agg(scan(), kll()))); - let root = apply(&estimate(agg(post, kll()))).unwrap(); - assert_eq!(child(&root).timing, Some(ExecutionTiming::QueryTime)); - } - - #[test] - fn function_under_summary_agg_is_legal_but_not_at_root() { - let operation = placed_at_ingestion(max(scan())); - assert_eq!( - apply(&operation).err(), - Some(ExecutionDataStateError::UnstoredIngestionRoot { - operator: "Aggregate" - }) - ); - let root = apply(&estimate(agg(operation, kll()))).unwrap(); - let timed_operation = child(&child(&root)); - assert_eq!( - data_state(&timed_operation), - Some(ExecutionDataState::INGESTION_ROWS) - ); - } - - #[test] - fn function_over_evaluation_is_rejected() { - let operation = placed_at_ingestion(max(estimate(agg(scan(), kll())))); - assert_eq!( - apply(&estimate(agg(operation, kll()))).err(), - Some(ExecutionDataStateError::UpstreamNotAtIngestion { - operator: "Aggregate" - }) - ); - } - - /// One shared sub-DAG reached as ingestion-time input and as query-time - /// input cannot be executed once for both (Stage 2's - /// `split_shared_by_timing` copies it first). - #[test] - fn a_shared_subtree_reached_at_two_timings_conflicts() { - let shared = scan(); - let root = OperatorNode::new_shared(crate::ir::Operator::NonASAP(NonASAPOp::Concat { - children: vec![ - max(estimate(agg(Rc::clone(&shared), kll()))), - max(Rc::clone(&shared)), - ], - })) - .unwrap(); - assert_eq!( - apply(&root).err(), - Some(ExecutionDataStateError::ConflictingTiming { - first: ExecutionDataState::INGESTION_ROWS, - second: ExecutionDataState::QUERY_ROWS, - }) - ); - } - - /// A finalize pinned to ingestion time (as fixed-window rate candidates - /// build it) over a state that is not `MatAtIngestion` names the missing - /// upstream, not an illegal edge. - #[test] - fn ingestion_pinned_finalize_over_query_time_state_is_upstream_not_at_ingestion() { - let finalize = placed_at_ingestion( - OperatorNode::new_shared(crate::ir::Operator::ASAP( - ASAPOp::FinalizeExactAccumulator { - child: agg(scan(), exact_sum()), - }, - )) - .unwrap(), - ); - assert_eq!( - apply_materialization_timings( - &estimate(agg(finalize, kll())), - &MaterializationAssignment::default(), - &mut NodeToTimedNodeCache::new() - ) - .err(), - Some(ExecutionDataStateError::UpstreamNotAtIngestion { - operator: "FinalizeExactAccumulator" - }) - ); - } - - /// Any node may be materialized, not only a summary state: stored rows - /// at ingestion time are read by a query-time consumer, and a node that - /// can only run at query time cannot be stored at ingestion time. - #[test] - fn any_node_may_be_materialized() { - let tumbling_windows = max(scan()); - let root = max(Rc::clone(&tumbling_windows)); - let mut assignment = MaterializationAssignment::default(); - assignment.set(&tumbling_windows, AT_INGESTION); - let timed = - apply_materialization_timings(&root, &assignment, &mut NodeToTimedNodeCache::new()) - .unwrap(); - assert_eq!(timed.timing, Some(ExecutionTiming::QueryTime)); - assert_eq!(child(&timed).materialization, AT_INGESTION); - assert_eq!(child(&timed).timing, Some(ExecutionTiming::IngestionTime)); - assert_eq!( - child(&child(&timed)).timing, - Some(ExecutionTiming::IngestionTime) - ); - - let over_evaluation = max(estimate(agg(scan(), kll()))); - let mut assignment = MaterializationAssignment::default(); - assignment.set(&over_evaluation, AT_INGESTION); - assert_eq!( - apply_materialization_timings( - &over_evaluation, - &assignment, - &mut NodeToTimedNodeCache::new() - ) - .err(), - Some(ExecutionDataStateError::UpstreamNotAtIngestion { - operator: "Aggregate" - }) - ); - } - - /// A summary build with a filter never runs at ingestion time: an - /// ingestion-time build reads raw data samples, which carry none of the - /// query's columns the filter reads. An explicit `MatAtIngestion` is an - /// error; the all-at-ingestion default leaves it at query time. - #[test] - fn a_filtered_summary_build_never_runs_at_ingestion_time() { - use crate::ir::scalar::expr_ir::ScalarValue; - use crate::ir::{Predicate, ScalarExpr}; - let plain = agg(scan(), kll()); - let Operator::ASAP(ASAPOp::SummaryAgg { - child: rows, - family, - input, - reduction, - grouping, - .. - }) = plain.operator.clone() - else { - unreachable!() - }; - let filtered = Rc::new( - OperatorNode::with_schema( - Operator::ASAP(ASAPOp::SummaryAgg { - child: rows, - family, - input, - reduction, - grouping, - filter: Some(Predicate(ScalarExpr::Literal(ScalarValue::Boolean(true)))), - }), - plain.schema.clone(), - ) - .with_guarantee(None), - ); - let mut assignment = MaterializationAssignment::default(); - assignment.set(&filtered, AT_INGESTION); - assert_eq!( - apply_materialization_timings( - &estimate(Rc::clone(&filtered)), - &assignment, - &mut NodeToTimedNodeCache::new() - ) - .err(), - Some(ExecutionDataStateError::FilteredSummaryAtIngestion { - operator: "SummaryAgg" - }) - ); - let timed = apply(&estimate(filtered)).unwrap(); - assert_eq!(child(&timed).timing, Some(ExecutionTiming::QueryTime)); - assert_eq!( - child(&timed).materialization, - Materialization::NotMaterialized - ); - assert!(validate_at_ingestion(&child(&timed), ExecutionTiming::QueryTime).is_ok()); - } - - fn finalize(child: Rc) -> Rc { - OperatorNode::new_shared(crate::ir::Operator::ASAP( - ASAPOp::FinalizeExactAccumulator { child }, - )) - .unwrap() - } - - /// The materialization the exported DAG records for `node`'s operator kind. - fn exported(root: &Rc, kind: &str) -> Materialization { - let dag = crate::ir::physical_export::compile_physical_asap_dag(root).unwrap(); - dag.validate().unwrap(); - dag.nodes - .iter() - .find(|n| n.payload.kind_name() == kind) - .unwrap() - .materialization - } - - /// A node whose timing its candidate fixed (the query-time read boundary - /// of `finalize_query_candidate`) keeps an explicit assignment that agrees - /// with that timing and rejects one that needs the other timing. - #[test] - fn fixed_timing_keeps_an_agreeing_assignment_and_rejects_a_conflict() { - let state = agg(scan(), exact_sum()); - let read = Rc::new( - (*finalize(Rc::clone(&state))) - .clone() - .with_timing(Some(ExecutionTiming::QueryTime)), - ); - let stored = Materialization::MatAtQuery(Medium::Disk); - let mut assignment = MaterializationAssignment::default(); - assignment.set(&read, stored); - let timed = - apply_materialization_timings(&read, &assignment, &mut NodeToTimedNodeCache::new()) - .unwrap(); - assert_eq!(timed.timing, Some(ExecutionTiming::QueryTime)); - assert_eq!(timed.materialization, stored); - assert_eq!(exported(&timed, "FinalizeExactAccumulator"), stored); - - let mut assignment = MaterializationAssignment::default(); - assignment.set(&state, AT_INGESTION); - assignment.set(&read, AT_INGESTION); - assert_eq!( - apply_materialization_timings(&read, &assignment, &mut NodeToTimedNodeCache::new()) - .err(), - Some( - ExecutionDataStateError::MaterializationConflictsWithFixedTiming { - operator: "FinalizeExactAccumulator", - fixed: ExecutionTiming::QueryTime, - materialization: AT_INGESTION, - } - ) - ); - } - - /// The Debug output names each assigned node by its operator kind, not - /// its whole sub-DAG. - #[test] - fn debug_names_each_assigned_node_by_its_kind() { - let mut assignment = MaterializationAssignment::default(); - assignment.set(&agg(scan(), kll()), AT_INGESTION); - let text = format!("{assignment:?}"); - assert!(text.contains("SummaryAgg@"), "{text}"); - assert!(!text.contains("fields"), "{text}"); - } - - /// A node fixed at ingestion time by its candidate rejects a `MatAtQuery` - /// assignment, which needs the other timing. - #[test] - fn ingestion_fixed_node_rejects_mat_at_query() { - let fixed = placed_at_ingestion(max(scan())); - let root = max(Rc::clone(&fixed)); - let at_query = Materialization::MatAtQuery(Medium::Memory); - let mut assignment = MaterializationAssignment::default(); - assignment.set(&fixed, at_query); - assert_eq!( - apply_materialization_timings(&root, &assignment, &mut NodeToTimedNodeCache::new()) - .err(), - Some( - ExecutionDataStateError::MaterializationConflictsWithFixedTiming { - operator: "Aggregate", - fixed: ExecutionTiming::IngestionTime, - materialization: at_query, - } - ) - ); - } - - /// An explicit `NotMaterialized` on a node whose timing its candidate - /// fixed agrees with either timing: the node keeps its fixed timing and - /// records `NotMaterialized`. - #[test] - fn explicit_not_materialized_keeps_a_fixed_timing() { - let read = Rc::new( - (*finalize(agg(scan(), exact_sum()))) - .clone() - .with_timing(Some(ExecutionTiming::QueryTime)), - ); - let root = max(Rc::clone(&read)); - let mut assignment = MaterializationAssignment::default(); - assignment.set(&read, Materialization::NotMaterialized); - let timed = - apply_materialization_timings(&root, &assignment, &mut NodeToTimedNodeCache::new()) - .unwrap(); - let read = child(&timed); - assert_eq!(read.timing, Some(ExecutionTiming::QueryTime)); - assert_eq!(read.materialization, Materialization::NotMaterialized); - } - - /// A query-time summary build cannot read ingestion-time rows that are - /// not stored: at query time they no longer exist. Storing the rows node - /// at ingestion time makes the same build valid (#738). - #[test] - fn query_time_summary_over_unstored_ingestion_rows_is_rejected() { - let rows = placed_at_ingestion(max(scan())); - let summary = agg(Rc::clone(&rows), kll()); - assert_eq!( - apply_materialization_timings( - &summary, - &MaterializationAssignment::default(), - &mut NodeToTimedNodeCache::new() - ) - .err(), - Some( - ExecutionDataStateError::UnstoredIngestionInputUnderQueryTimeSummary { - edge: "SummaryAgg.child", - operator: "Aggregate", - } - ) - ); - - let mut assignment = MaterializationAssignment::default(); - assignment.set(&rows, AT_INGESTION); - let timed = - apply_materialization_timings(&summary, &assignment, &mut NodeToTimedNodeCache::new()) - .unwrap(); - assert_eq!(timed.timing, Some(ExecutionTiming::QueryTime)); - assert_eq!(child(&timed).materialization, AT_INGESTION); - } - - /// The same holds for exact-accumulator state: a query-time build cannot - /// read an ingestion-time exact state that is not stored. - #[test] - fn query_time_summary_over_unstored_ingestion_state_is_rejected() { - let state = placed_at_ingestion(agg(scan(), exact_sum())); - let summary = agg(Rc::clone(&state), kll()); - assert_eq!( - apply_materialization_timings( - &summary, - &MaterializationAssignment::default(), - &mut NodeToTimedNodeCache::new() - ) - .err(), - Some( - ExecutionDataStateError::UnstoredIngestionInputUnderQueryTimeSummary { - edge: "SummaryAgg.child", - operator: "SummaryAgg", - } - ) - ); - - let mut assignment = MaterializationAssignment::default(); - assignment.set(&state, AT_INGESTION); - let timed = - apply_materialization_timings(&summary, &assignment, &mut NodeToTimedNodeCache::new()) - .unwrap(); - assert_eq!(timed.timing, Some(ExecutionTiming::QueryTime)); - assert_eq!(child(&timed).materialization, AT_INGESTION); - } - - /// A root stored at ingestion time is valid: the query reads the stored - /// output. A root that runs at ingestion time without being stored there - /// is not. - #[test] - fn a_root_may_be_stored_at_ingestion_time() { - let root = max(scan()); - let mut assignment = MaterializationAssignment::default(); - assignment.set(&root, AT_INGESTION); - let timed = - apply_materialization_timings(&root, &assignment, &mut NodeToTimedNodeCache::new()) - .unwrap(); - assert_eq!(timed.timing, Some(ExecutionTiming::IngestionTime)); - assert_eq!(exported(&timed, "Aggregate"), AT_INGESTION); - - let state = agg(scan(), exact_sum()); - let root = finalize(Rc::clone(&state)); - let mut assignment = MaterializationAssignment::default(); - assignment.set(&state, AT_INGESTION); - assignment.set(&root, AT_INGESTION); - let timed = - apply_materialization_timings(&root, &assignment, &mut NodeToTimedNodeCache::new()) - .unwrap(); - assert_eq!(timed.timing, Some(ExecutionTiming::IngestionTime)); - assert_eq!(exported(&timed, "FinalizeExactAccumulator"), AT_INGESTION); - - let mut assignment = MaterializationAssignment::default(); - let root = placed_at_ingestion(max(scan())); - assignment.set(&root, Materialization::NotMaterialized); - assert_eq!( - apply_materialization_timings(&root, &assignment, &mut NodeToTimedNodeCache::new()) - .err(), - Some(ExecutionDataStateError::UnstoredIngestionRoot { - operator: "Aggregate" - }) - ); - } - - /// Both paired operands must be plain; an unrelated state column is not - /// an input. - #[test] - fn pearson_corr_checks_both_operand_states() { - let corr_over = |state_column: usize| { - let mut fields = vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("x", DataType::Float64, false), - Field::plain("y", DataType::Float64, false), - Field::plain("unused", DataType::Float64, false), - ]; - fields[state_column].dtype = kll(); - aggregate( - AggIntent::PearsonCorr { left: 1, right: 2 }, - scan_with(fields), - ) - }; - for operand in [1, 2] { - assert!(matches!( - validate_at_ingestion(&corr_over(operand), ExecutionTiming::QueryTime), - Err(ExecutionDataStateError::NonPlainOperand { .. }) - )); - } - validate_at_ingestion(&corr_over(3), ExecutionTiming::QueryTime).unwrap(); - } - // Typed derived updates retain one opaque series identity only when both - // operands cover the same series; arbitrary labels are never admitted. - #[test] - fn ingestion_time_binary_accepts_only_well_typed_series_identity() { - use crate::ir::operator::non_asap::BinaryOperator; - use crate::ir::operator::BinaryOpKind; - use crate::ir::scalar::ArithmeticOpKind; - let identity = crate::ir::schema::PROMQL_SERIES_IDENTITY; - // A finalized per-series Sum of `metric`'s rows. - let operand = |metric: &str, schema: &Schema| { - let rows = OperatorNode::new_shared(crate::ir::Operator::NonASAP(NonASAPOp::Scan { - source: Source::TimeSeries { - metric: metric.into(), - }, - predicates: vec![], - schema: Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("value", DataType::Float64, false), - ], - 0, - vec![], - ), - })) - .unwrap(); - let mut state = schema.clone(); - state.fields[0].dtype = exact_sum(); - let sum = Rc::new(OperatorNode::with_schema( - crate::ir::Operator::ASAP(ASAPOp::SummaryAgg { - child: rows, - family: exact_sum(), - input: SummaryUpdate::column(ColumnRef::SampleValue), - reduction: Reduction::PerEntity, - grouping: GroupingStrategy::default(), - filter: None, - }), - state, - )); - Rc::new(OperatorNode::with_schema( - crate::ir::Operator::ASAP(ASAPOp::FinalizeExactAccumulator { child: sum }), - schema.clone(), - )) - }; - let validate = |schema: Schema, rhs: &str| { - let binary = Rc::new(OperatorNode::with_schema( - crate::ir::Operator::NonASAP(NonASAPOp::BinaryOp { - operator: BinaryOperator { - kind: BinaryOpKind::Arithmetic(ArithmeticOpKind::Add), - vector_match: None, - checked_relative_division: false, - checked_finite_division: false, - }, - return_bool: false, - lhs: operand("m", &schema), - rhs: operand(rhs, &schema), - }), - schema, - )); - validate_at_ingestion(&placed_at_ingestion(binary), ExecutionTiming::IngestionTime) - }; - let mut schema = Schema::lifted( - vec![ - Field::plain("value", DataType::Float64, false), - Field::plain("ts", DataType::Timestamp, false), - ], - None, - ); - schema.time_index = Some(1); - assert!(validate(schema.clone(), "m").is_ok()); - assert!( - validate(schema.clone(), "n").is_ok(), - "no identity to align" - ); - schema - .fields - .push(Field::plain(identity, DataType::Utf8, false)); - assert!(validate(schema.clone(), "m").is_ok()); - assert_eq!( - validate(schema.clone(), "n"), - Err(ExecutionDataStateError::InvalidIngestionTimeBinary), - "different selectors may cover different series" - ); - for mutation in 0..4 { - let mut invalid = schema.clone(); - match mutation { - 0 => invalid.fields[2].nullable = true, - 1 => invalid.fields[2].dtype = FieldDataType::Plain(DataType::Timestamp), - 2 => invalid.fields.push(invalid.fields[2].clone()), - _ => invalid.fields[2].name = "label".into(), - } - assert_eq!( - validate(invalid, "m"), - Err(ExecutionDataStateError::InvalidIngestionTimeBinary) - ); - } - } -} diff --git a/crates/types/src/ir/scalar/mod.rs b/crates/types/src/ir/scalar/mod.rs index 851e076bb..1b4753b89 100644 --- a/crates/types/src/ir/scalar/mod.rs +++ b/crates/types/src/ir/scalar/mod.rs @@ -19,6 +19,7 @@ use std::rc::Rc; use serde::{Deserialize, Serialize}; +use crate::ir::operator::input::OperatorInput; use crate::ir::operator::node::OperatorNode; use crate::ir::scalar::scalar_type_rules::MapScalarFunction; use crate::ir::schema::{ColumnId, DataType, Schema}; @@ -353,7 +354,7 @@ impl ScalarExpr { } } -impl ScalarExpr { +impl ScalarExpr { /// Infer the `(DataType, nullable)` this expression produces against the /// input schema its owner evaluates it in. Unregistered functions are /// rejected. A reference to a field carrying summary state is an error: @@ -579,7 +580,7 @@ impl ScalarExpr { // `scalar(v)` is one float sample (NaN when the vector is not // exactly one series); a scalar subquery is its single column. ScalarExpr::PromqlScalarFromVector(node) => { - if node.result_kind != super::OperatorResultKind::InstantVector { + if node.result_kind() != super::OperatorResultKind::InstantVector { return Err(signature("scalar() requires an instant vector")); } (DataType::Float64, false) @@ -622,25 +623,25 @@ fn common_scalar_type(a: &DataType, b: &DataType) -> Result Result<(), SchemaDerivationError> { - if node.result_kind == super::OperatorResultKind::Relation { +fn relation(node: &C) -> Result<(), SchemaDerivationError> { + if node.result_kind() == super::OperatorResultKind::Relation { Ok(()) } else { Err(signature("SQL subquery requires a relation")) } } -fn scalar_subquery_field( - node: &OperatorNode, +fn scalar_subquery_field( + node: &C, ) -> Result<&crate::ir::schema::Field, SchemaDerivationError> { relation(node)?; - match node.schema.fields.as_slice() { + match node.schema().fields.as_slice() { [field] => Ok(field), _ => Err(signature("scalar subquery requires exactly one column")), } } -fn sql_function_type( +fn sql_function_type( name: &str, - args: &[ScalarExpr], + args: &[ScalarExpr], schema: &Schema, ) -> Result<(DataType, bool), SchemaDerivationError> { let types = args @@ -695,7 +696,10 @@ fn sql_function_type( /// Resolve the bounded canonical `asap_struct_field(struct, selector)` operation. /// Selectors are positive 1-based literal ordinals or exact literal field names. -pub fn struct_field_type(args: &[ScalarExpr], schema: &Schema) -> Result<(DataType, bool), String> { +pub fn struct_field_type( + args: &[ScalarExpr], + schema: &Schema, +) -> Result<(DataType, bool), String> { let [input, selector] = args else { return Err("struct field access requires a struct and constant selector".into()); }; @@ -732,8 +736,8 @@ pub fn struct_field_type(args: &[ScalarExpr], schema: &Schema) -> Result<(DataTy /// Resolve `asap_element_access(collection, index)`: Map access through the /// map function contract, List access with integer indices. -pub fn element_access_type( - args: &[ScalarExpr], +pub fn element_access_type( + args: &[ScalarExpr], schema: &Schema, ) -> Result<(DataType, bool), String> { let [input, index] = args else { diff --git a/crates/types/src/ir/schema/family_capability.rs b/crates/types/src/ir/schema/family_capability.rs index 6af62d919..960403e36 100644 --- a/crates/types/src/ir/schema/family_capability.rs +++ b/crates/types/src/ir/schema/family_capability.rs @@ -175,8 +175,8 @@ impl FamilyCapability { HydraKind::HydraKll => SketchAlgorithm::HydraKll, }) } - FieldDataType::Sketch(kind, GroupingStrategy::PerSubpopulationInstance) => { - return Self::of_per_instance_sketch(kind.algorithm()) + FieldDataType::Sketch(algorithm, GroupingStrategy::PerSubpopulationInstance) => { + return Self::of_per_instance_sketch(algorithm) } FieldDataType::Plain(_) | FieldDataType::SummarySample(..) @@ -194,13 +194,10 @@ impl FamilyCapability { mod tests { use super::*; use crate::ir::scalar::expr_ir::ColumnRef; - use crate::ir::schema::{ExactParams, SketchKind, SketchParams}; + use crate::ir::schema::{ExactParams, SketchParams}; - fn sketch(algorithm: SketchAlgorithm, params: SketchParams) -> FieldDataType { - FieldDataType::Sketch( - SketchKind::new(algorithm, params), - GroupingStrategy::default(), - ) + fn sketch(algorithm: SketchAlgorithm, _params: SketchParams) -> FieldDataType { + FieldDataType::Sketch(algorithm, GroupingStrategy::default()) } fn total() -> SketchStatistic { @@ -335,7 +332,7 @@ mod tests { /// per group: not counters, IRate or a Hydra. #[test] fn only_per_group_mergeable_families_merge_groups() { - use crate::ir::schema::{HydraKind, HydraParams}; + use crate::ir::schema::HydraKind; let kll = sketch( SketchAlgorithm::Kll, SketchParams::Kll { @@ -369,12 +366,6 @@ mod tests { kind, GroupingStrategy::SharedMultiSubpopulation { kind: HydraKind::HydraCms, - params: HydraParams::HydraCms { - width: 64, - depth: 5, - shared_rows: 3, - shared_columns: 256, - }, }, ); assert_eq!( diff --git a/crates/types/src/ir/schema/mod.rs b/crates/types/src/ir/schema/mod.rs index 90796a76b..cce079379 100644 --- a/crates/types/src/ir/schema/mod.rs +++ b/crates/types/src/ir/schema/mod.rs @@ -164,11 +164,11 @@ pub enum FieldDataType { /// `Increase`). Value consumers require an explicit finalization boundary. ExactAggregate(ExactKind, ExactParams), /// Approximate sketch state (KLL/CMS/HLL/…), evaluated by a - /// `SummaryEstimate`. A [`SketchKind`] already carries the concrete - /// algorithm, params, and grouping layout committed to, not just its - /// category — a bound node needs to know it's specifically independent - /// KLL or shared Hydra-backed CMS, not merely "some sketch". - Sketch(SketchKind, GroupingStrategy), + /// `SummaryEstimate`: its family and grouping layout. The family's + /// parameters θ are not part of the state's type, so states of one family + /// and grouping are one memo group whatever their size; θ is a + /// candidate's `LogicalProperties::parameters` (concepts doc, section 3). + Sketch(SketchAlgorithm, GroupingStrategy), /// A summary sample: the subset of input rows a [`SamplingKind`] keeps, /// such as a reservoir. Not raw data samples. Serialized as `Sample`. #[serde(rename = "Sample")] diff --git a/crates/types/src/ir/schema/schema_support.rs b/crates/types/src/ir/schema/schema_support.rs index 913ab8728..61bc53e69 100644 --- a/crates/types/src/ir/schema/schema_support.rs +++ b/crates/types/src/ir/schema/schema_support.rs @@ -64,7 +64,7 @@ pub fn with_promql_series_identity( // A series has at most one sample per timestamp, so the full // identity and the timestamp key each row. Pass 2's // identical-expression rule reads this key: only a keyed - // node is legal to share (Stage 0 hash-consing, `cse` in `asap_logical_asap_optimization`). + // node is legal to share (hash-consing, `Memo::find_or_add_group`). if let Some(time) = schema.time_index { schema.add_unique_key(vec![schema.fields.len() - 1, time]); } @@ -92,8 +92,6 @@ pub fn with_promql_series_identity( } let mut rebuilt = OperatorNode::new(operator).map_err(|e| e.to_string())?; rebuilt.guarantee = node.guarantee.clone(); - rebuilt.timing = node.timing; - rebuilt.materialization = node.materialization; let rebuilt = Rc::new(rebuilt); cache.insert(Rc::as_ptr(node), Rc::clone(&rebuilt)); Ok(rebuilt) diff --git a/crates/types/src/ir/schema/sketch.rs b/crates/types/src/ir/schema/sketch.rs index a2c3a21e1..b4c4ace79 100644 --- a/crates/types/src/ir/schema/sketch.rs +++ b/crates/types/src/ir/schema/sketch.rs @@ -133,6 +133,10 @@ pub enum SketchParams { depth: u32, heap_size: u32, }, + /// The sizes of a Hydra structure shared across subpopulations + /// ([`GroupingStrategy::SharedMultiSubpopulation`]): θ, like every other + /// family's parameters, not part of the grouping. + Hydra(HydraParams), } impl PartialEq for SketchParams { @@ -203,6 +207,7 @@ impl PartialEq for SketchParams { (P::Hll { precision: a, .. }, P::Hll { precision: b, .. }) => a == b, (P::DDSketch { alpha: a }, P::DDSketch { alpha: b }) => a == b, (P::Kmv { k: a, .. }, P::Kmv { k: b, .. }) => a == b, + (P::Hydra(a), P::Hydra(b)) => a == b, // Different variants. Listed by `self`'s variant, not `_`, so a // new variant fails to compile here until it is compared above. (P::UnivMon { .. }, _) @@ -213,7 +218,8 @@ impl PartialEq for SketchParams { | (P::CmsWithHeap { .. }, _) | (P::Kmv { .. }, _) | (P::CountSketch { .. }, _) - | (P::CountSketchWithHeap { .. }, _) => false, + | (P::CountSketchWithHeap { .. }, _) + | (P::Hydra(_), _) => false, } } } @@ -490,6 +496,23 @@ pub enum HydraParams { }, } +impl HydraParams { + /// The parameters of the per-subpopulation sketch this shared structure + /// emulates, which its error bound and per-group state follow. + pub fn per_subpopulation(&self) -> SketchParams { + match *self { + HydraParams::HydraKll { k, .. } => SketchParams::Kll { + k, + failure_probability: None, + }, + HydraParams::HydraCms { width, depth, .. } => SketchParams::Cms { width, depth }, + HydraParams::HydraCountSketch { width, depth, .. } => { + SketchParams::CountSketch { width, depth } + } + } + } +} + /// How a grouped aggregate's summary state is physically instantiated /// across its `by` subpopulations — orthogonal to *which* /// `SketchKind`/`SamplingKind`/`WaveletKind`/`StatModelKind` answers the @@ -517,10 +540,8 @@ pub enum GroupingStrategy { /// shares across a *query's own* subpopulations/group-by keys, not /// across tenants in a deployment-isolation sense — a different, and /// unrelated, kind of "sharing". - SharedMultiSubpopulation { - kind: HydraKind, - params: HydraParams, - }, + /// Its sizes are θ ([`SketchParams::Hydra`]), not part of the grouping. + SharedMultiSubpopulation { kind: HydraKind }, } impl Default for GroupingStrategy { @@ -751,33 +772,6 @@ mod tests { assert_eq!(kind.algorithm(), &SketchAlgorithm::Kll); } - /// Two KLLs with the same `k` are the same sketch, whatever failure - /// probability each was sized for, so they compare equal and can be shared. - #[test] - fn kll_params_compare_equal_whatever_their_failure_probability() { - let kll = |k, failure_probability| SketchParams::Kll { - k, - failure_probability, - }; - assert_eq!(kll(331, Some(0.001)), kll(331, None)); - assert_eq!(kll(331, Some(0.001)), kll(331, Some(0.0001))); - assert_ne!(kll(331, None), kll(269, None)); - assert_ne!( - kll(331, None), - SketchParams::Kmv { - k: 331, - relative_error: None - } - ); - let family = |params| { - crate::ir::schema::FieldDataType::Sketch( - SketchKind::new(SketchAlgorithm::Kll, params), - GroupingStrategy::default(), - ) - }; - assert_eq!(family(kll(331, Some(0.001))), family(kll(331, None))); - } - #[test] #[should_panic(expected = "SketchKind parameter mismatch")] fn sketch_kind_rejects_params_from_another_algorithm() { diff --git a/crates/types/src/ir/schema/topk.rs b/crates/types/src/ir/schema/topk.rs index 1ad5c2186..6600fd1ca 100644 --- a/crates/types/src/ir/schema/topk.rs +++ b/crates/types/src/ir/schema/topk.rs @@ -1,9 +1,10 @@ //! Ranked-row output contract shared by logical TopK and its implementations. +use crate::ir::operator::input::OperatorInput; use crate::ir::operator::{Reduction, Source}; use crate::ir::scalar::column_resolution::{resolve_column_ref, ResolveError}; use crate::ir::scalar::{ColumnRef, ProjectItem}; use crate::ir::schema::{ColumnId, DataType, FieldDataType, PROMQL_SERIES_IDENTITY}; -use crate::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode, ScalarExpr}; +use crate::ir::{ASAPOp, NonASAPOp, Operator, ScalarExpr}; /// The column a top-k ranks `child` by. Over a one-measure grouped /// aggregate (the SQL heavy-hitter shape and PromQL `topk(k, sum by ..)`) it @@ -12,17 +13,17 @@ use crate::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode, ScalarExpr}; /// `FinalizeExactAccumulator` or `SummaryEstimate`, under the `Project` /// restoring its names (a renaming) and a value `HAVING`'s `Filter`. Over /// per-series rows (PromQL `topk by (..) (k, rows)`) it is the sample value. -pub fn ranked_column(child: &OperatorNode) -> Result { - let mut node = child; +pub fn ranked_column(child: &I) -> Result { + let mut node = child.clone(); let reduction = loop { - match &node.operator { + match node.operator() { // Only a renaming keeps the measure last. Operator::NonASAP(NonASAPOp::Project { cols, child, .. }) - if cols.len() == child.schema.fields.len() + if cols.len() == child.schema().fields.len() && cols .iter() .enumerate() - .all(|(i, item)| item.expr == ScalarExpr::Column(i)) => + .all(|(i, item)| matches!(item.expr, ScalarExpr::Column(c) if c == i)) => { node = child } @@ -49,34 +50,34 @@ pub fn ranked_column(child: &OperatorNode) -> Result { } }; match reduction { - Some(Reduction::Reduce(_)) => Ok(child.schema.fields.len() - 1), - _ => resolve_column_ref(&ColumnRef::SampleValue, &child.schema), + Some(Reduction::Reduce(_)) => Ok(child.schema().fields.len() - 1), + _ => resolve_column_ref(&ColumnRef::SampleValue, child.schema()), } } /// Ordered projection of ranked rows: partition keys, ranked value, then /// item identity. TopK retains rows; partition keys alone are not unique. -pub fn ranked_row_projection( - child: &OperatorNode, +pub(crate) fn ranked_row_projection( + child: &I, reduction: &Reduction, output_name: &str, -) -> Result, crate::ir::SchemaDerivationError> { +) -> Result>, crate::ir::SchemaDerivationError> { use crate::ir::SchemaDerivationError; let value = ranked_column(child) .map_err(|error| SchemaDerivationError::InvalidScalarSignature(error.to_string()))?; let keys = reduction.group_keys().map_or(&[][..], |keys| keys.keys()); for &index in keys { - if index >= child.schema.fields.len() { + if index >= child.schema().fields.len() { return Err(SchemaDerivationError::InvalidGroupByColumn( index, - child.schema.fields.len(), + child.schema().fields.len(), )); } } - let mut columns: Vec = keys + let mut columns: Vec> = keys .iter() .map(|&index| ProjectItem { - alias: Some(child.schema.fields[index].name.clone()), + alias: Some(child.schema().fields[index].name.clone()), expr: ScalarExpr::Column(index), }) .collect(); @@ -85,7 +86,7 @@ pub fn ranked_row_projection( expr: ScalarExpr::Column(value), }); let promql = reads_only_time_series(child); - for (index, field) in child.schema.fields.iter().enumerate() { + for (index, field) in child.schema().fields.iter().enumerate() { let names_item = !promql || field.dtype == FieldDataType::Plain(DataType::Utf8); if names_item && index != value @@ -102,7 +103,7 @@ pub fn ranked_row_projection( // so consumers' already-bound column positions remain valid. if let Ok(identity) = resolve_column_ref( &ColumnRef::Named(PROMQL_SERIES_IDENTITY.into()), - &child.schema, + child.schema(), ) { if !keys.contains(&identity) { columns.push(ProjectItem { @@ -142,12 +143,11 @@ pub fn series_identity_item( } } -fn reads_only_time_series(node: &OperatorNode) -> bool { - match node.non_asap() { - Some(NonASAPOp::Scan { source, .. }) => matches!(source, Source::TimeSeries { .. }), - _ => node - .children() - .iter() - .all(|child| reads_only_time_series(child)), +fn reads_only_time_series(node: &I) -> bool { + match node.operator() { + Operator::NonASAP(NonASAPOp::Scan { source, .. }) => { + matches!(source, Source::TimeSeries { .. }) + } + operator => operator.children().into_iter().all(reads_only_time_series), } } diff --git a/crates/types/src/lib.rs b/crates/types/src/lib.rs index b6e8c0157..e219b78f5 100644 --- a/crates/types/src/lib.rs +++ b/crates/types/src/lib.rs @@ -12,8 +12,8 @@ //! - [`workload`] — the planner inputs (#509): the query and data //! workloads and the parsed workload the query-language frontends produce //! ([`workload::parsed_workload`]). -//! - [`physical`] — Stage 2 decision data: the materialization of a -//! sub-DAG ([`physical::Materialization`]). +//! - [`physical`] — Stage 2 decision data: the physical properties of a +//! candidate ([`physical::PhysicalProperties`]: timing and lifetime). //! - [`stages`] — the types one planning stage hands the next: the Stage 1 //! → Stage 2 memo of groups and implementation candidates //! ([`stages::LogicalASAPMemoCandidates`]) and the Stage 2 → Stage 3 diff --git a/crates/types/src/physical/materialization.rs b/crates/types/src/physical/materialization.rs deleted file mode 100644 index e30e192b3..000000000 --- a/crates/types/src/physical/materialization.rs +++ /dev/null @@ -1,57 +0,0 @@ -//! The materialization option of each sub-DAG (#509 §Materialization), the -//! Stage 2 decision a node's execution timing -//! ([`crate::ir::properties::ExecutionTiming`]) is derived -//! from. - -use crate::ir::properties::ExecutionTiming; - -/// Where a materialized output is stored (#509 §Materialization). -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] -pub enum Medium { - Memory, - Disk, -} - -/// The materialization option of one sub-DAG (#509 §Materialization, -/// Algorithm 3): whether its output is stored across query executions, and -/// whether it is computed at ingestion time or at query time. -#[derive( - Default, Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize, -)] -pub enum Materialization { - /// Runs at query time for each execution; its output is discarded. - #[default] - NotMaterialized, - /// Runs as data arrives; its output is stored until a query reads it. - MatAtIngestion(Medium), - /// Runs when a query first needs it; its output is reused by the other - /// queries of the same evaluation and discarded after it. - MatAtQuery(Medium), - /// Runs at query time and keeps its output across evaluations: a - /// tumbling-window chain whose readers evaluate together every tumbling window - /// width builds its newest tumbling window at each evaluation and keeps the older - /// ones for the next (#509 Example 4's B3). Offered only when the - /// deployment keeps query-time results. - MatAtQueryKeptAcrossEvaluations(Medium), -} - -impl Materialization { - /// When the sub-DAG runs: only `MatAtIngestion` runs at ingestion time. - pub fn timing(self) -> ExecutionTiming { - match self { - Self::MatAtIngestion(_) => ExecutionTiming::IngestionTime, - Self::NotMaterialized - | Self::MatAtQuery(_) - | Self::MatAtQueryKeptAcrossEvaluations(_) => ExecutionTiming::QueryTime, - } - } - - /// Whether a node of this materialization is a materialized view: its - /// output is kept across evaluations. - pub(crate) fn is_materialized_view(self) -> bool { - matches!( - self, - Self::MatAtIngestion(_) | Self::MatAtQueryKeptAcrossEvaluations(_) - ) - } -} diff --git a/crates/types/src/physical/mod.rs b/crates/types/src/physical/mod.rs index bf10811ac..28db939f6 100644 --- a/crates/types/src/physical/mod.rs +++ b/crates/types/src/physical/mod.rs @@ -1,11 +1,13 @@ -//! #509 Stage 2 decision data: the materialization of each sub-DAG. +//! #509 Stage 2 decision data: the physical properties of each candidate. //! -//! - [`materialization`] — whether a sub-DAG's output is stored and when it -//! is computed ([`Materialization`], [`Medium`]). +//! - [`properties`] — when a candidate runs and how long its output lives +//! ([`PhysicalProperties`], [`Lifetime`], [`StorageMedium`]). //! - [`workload_plan`] — one evaluated assignment rebuilt as a physical DAG //! ([`PhysicalWorkloadPlan`]). -pub mod materialization; +pub mod properties; pub mod workload_plan; -pub use materialization::{Materialization, Medium}; +pub use properties::{ + ExecutionTiming, Lifetime, PhysicalProperties, PhysicalPropertiesError, StorageMedium, +}; pub use workload_plan::PhysicalWorkloadPlan; diff --git a/crates/types/src/physical/properties.rs b/crates/types/src/physical/properties.rs new file mode 100644 index 000000000..3bc12bf0a --- /dev/null +++ b/crates/types/src/physical/properties.rs @@ -0,0 +1,120 @@ +//! A physical candidate's physical properties $p^{\mathrm{phys}}_v = (\tau, \ell)$ +//! (#1200; concepts doc `memo-groups-properties-requirements.md`, sections 1 +//! and 3): when it runs ([`ExecutionTiming`]) and how long its output lives +//! ([`Lifetime`]). There is no schedule property: a `None` or `OneEvaluation` +//! output runs on the schedule of the context κ that runs it, and a tumbling +//! grid is anchored at phase 0 with its width in the logical window layout. +//! +//! | timing | lifetime | runs | +//! |---|---|---| +//! | query | `None` | in each context κ that reads it | +//! | query | `OneEvaluation` | in one context κ; that evaluation's consumers share it | +//! | query | `AcrossEvaluations` | one window when each window ends | +//! | ingestion | `None` | inside the ingestion work that reads it | +//! | ingestion | `AcrossEvaluations` | one window when each window ends | +//! | ingestion | `OneEvaluation` | invalid: there is no evaluation at ingestion time | +//! +//! There is no stored physical requirement: $r^{\mathrm{phys}}_{c,i}$ is a +//! function of the consumer's own properties +//! ([`crate::stages::producer_satisfies_physical_requirements`]). + +pub use crate::ir::properties::ExecutionTiming; + +/// Where a materialized output is stored. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] +pub enum StorageMedium { + Memory, + Disk, +} + +/// $\ell_v$: how long a candidate's output lives. Only a materialized output +/// has a medium. +#[derive( + Default, Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize, +)] +pub enum Lifetime { + /// Discarded after its consumer's execution completes. + #[default] + None, + /// Materialized until every consumer within one evaluation finishes. + OneEvaluation(StorageMedium), + /// Materialized across evaluations, so later evaluations reuse it. + AcrossEvaluations(StorageMedium), +} + +/// A timing and lifetime that cannot go together (the invalid combination +/// of the concepts doc, section 2). +#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +pub enum PhysicalPropertiesError { + #[error("an ingestion-time output cannot be materialized for one evaluation: there is no evaluation at ingestion time")] + OneEvaluationAtIngestion, +} + +/// $p^{\mathrm{phys}}_v = (\tau, \ell)$. [`Self::new`] rejects the invalid +/// combination (concepts doc, section 2). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct PhysicalProperties { + timing: ExecutionTiming, + lifetime: Lifetime, +} + +impl PhysicalProperties { + /// The properties `(timing, lifetime)`, unless they cannot go together. + pub fn new( + timing: ExecutionTiming, + lifetime: Lifetime, + ) -> Result { + let _ = (timing, lifetime); + todo!("#1207 PR 1: reject the invalid combination of section 2") + } + + /// $\tau_v$. + pub fn timing(&self) -> ExecutionTiming { + self.timing + } + + /// $\ell_v$. + pub fn lifetime(&self) -> Lifetime { + self.lifetime + } + + /// Whether $\ell_v \ne$ None. Derived, not stored. + pub fn is_materialized(&self) -> bool { + self.lifetime != Lifetime::None + } +} + +#[cfg(test)] +mod tests { + use super::*; + use ExecutionTiming::{IngestionTime, QueryTime}; + + const MEMORY: StorageMedium = StorageMedium::Memory; + + /// The one invalid combination of the concepts doc, section 2. + #[test] + fn new_rejects_one_evaluation_at_ingestion() { + assert_eq!( + PhysicalProperties::new(IngestionTime, Lifetime::OneEvaluation(MEMORY)), + Err(PhysicalPropertiesError::OneEvaluationAtIngestion) + ); + } + + /// The valid (τ, ℓ) pairs of section 2. + #[test] + fn new_accepts_each_valid_pair() { + for (timing, lifetime) in [ + (QueryTime, Lifetime::None), + (IngestionTime, Lifetime::None), + (QueryTime, Lifetime::OneEvaluation(MEMORY)), + (QueryTime, Lifetime::AcrossEvaluations(MEMORY)), + ( + IngestionTime, + Lifetime::AcrossEvaluations(StorageMedium::Disk), + ), + ] { + let properties = PhysicalProperties::new(timing, lifetime).unwrap(); + assert_eq!(properties.is_materialized(), lifetime != Lifetime::None); + } + } +} diff --git a/crates/types/src/physical/workload_plan.rs b/crates/types/src/physical/workload_plan.rs index 47412bd9b..ce8ed9653 100644 --- a/crates/types/src/physical/workload_plan.rs +++ b/crates/types/src/physical/workload_plan.rs @@ -1,10 +1,8 @@ -//! One evaluated materialization assignment rebuilt as a physical DAG. -use std::collections::BTreeMap; - -use crate::ir::physical_export::{PhysicalASAPDAG, PhysicalASAPNodeId}; +//! One realized selection as a physical DAG. +use crate::ir::physical_export::PhysicalASAPDAG; use crate::ir::QueryRoot; - -use super::Materialization; +#[cfg(any(test, feature = "test-support"))] +use crate::physical::Lifetime; /// One CandidatePhysicalASAPDAG (Stage 2 output), derived from exactly one /// CandidateLogicalASAPDAG (Stage 1 output). It holds planning data only; @@ -15,25 +13,20 @@ pub struct PhysicalWorkloadPlan { pub id: String, /// The id of the logical candidate it implements. pub from_logical: String, - /// The timed query roots, one per query in workload order, that `dag` + /// The query roots, one per query in workload order, that `dag` /// exports. pub roots: Vec, pub dag: PhysicalASAPDAG, - /// Its materialization assignment: the option chosen for each summary - /// build (`SummaryAgg` or `SummaryTumblingWindowChain`) of `dag`, by node id, including - /// the builds left `NotMaterialized`, and each other node materialized - /// (a shared node built once per evaluation). A build one query reads - /// at ingestion time and another at query time appears once per copy. - pub materialization: BTreeMap, } impl PhysicalWorkloadPlan { - /// Whether any node is materialized: a summary build, or a shared + /// Whether any node of `dag` is materialized: a summary build, or a shared /// non-summary node built once per evaluation. #[cfg(any(test, feature = "test-support"))] pub fn materializes_any(&self) -> bool { - self.materialization - .values() - .any(|m| *m != Materialization::NotMaterialized) + self.dag + .nodes + .iter() + .any(|node| node.lifetime != Lifetime::None) } } diff --git a/crates/types/src/stages/candidates.rs b/crates/types/src/stages/candidates.rs index 8d18a64bd..04c78b7d5 100644 --- a/crates/types/src/stages/candidates.rs +++ b/crates/types/src/stages/candidates.rs @@ -1,96 +1,107 @@ -//! Implementation candidates attached to memo groups. +//! The memo and each group's candidates +//! $v = (e_v, p^{\log}_v)$ and each candidate's input requirements +//! $r^{\log}_{c,i}$ (concepts doc `memo-groups-properties-requirements.md`, +//! section 3; #1203). use std::collections::HashSet; -use std::rc::Rc; -use super::{MemoGroup, MemoGroupId}; -use crate::ir::{OperatorNode, QueryRoot}; +use super::{GroupProperties, MemoGroup, MemoGroupAsInput, MemoGroupId}; +use crate::ir::properties::summary_coverage::SummaryCoverage; +use crate::ir::schema::{SketchAlgorithm, SketchParams}; +use crate::ir::ASAPOp; +use crate::ir::Operator; +use crate::ir::QueryRoot; +use crate::ir::SchemaDerivationError; -/// The memo: the memo groups, each query's root over them, and each group's -/// candidate implementations. +/// One candidate: position `index_in_group` in group `group`'s candidates +/// ([`MemoGroup::candidates`]). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct CandidateForMemoGroupId { + pub group: MemoGroupId, + pub index_in_group: usize, +} + +/// The memo, Stage 0's and Stage 1's one type: the memo groups, each with +/// its candidates ([`MemoGroup::candidates`]: empty after Stage 0), and each +/// query's root over them. #[derive(Debug, Clone, Default)] pub struct Memo { - pub groups: Vec, - /// Each query's root over groups, by index in the workload (the - /// initial logical memo's query roots); groups added later are never a - /// query's root. + /// The groups, by id ([`Self::groups`]); added only through + /// [`Self::find_or_add_group`]. + groups: Vec, + /// Each query's root over groups, by index in the workload. pub query_roots: Vec>, - /// Each group's implementation candidates, by group index. - pub candidates: Vec>, } -/// One way to implement a group: its node as Stage 2 implements it, over -/// child groups. -#[derive(Debug, Clone)] -pub struct CandidateForMemoGroup { - /// The groups a fused computation also implements, beyond the group - /// whose candidate it is (its owner, implicit from its position in - /// [`Memo::candidates`]); empty otherwise. - pub fused_groups: Vec, - /// The implementation's own node(s): its root node, whose descendants - /// down to [`Self::input_groups`] are its private nodes. - pub root_node: Rc, - /// The groups it reads, whose implementations are chosen separately. - pub input_groups: Vec, - /// How it was made: by Pass 1, or by which Pass 2 transformation. - pub origin: OriginOfCandidateForMemoGroup, - /// The number of equal shares of its measures' accuracy budget its - /// summaries are sized for: 1 for the whole budget, `n` when it is one - /// of `n` approximate inputs of a query (a ratio's operand). This positive - /// generation parameter is already encoded in concrete sketch parameters; - /// it is not an equality constraint on the selected number of readers or - /// approximate inputs. Recompose and validate accuracy at query roots. - /// Larger allocations may be valid but costlier than necessary. - pub accuracy_budget_parts: usize, -} +impl Memo { + /// The groups, by id. + pub fn groups(&self) -> &[MemoGroup] { + &self.groups + } -/// Where an implementation candidate came from. -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum OriginOfCandidateForMemoGroup { - /// Pass 1: exact, exact accumulator, or a summary family and size. - Pass1, - /// One build serving several reader groups (summary capability). - SharedBuild, - /// A summary readout factored from a composite implementation. - SummaryResult, - /// Canonical value preprocessing feeding a summary build. - SummaryInput, - /// Tumbling windows, or the newest tumbling windows of another group's chain. - TumblingWindows, - /// A coarse group regrouping a finer group. - Rollup, - /// A top-k read from another group's ranking. - TopKReuse, - /// Its metric's scans widened to a shared label set, read through a - /// `Project` (scan widening). - WidenedScans, -} + /// Adds a query whose root is `root` and returns its index: the root's + /// groups and every group below them list it + /// ([`MemoGroup::reached_by_queries`]). Stage 0 adds each query once. + pub fn add_query(&mut self, root: QueryRoot) -> usize { + let _ = root; + todo!("#1207 PR 2") + } + + /// Hash-consing: the group with this expression, added if new, with the + /// `GroupProperties` [`derive_group_properties`] derives. Fails when the + /// expression breaks the node rules (schema derivation, input + /// validation, merge check). + pub fn find_or_add_group( + &mut self, + expression: Operator, + ) -> Result { + let _group_properties = derive_group_properties(self, &expression)?; + todo!("#1207 PR 2: hash-consing over groups") + } + + /// Adds `candidate` to `group`'s candidates. Fails when its expression + /// breaks the node rules ([`derive_group_properties`]) or its output + /// schema or kind differs from the group's. + pub fn add_candidate( + &mut self, + group: MemoGroupId, + candidate: CandidateForMemoGroup, + ) -> Result { + let _output = derive_group_properties(self, &candidate.expression.operator) + .map_err(CandidateError::Schema)?; + let _ = group; + todo!("#1207 PR 2") + } + + /// The candidate `id`. + pub fn candidate(&self, id: CandidateForMemoGroupId) -> &CandidateForMemoGroup { + let _ = id; + todo!("#1207 PR 2") + } -impl Memo { /// The groups, each after every group it or one of its candidates /// reads, the lower index first among those ready: a dependency order for - /// candidate traversal, whatever order transformations appended - /// groups in. `None` when those reads form a cycle, a group reading - /// itself included. + /// candidate traversal, whatever order rules appended groups in. `None` + /// when those reads form a cycle, a group reading itself included. pub fn topological_order(&self) -> Option> { let count = self.groups.len(); let mut reads: Vec> = vec![HashSet::new(); count]; for (g, group) in self.groups.iter().enumerate() { - let candidates = self.candidates.get(g).map_or(&[][..], Vec::as_slice); - for child in group - .child_groups - .iter() - .chain(candidates.iter().flat_map(|c| &c.input_groups)) - { + for child in group.expression.children().into_iter().chain( + group + .candidates + .iter() + .flat_map(|c| c.expression.operator.children()), + ) { // A group reading itself never becomes ready: a cycle. reads[g].insert(child.0 as usize); } } - let mut readers: Vec> = vec![Vec::new(); count]; + let mut consumers: Vec> = vec![Vec::new(); count]; let mut waiting: Vec = reads.iter().map(HashSet::len).collect(); for (g, read) in reads.iter().enumerate() { for &child in read { - readers[child].push(g); + consumers[child].push(g); } } let mut ready: std::collections::BinaryHeap> = (0..count) @@ -100,13 +111,222 @@ impl Memo { let mut order = Vec::with_capacity(count); while let Some(std::cmp::Reverse(g)) = ready.pop() { order.push(MemoGroupId(g as u32)); - for &reader in &readers[g] { - waiting[reader] -= 1; - if waiting[reader] == 0 { - ready.push(std::cmp::Reverse(reader)); + for &consumer in &consumers[g] { + waiting[consumer] -= 1; + if waiting[consumer] == 0 { + ready.push(std::cmp::Reverse(consumer)); } } } (order.len() == count).then_some(order) } } + +/// The properties of a group, or a candidate, computing `expression` over +/// its input groups: the rules that check operator nodes, reading each +/// input group's `group_properties` and `expression` +/// ([`MemoGroupAsInput`]), never a node tree. Fails when the expression +/// breaks them (schema derivation, input validation, merge check). +fn derive_group_properties( + memo: &Memo, + expression: &Operator, +) -> Result { + let operator = expression.map_children(|&group_id| MemoGroupAsInput { memo, group_id }); + let schema = operator.output_schema()?; + operator.validate_inputs()?; + let summary_coverage = match SummaryCoverage::derive_from_inputs(&operator) { + Ok(coverage) => Some(coverage), + // A merge is valid only over states of one definition it may merge. + Err(error) if matches!(operator, Operator::ASAP(ASAPOp::SummaryMerge { .. })) => { + return Err(error.into()) + } + Err(_) => None, + }; + Ok(GroupProperties { + schema, + result_kind: operator.output_kind(), + summary_coverage, + shareable_when_equal: operator.is_shareable_when_equal(), + }) +} + +/// A candidate [`Memo::add_candidate`] rejects; the memo is unchanged. +#[derive(Debug, thiserror::Error)] +pub enum CandidateError { + /// The candidate's expression fails the node rules (schema derivation, + /// input validation, merge check): a bug in the rule that built it. + #[error("the candidate's expression breaks the node rules: {0}")] + Schema(SchemaDerivationError), + /// Its output schema or output kind differs from the group's: it does + /// not compute the group's result (a bug in the rule that built it). + #[error("the candidate's output differs from group {group:?}'s")] + OutputDiffersFromGroup { group: MemoGroupId }, +} + +/// $v = (e_v, p^{\log}_v)$: one candidate of a memo group. +#[derive(Debug, Clone)] +pub struct CandidateForMemoGroup { + /// $e_v$. + pub expression: CandidateExpression, + /// $p^{\log}_v$. + pub logical_properties: LogicalProperties, + /// $r^{\log}_{c,i}$, one per input group: per child of + /// `expression.operator`, in [`Operator::children`] order. + pub input_requirements: Vec, + /// The groups a fused computation also implements, beyond the group + /// whose candidate it is; empty otherwise. + pub fused_groups: Vec, + /// Which rule added it. Not in the formulation: kept for + /// `stage_pipeline --text` and the viewer (approved 2026-11-05). + pub origin: OriginOfCandidateForMemoGroup, +} + +/// Which rule added a candidate. Nothing in planning reads it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum OriginOfCandidateForMemoGroup { + /// Pass 1: exact, exact accumulator, or a summary family. + Pass1, + /// Canonical value preprocessing feeding a summary build. + SummaryInput, + /// A tumbling-window layout of a summary state group. + TumblingWindows, + /// A coarse group regrouping a finer group. + Rollup, + /// A top-k read from another group's ranking. + TopKReuse, + /// Its metric's scans widened to a shared label set, read through a + /// `Project` (scan widening). + WidenedScans, +} + +/// $e_v = \operatorname{op}_v(G_{v,1}, \dots, G_{v,n})$: one operator, with +/// its arguments, whose children are groups. The input groups $G_{v,i}$ are +/// [`Operator::children`], in that order. +#[derive(Debug, Clone, PartialEq)] +pub struct CandidateExpression { + pub operator: Operator, +} + +/// $p^{\log}_v = (\boldsymbol\theta_v, \omega_v)$. The family is the +/// operator's argument, not a property. [`Self::new`] rejects parameters of +/// another family, not in the family's list, missing on a sketch build or +/// present on an exact operator, and a window on an operator that cannot be +/// windowed. +#[derive(Debug, Clone)] +pub struct LogicalProperties { + /// $\boldsymbol\theta_v$, from the family's predefined list; `None` for + /// an exact operator. + parameters: Option, + /// $\omega_v$. + window_layout: Option, +} + +impl LogicalProperties { + pub fn new( + expression: &CandidateExpression, + parameters: Option, + window_layout: Option, + ) -> Result { + let _ = (expression, parameters, window_layout); + todo!("#1207 PR 1: reject invalid parameters and invalid windows") + } + + /// $\boldsymbol\theta_v$. + pub fn parameters(&self) -> Option<&SketchParams> { + self.parameters.as_ref() + } + + /// $\omega_v$. + pub fn window_layout(&self) -> Option<&WindowLayout> { + self.window_layout.as_ref() + } +} + +/// $\omega_v$. The window count is the group's query range ÷ width; +/// retention is $h_v$ (Stage 3). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum WindowLayout { + Tumbling { width_ms: u64 }, +} + +/// Logical properties whose dimensions cannot go together: bad input to +/// [`LogicalProperties::new`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +pub enum LogicalPropertiesError { + #[error("the parameters belong to another family than the operator's")] + ParametersOfAnotherFamily, + #[error("an exact operator has no summary parameters")] + ParametersOnExactOperator, + #[error("the operator cannot be windowed")] + WindowOnOperatorThatCannotBeWindowed, + #[error("a tumbling window has a positive width")] + ZeroWindowWidth, + #[error("a sketch build needs parameters from its family's list")] + SketchBuildWithoutParameters, + #[error("the parameters are not in their family's predefined list")] + ParametersNotInList, +} + +/// The family's predefined $\boldsymbol\theta$ list (#1207 step 6), each with +/// its error bound derived through `estimators::local_guarantee` for a given +/// statistic. +pub fn parameter_list(family: SketchAlgorithm) -> &'static [SketchParams] { + let _ = family; + todo!("#1207 PR 1: the predefined θ list per family") +} + +/// $r^{\log}_{c,i}$: only what $G_{c,i}$'s identity does not fix. +#[derive(Debug, Clone)] +pub struct LogicalRequirements { + /// $\boldsymbol\theta^{\min}_{c,i}$: the weakest parameters whose + /// $\varepsilon$, for the statistic the consumer reads, is at most + /// $\text{cap}_{c,i}$ = min(validity cap of $g_c$, pruning cap), or a + /// direct need such as heap_size ≥ k. Any value of the family's parameter + /// type; it need not be in the family's list. + parameter_floor_for_required_accuracy: Option, +} + +impl LogicalRequirements { + pub fn new(parameter_floor_for_required_accuracy: Option) -> Self { + Self { + parameter_floor_for_required_accuracy, + } + } + + /// $\boldsymbol\theta^{\min}_{c,i}$. + pub fn parameter_floor_for_required_accuracy(&self) -> Option<&SketchParams> { + self.parameter_floor_for_required_accuracy.as_ref() + } + + /// $g_c$'s linear bound: $\beta_{c,i}$ and $b_{c,i}$ per input, read + /// from the operator and its arguments. An operator without an error + /// contract is reported as unsupported; no factor is guessed. + pub fn error_propagation(consumer: &Operator) -> ErrorPropagation { + let _ = consumer; + todo!("#1207 PR 4: the error contract of each operator") + } +} + +/// The local error propagation of one consumer: its output error is at +/// most $\sum_i \beta_{c,i}\varepsilon_{c,i}$ while each input's error stays +/// within its validity cap (concepts doc, section 4; approved 2026-11-05). +#[derive(Debug, Clone, PartialEq)] +pub struct ErrorPropagation { + /// $\beta_{c,i}$ per input, for that input's metric. + pub factor_per_input: Vec, + /// $b_{c,i}$ per input: the cap under which the linear bound holds; + /// `None` when the bound holds everywhere (e.g. a ratio caps only its + /// denominator). + pub validity_cap_per_input: Vec>, +} + +/// $\alpha^{\log}_{u,c,i}$: $\boldsymbol\theta_u \succeq +/// \boldsymbol\theta^{\min}_{c,i}$, compared per family on the parameters +/// that set $\varepsilon$. No floor, or an exact producer, always satisfies. +pub fn producer_satisfies_logical_requirements( + producer: &LogicalProperties, + requirement: &LogicalRequirements, +) -> bool { + let _ = (producer, requirement); + todo!("#1207 PR 1: per-family dominance") +} diff --git a/crates/types/src/stages/logical_memo_candidates.rs b/crates/types/src/stages/logical_memo_candidates.rs index d9b4899c8..9a313ec76 100644 --- a/crates/types/src/stages/logical_memo_candidates.rs +++ b/crates/types/src/stages/logical_memo_candidates.rs @@ -1,14 +1,25 @@ -//! Stage 1, Pass 2's output, which Stage 2 implements. +//! Stage 1, Pass 2's output, which Stage 2 expands: the memo after every +//! rule, and the logical edges $(u, c, i)$ with $\alpha^{\log}$ (#1201). -use super::Memo; +use super::{CandidateForMemoGroupId, Memo}; -/// Stage 1, Pass 2's output: the Pass 1 groups plus the groups Pass 2 adds -/// (shared producers, shared summary states), each with its Pass 1 -/// candidates plus every Pass 2 rule's (by `origin`), including the -/// candidates reading a shared summary state. +/// $(u, c, i)$ over logical candidates: `producer_candidate_id` is a +/// candidate of $G_{c,i}$, the group input `input_index_of_consumer` of +/// `consumer_candidate_id` reads. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct LogicalEdge { + pub producer_candidate_id: CandidateForMemoGroupId, + pub consumer_candidate_id: CandidateForMemoGroupId, + pub input_index_of_consumer: usize, + /// $\alpha^{\log}_{u,c,i}$. + pub satisfies_logical_requirement: bool, +} + +/// Stage 1, Pass 2's output. #[derive(Debug, Clone)] pub struct LogicalASAPMemoCandidates { - /// The expanded memo: groups, query roots (passed through from Pass 1) - /// and each group's candidates. + /// The memo after the last rule. pub memo: Memo, + /// Every $(u, c, i)$ with $u \in G_{c,i}$, grouped by $(c, i)$. + pub edges: Vec, } diff --git a/crates/types/src/stages/memo.rs b/crates/types/src/stages/memo.rs index 765ef09ca..153346093 100644 --- a/crates/types/src/stages/memo.rs +++ b/crates/types/src/stages/memo.rs @@ -1,93 +1,97 @@ -//! Memo groups: the distinct logical sub-expressions of the workload. +//! Memo groups: the distinct logical sub-expressions of the workload, each +//! with its candidates. -use std::collections::{BTreeSet, HashMap}; -use std::rc::Rc; +use std::collections::BTreeSet; -use crate::ir::{OperatorNode, QueryRoot}; +use super::{CandidateForMemoGroup, Memo}; +use crate::ir::operator::input::OperatorInput; +use crate::ir::properties::summary_coverage::SummaryCoverage; +use crate::ir::schema::Schema; +use crate::ir::{Operator, OperatorResultKind, QueryRoot}; /// A group, by index in the memo's groups. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] pub struct MemoGroupId(pub u32); -/// One distinct logical sub-expression of the workload. +/// One distinct logical sub-expression of the workload: what it computes +/// and the output every candidate of it must produce. #[derive(Debug, Clone)] pub struct MemoGroup { - /// Its logical node as the frontends (and Pass 1 targets) see it; its - /// children are other groups' nodes. - pub logical_node: Rc, - /// Its child groups, in the node's child order. - pub child_groups: Vec, - /// The groups that read it ([`Self::parent_groups`]). Public only so - /// Stage 1, which builds and extends the memo, can maintain it; later - /// stages read it through [`Self::parent_groups`]. + /// What it computes: one operator, with its arguments. Each + /// `MemoGroupId` in it is an input group, never this group's own id. θ + /// is no argument of an operator, so it is not part of a group. + pub expression: Operator, + /// Derived once when the group is created, from its input groups; never + /// changes. + pub group_properties: GroupProperties, + /// The groups that read it ([`Self::parent_groups`]) through their + /// expression or a candidate's. pub parents: Vec, - /// The queries whose roots reach it ([`Self::reached_by_queries`]). - /// Public only so Stage 1, which builds and extends the memo, can - /// maintain it; later stages read it through - /// [`Self::reached_by_queries`]. + /// The queries whose roots reach it ([`Self::reached_by_queries`]), + /// ascending. pub queries: Vec, + /// Its candidates: empty after Stage 0, filled by Stage 1. + pub candidates: Vec, } -impl MemoGroup { - /// The groups that read it. - pub fn parent_groups(&self) -> &[MemoGroupId] { - &self.parents - } +/// What every candidate of a group produces, derived from the group's +/// expression and its input groups' properties by the rules that check +/// operator nodes. +#[derive(Debug, Clone)] +pub struct GroupProperties { + pub schema: Schema, + pub result_kind: OperatorResultKind, + /// What a summary-state group covers; `None` for any other group. + pub summary_coverage: Option, + /// Whether two equal expressions return the same rows for the same + /// input rows, so they may be one group (not an unordered `LIMIT`). + pub shareable_when_equal: bool, +} - /// The queries whose roots reach it, by index in the workload, ascending. - pub fn reached_by_queries(&self) -> &[usize] { - &self.queries - } +/// A memo group read as an operator's input: its `group_properties` and +/// its `expression`, whose inputs are read the same way. +#[derive(Clone, Copy)] +pub(crate) struct MemoGroupAsInput<'m> { + pub(crate) memo: &'m Memo, + pub(crate) group_id: MemoGroupId, } -/// Stage 0's output, Pass 1's input: the workload's memo groups, one per -/// distinct logical sub-expression, without implementation choices or -/// solver state. Stage 1 builds it by hash-consing the query roots. -#[derive(Debug, Clone)] -pub struct DagOfLogicalMemoGroups { - pub groups: Vec, - /// Each query's root over groups, by index in the workload: its - /// operator's group, or its scalar expression with every operator - /// reference replaced by that operator's group. - pub query_roots: Vec>, - /// Each node of the original workload, by address, to its group. - group_of: HashMap<*const OperatorNode, MemoGroupId>, +impl<'m> MemoGroupAsInput<'m> { + fn group(&self) -> &'m MemoGroup { + &self.memo.groups()[self.group_id.0 as usize] + } } -impl DagOfLogicalMemoGroups { - /// The memo groups `groups` (children before parents), each query's - /// root over them, and `group_of`: each node of the original workload, - /// by address, to its group. - pub fn new( - groups: Vec, - query_roots: Vec>, - group_of: HashMap<*const OperatorNode, MemoGroupId>, - ) -> Self { - Self { - groups, - query_roots, - group_of, - } +impl OperatorInput for MemoGroupAsInput<'_> { + fn schema(&self) -> &Schema { + &self.group().group_properties.schema } - /// Group of a node from the original workload, not a rewritten candidate. - pub fn group_for_node(&self, node: &Rc) -> Option { - self.group_of.get(&Rc::as_ptr(node)).copied() + fn result_kind(&self) -> OperatorResultKind { + self.group().group_properties.result_kind } - /// Each query's root over the groups' logical nodes, in workload order: - /// a group several queries read is one node. - pub fn roots_over_groups(&self) -> Vec { - let node = |g: &MemoGroupId| Rc::clone(&self.groups[g.0 as usize].logical_node); - self.query_roots - .iter() - .map(|root| match root { - QueryRoot::Operator(g) => QueryRoot::Operator(node(g)), - QueryRoot::Scalar(expr) => { - QueryRoot::Scalar(expr.map_operator_refs(&mut |g| node(g))) - } - }) - .collect() + fn operator(&self) -> Operator { + self.group().expression.map_children(|&group_id| Self { + memo: self.memo, + group_id, + }) + } + + fn summary_coverage(&self) -> Option<&SummaryCoverage> { + self.group().group_properties.summary_coverage.as_ref() + } +} + +impl MemoGroup { + /// The groups that read it. + pub fn parent_groups(&self) -> &[MemoGroupId] { + &self.parents + } + + /// The queries whose roots reach it, by index in the workload, ascending. + pub fn reached_by_queries(&self) -> &[usize] { + &self.queries } } diff --git a/crates/types/src/stages/mod.rs b/crates/types/src/stages/mod.rs index 109ce200e..596aee4f4 100644 --- a/crates/types/src/stages/mod.rs +++ b/crates/types/src/stages/mod.rs @@ -1,7 +1,6 @@ -//! The Stage 0 → Stage 1 boundary: the workload's memo groups -//! ([`DagOfLogicalMemoGroups`]). Inside Stage 1, Pass 1 → Pass 2: the memo -//! groups with their Pass 1 candidates -//! ([`PerComputationLogicalMemoCandidates`]). The Stage 1 → Stage 2 boundary: the memo of logical groups and their +//! The Stage 0 → Stage 1 boundary: the workload's memo groups, without +//! candidates ([`Memo`]). Inside Stage 1, Pass 1 → Pass 2: the memo with +//! its Pass 1 candidates. The Stage 1 → Stage 2 boundary: the memo of logical groups and their //! implementation candidates that Stage 1 (Pass 1 and Pass 2) produces and //! Stage 2 implements (#1047, #1054). The Stage 2 → Stage 3 boundary: the //! physical candidates Stage 2 builds and Stage 3 selects from. Stage @@ -16,20 +15,22 @@ mod physical; mod pruning; mod realizer; -pub use candidates::{CandidateForMemoGroup, Memo, OriginOfCandidateForMemoGroup}; -pub use logical_memo_candidates::LogicalASAPMemoCandidates; -pub use memo::{queries_rooted_at, root_groups, DagOfLogicalMemoGroups, MemoGroup, MemoGroupId}; +pub use candidates::{ + parameter_list, producer_satisfies_logical_requirements, CandidateError, CandidateExpression, + CandidateForMemoGroup, CandidateForMemoGroupId, ErrorPropagation, LogicalProperties, + LogicalPropertiesError, LogicalRequirements, Memo, OriginOfCandidateForMemoGroup, WindowLayout, +}; +pub use logical_memo_candidates::{LogicalASAPMemoCandidates, LogicalEdge}; +pub(crate) use memo::MemoGroupAsInput; +pub use memo::{queries_rooted_at, root_groups, GroupProperties, MemoGroup, MemoGroupId}; pub use per_computation_logical_memo_candidates::{ - ComputationTarget, FusedComputation, HavingEntry, HavingPlacement, - HowToComputeAggregateMeasure, PerComputationLogicalMemoCandidates, + ComputationTarget, FusedComputation, HavingEntry, HavingPlacement, HowToComputeAggregateMeasure, }; pub use physical::{ - output_references, CandidateForMemoGroupId, DagOfSelectedCandidates, - OperatorNodeOfCandidateForMemoGroup, OperatorNodeOfCandidateForMemoGroupId, - PhysicalASAPMemoCandidates, PhysicalCandidate, PhysicalCandidateId, PhysicalCandidateKey, - ProducerCandidates, ProducerSetInterner, ReaderConstraint, SelectedEdge, SelectedNodeId, + output_references, producer_satisfies_physical_requirements, DagOfSelectedCandidates, + ExecutionContext, ExecutionContextId, PhysicalASAPMemoCandidates, PhysicalCandidate, + PhysicalCandidateId, PhysicalCandidateKey, PhysicalEdge, PhysicalEdgeId, RefreshWindowId, + SelectedEdge, SelectedNodeId, }; -#[cfg(any(test, feature = "test-support"))] -pub use physical::{producer_storage_metrics, ProducerStorageMetrics}; pub use pruning::{PruneReason, PrunedCandidate}; pub use realizer::{AssignmentRealizer, PhysicalCandidateError, RealizedPlan}; diff --git a/crates/types/src/stages/per_computation_logical_memo_candidates.rs b/crates/types/src/stages/per_computation_logical_memo_candidates.rs index 307f4bdc4..137360946 100644 --- a/crates/types/src/stages/per_computation_logical_memo_candidates.rs +++ b/crates/types/src/stages/per_computation_logical_memo_candidates.rs @@ -1,27 +1,11 @@ -//! Stage 1, Pass 1's output, which Pass 2 extends (the memo groups with -//! their Pass 1 candidates), and the computation-target types Pass 1 builds -//! those candidates from. +//! The computation-target types Stage 1, Pass 1 builds its candidates from. use std::rc::Rc; -use super::{Memo, MemoGroupId, PrunedCandidate}; +use super::MemoGroupId; use crate::ir::schema::{ExactKind, SketchAlgorithm}; use crate::ir::{OperatorNode, Predicate}; -/// Stage 1, Pass 1's output: the workload's memo groups, each with its -/// Pass 1 candidates, and the candidates Pass 1 pruned. It is also Pass 2's -/// input. -#[derive(Debug, Clone)] -pub struct PerComputationLogicalMemoCandidates { - /// The groups moved from the [`DagOfLogicalMemoGroups`](super::DagOfLogicalMemoGroups), - /// its query roots, and every group's candidates (origin - /// [`Pass1`](super::OriginOfCandidateForMemoGroup::Pass1)). - pub memo: Memo, - /// The ways to compute a target's measure that Pass 1 pruned, each on - /// its target's aggregate group, in pruning order. - pub pruned_candidates: Vec>, -} - /// How one Pass 1 target is computed: exact or a summary. /// /// Each target has exactly one exact option: `ExactAccumulator` when the @@ -84,8 +68,6 @@ pub struct ComputationTarget { /// Index into the aggregate's measures. pub measure_index: usize, /// The measure's unranked ways to compute it, the exact option first. - /// The ones Pass 1 pruned are in - /// [`PerComputationLogicalMemoCandidates::pruned_candidates`]. pub ways_to_compute: Vec, /// The measure's own filter; only rows that pass it update the state /// (`SummaryAgg.filter`). @@ -114,10 +96,8 @@ pub struct FusedComputation { /// computations absorb. None of them is computed when this is picked. pub absorbed_targets: Vec, /// How the fused computation is computed: exact or a summary. Empty - /// when Pass 1 pruned every one (today only for - /// [`PruneReason::UnsupportedByDeployment`](super::PruneReason::UnsupportedByDeployment), - /// recorded in [`PerComputationLogicalMemoCandidates::pruned_candidates`]); the fusion is - /// then not offered. + /// when Pass 1 pruned every one (today only for a family the deployment + /// does not support); the fusion is then not offered. pub ways_to_compute: Vec, } diff --git a/crates/types/src/stages/physical/execution_context.rs b/crates/types/src/stages/physical/execution_context.rs new file mode 100644 index 000000000..0ab1630e8 --- /dev/null +++ b/crates/types/src/stages/physical/execution_context.rs @@ -0,0 +1,36 @@ +//! $\kappa$: where a candidate with $\ell$ = None or `OneEvaluation` runs +//! (concepts doc, section 5). Stage 3 indexes $x_{\kappa,v}$ and +//! $y_{\kappa,u,c,i}$ by it; [`super::PhysicalCandidate::delivers_on`] reads +//! it, so it lives with the physical candidates. + +use super::graph::PhysicalCandidateId; +use crate::workload::{QueryRecurrence, RangeEvaluation}; + +/// A refresh window of the workload's ingestion pipelines, by index. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct RefreshWindowId(pub usize); + +/// Index of an [`ExecutionContext`] in the workload's list of distinct +/// contexts. Contexts are deduplicated by equality: [`QueryRecurrence`] +/// holds an `f64` (`EstimatedRate`), so a context cannot be hashed. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct ExecutionContextId(pub usize); + +/// $\kappa$: a class of executions with a common owner and schedule, where +/// a candidate with $\ell$ = None or `OneEvaluation` runs. +#[derive(Debug, Clone, PartialEq)] +pub enum ExecutionContext { + /// The evaluation of every query with this recurrence and range + /// evaluation, the workload's own description of when it runs: queries + /// with equal ones are evaluated together and share its `OneEvaluation` + /// results. A and B both every 10 s at phase 0 give one context; a + /// one-off query's context is its `OneTime` recurrence. + Evaluation { + recurrence: QueryRecurrence, + range_evaluation: Option, + }, + /// $I_r$: the ingestion pipeline of a refresh window. + IngestionPipeline(RefreshWindowId), + /// $\kappa_m$: inside an `AcrossEvaluations` candidate's own computation. + Materialized(PhysicalCandidateId), +} diff --git a/crates/types/src/stages/physical/graph.rs b/crates/types/src/stages/physical/graph.rs index c4537eaa4..6e886c505 100644 --- a/crates/types/src/stages/physical/graph.rs +++ b/crates/types/src/stages/physical/graph.rs @@ -1,72 +1,12 @@ //! Selections over Stage 2's physical candidates, and their structural //! validation. use super::memo_candidates::PhysicalASAPMemoCandidates; -use super::producers::ProducerCandidates; -use std::collections::{HashMap, HashSet}; +use std::collections::HashSet; /// Index into [`PhysicalASAPMemoCandidates::candidates`](super::PhysicalASAPMemoCandidates::candidates). #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] pub struct PhysicalCandidateId(pub usize); -/// A rule on which output references may read a candidate, applied to each -/// node of a [`DagOfSelectedCandidates`] that runs it: the output references -/// that reach the node are its readers. Output references -/// index the group references of the query roots -/// ([`output_references`](super::output_references)); several references of -/// one query stay distinct. -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub enum ReaderConstraint { - /// Each node is read by at least `minimum` queries. Each bucket of - /// `query_outputs` is one query's output references; together they - /// partition every output exactly once. Empty buckets represent - /// literal-only queries. - MinimumQueryReaders { - candidate: PhysicalCandidateId, - query_outputs: Vec>, - minimum: usize, - }, - /// A non-summary query materialization: each node produces an - /// output reference, or has an immediate selected consumer reached by - /// at most one query (`query_outputs` as in `MinimumQueryReaders`). - HighestSharedQueryMaterialization { - candidate: PhysicalCandidateId, - query_outputs: Vec>, - }, - /// Only `outputs` may read a node. - AllowedOutputs { - candidate: PhysicalCandidateId, - outputs: Vec, - }, - /// No node is read by both `left` and `right`. - IncompatibleOutputs { - candidate: PhysicalCandidateId, - left: usize, - right: usize, - }, -} - -impl ReaderConstraint { - /// The candidate it constrains. - pub fn candidate(&self) -> PhysicalCandidateId { - match self { - Self::MinimumQueryReaders { candidate, .. } - | Self::HighestSharedQueryMaterialization { candidate, .. } - | Self::AllowedOutputs { candidate, .. } - | Self::IncompatibleOutputs { candidate, .. } => *candidate, - } - } - - /// The candidate it constrains, to renumber. - pub fn candidate_mut(&mut self) -> &mut PhysicalCandidateId { - match self { - Self::MinimumQueryReaders { candidate, .. } - | Self::HighestSharedQueryMaterialization { candidate, .. } - | Self::AllowedOutputs { candidate, .. } - | Self::IncompatibleOutputs { candidate, .. } => candidate, - } - } -} - /// Index into [`DagOfSelectedCandidates::candidate_of_node`]: one run of a /// candidate. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] @@ -78,15 +18,15 @@ pub struct SelectedNodeId(pub usize); pub struct SelectedEdge { pub producer: SelectedNodeId, pub consumer: SelectedNodeId, - /// Index in the consumer's inputs (its candidate's - /// `producer_candidates_per_input`). - pub input: usize, + /// Index in the consumer's inputs: its logical candidate's + /// [`Operator::children`](crate::ir::operator::Operator::children). + pub input_index_of_consumer: usize, } /// The selected plan as a DAG with one node per run of a candidate: what /// Stage 3 selects and the facade builds through /// [`AssignmentRealizer::realize`](crate::stages::AssignmentRealizer::realize). -/// A candidate that runs twice (recomputed by two readers, or two shared +/// A candidate that runs twice (recomputed by two consumers, or two shared /// instances over different inputs) is two nodes. #[derive(Debug, Clone, PartialEq)] pub struct DagOfSelectedCandidates { @@ -102,47 +42,63 @@ pub struct DagOfSelectedCandidates { impl DagOfSelectedCandidates { /// Structural check against `physical`, instead of trusting rounded /// solver values: every node runs an existing candidate, every input of - /// every node has exactly one edge from a node of an allowed producer - /// candidate, the graph is acyclic, every node is read by some output - /// reference, each output reference comes from one of its - /// `query_output_candidates`, and every node keeps its candidate's - /// reader constraints. Costs and resource budgets are Stage 3's to - /// check. + /// every node has exactly one edge from a node of a producer on one of + /// its admissible edges, the graph is acyclic, every node is reached by + /// some output reference, each output reference comes from a candidate + /// of its group. Costs and accuracy are Stage 3's to check. pub fn validate(&self, physical: &PhysicalASAPMemoCandidates) -> Result<(), String> { - let query_output_candidates: Vec> = (0..physical.output_reference_count()) - .map(|output| physical.query_output_candidates(output).collect()) + let memo = &physical.logical_candidates.memo; + let output_groups: Vec<_> = super::output_references(&physical.query_roots) + .into_iter() + .map(|(_, group)| group) + .collect(); + let query_output_candidates: Vec> = output_groups + .iter() + .map(|&group| { + (0..physical.physical_candidates.len()) + .map(PhysicalCandidateId) + .filter(|id| { + physical.physical_candidates[id.0] + .logical_candidate_id + .group + == group + }) + .collect() + }) .collect(); let query_output_candidates: Vec<&[_]> = query_output_candidates.iter().map(Vec::as_slice).collect(); + let edges: Vec<_> = physical.edges().collect(); self.validate_against( - physical.candidates.len(), + physical.physical_candidates.len(), |candidate| { - physical.candidates[candidate.0] - .producer_candidates_per_input + memo.candidate(physical.physical_candidates[candidate.0].logical_candidate_id) + .expression + .operator + .children() .len() }, - |candidate, input| { - physical.candidates[candidate.0] - .producer_candidates_per_input - .get(input) + |candidate, input_index_of_consumer, producer| { + physical + .admissible_edges_into(candidate, input_index_of_consumer) + .iter() + .any(|edge| edges[edge.0].1.producer_candidate_id == producer) }, &query_output_candidates, - &physical.reader_constraints, ) } /// [`Self::validate`] over a catalog given by its parts: /// `candidate_count` candidates; `input_count(c)` inputs of candidate - /// `c`; `producers_of_input(c, i)` the candidates that may produce its - /// input `i` (`None` past its last input); `query_output_candidates[r]` - /// the candidates that may produce output reference `r`. - fn validate_against<'a>( + /// `c`; `admissible(c, i, u)` whether an admissible edge `(u, c, i)` + /// exists; `query_output_candidates[r]` the candidates that may produce + /// output reference `r`. + fn validate_against( &self, candidate_count: usize, input_count: impl Fn(PhysicalCandidateId) -> usize, - producers_of_input: impl Fn(PhysicalCandidateId, usize) -> Option<&'a ProducerCandidates>, + admissible: impl Fn(PhysicalCandidateId, usize, PhysicalCandidateId) -> bool, query_output_candidates: &[&[PhysicalCandidateId]], - reader_constraints: &[ReaderConstraint], ) -> Result<(), String> { let invalid = || "decoded candidate selection violates dependency or root constraints".to_string(); @@ -176,15 +132,15 @@ impl DagOfSelectedCandidates { if producer >= count || consumer >= count { return Err(invalid()); } - let Some(producer_candidates) = - producers_of_input(self.candidate_of_node[consumer], edge.input) - else { + let Some(target) = producers[consumer].get_mut(edge.input_index_of_consumer) else { return Err(invalid()); }; - let Some(target) = producers[consumer].get_mut(edge.input) else { - return Err(invalid()); - }; - if target.is_some() || !producer_candidates.contains(&self.candidate_of_node[producer]) + if target.is_some() + || !admissible( + self.candidate_of_node[consumer], + edge.input_index_of_consumer, + self.candidate_of_node[producer], + ) { return Err(invalid()); } @@ -228,54 +184,6 @@ impl DagOfSelectedCandidates { if (0..count).any(|node| !output_reach.iter().any(|reach| reach.contains(&node))) { return Err(invalid()); } - let queries_reaching = |node: usize, query_outputs: &[Vec]| { - query_outputs - .iter() - .filter(|outputs| { - outputs - .iter() - .any(|&output| output_reach[output].contains(&node)) - }) - .count() - }; - let output_nodes: HashSet = self - .node_of_output_reference - .iter() - .map(|node| node.0) - .collect(); - let mut nodes_of_candidate: HashMap> = HashMap::new(); - for (node, candidate) in self.candidate_of_node.iter().enumerate() { - nodes_of_candidate.entry(*candidate).or_default().push(node); - } - for constraint in reader_constraints { - let Some(nodes) = nodes_of_candidate.get(&constraint.candidate()) else { - continue; - }; - let violated = nodes.iter().any(|&node| match constraint { - ReaderConstraint::MinimumQueryReaders { - query_outputs, - minimum, - .. - } => queries_reaching(node, query_outputs) < *minimum, - ReaderConstraint::HighestSharedQueryMaterialization { query_outputs, .. } => { - !output_nodes.contains(&node) - && !consumers[node] - .iter() - .any(|&consumer| queries_reaching(consumer, query_outputs) <= 1) - } - ReaderConstraint::AllowedOutputs { outputs, .. } => { - (0..output_reach.len()).any(|output| { - !outputs.contains(&output) && output_reach[output].contains(&node) - }) - } - ReaderConstraint::IncompatibleOutputs { left, right, .. } => { - output_reach[*left].contains(&node) && output_reach[*right].contains(&node) - } - }); - if violated { - return Err("decoded selection violates selected-reader compatibility".into()); - } - } Ok(()) } } diff --git a/crates/types/src/stages/physical/memo_candidates.rs b/crates/types/src/stages/physical/memo_candidates.rs index 1d500fc27..5b64ece64 100644 --- a/crates/types/src/stages/physical/memo_candidates.rs +++ b/crates/types/src/stages/physical/memo_candidates.rs @@ -1,131 +1,144 @@ -//! Stage 2's output: every operator node of every implementation candidate, -//! each with the timings and materializations Stage 2 allows, and which -//! candidates may produce each input. Stage 3 selects among them; Stage 2 -//! prices nothing. -use std::rc::Rc; +//! Stage 2's output: each logical candidate's physical alternatives +//! $\mathcal C$, one per valid $p^{\mathrm{phys}} = (\tau, \ell)$, and +//! the physical edges $\mathcal E$ with $\alpha^{\log}$ and +//! $\alpha^{\mathrm{phys}}$ (#1200, #1201). Stage 3 selects among them; +//! Stage 2 prices nothing. -use super::graph::{PhysicalCandidateId, ReaderConstraint}; -use super::producers::ProducerCandidates; -use crate::ir::properties::ExecutionTiming; -use crate::ir::{OperatorNode, QueryRoot}; -use crate::physical::Materialization; -use crate::stages::{MemoGroupId, PrunedCandidate}; +use std::collections::HashMap; -/// One implementation candidate: index `index` in group `group`'s -/// candidates ([`Memo::candidates`](crate::stages::Memo::candidates)). -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub struct CandidateForMemoGroupId { - pub group: MemoGroupId, - pub index: usize, -} +use super::execution_context::ExecutionContext; +use super::graph::PhysicalCandidateId; +use crate::ir::QueryRoot; +use crate::physical::PhysicalProperties; +use crate::stages::{ + CandidateForMemoGroupId, LogicalASAPMemoCandidates, Memo, MemoGroupId, PrunedCandidate, +}; -/// Index into -/// [`PhysicalASAPMemoCandidates::operator_nodes_of_candidates_for_memo_groups`]. +/// Index into [`PhysicalASAPMemoCandidates::edges`]. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub struct OperatorNodeOfCandidateForMemoGroupId(pub usize); +pub struct PhysicalEdgeId(pub usize); -/// One operator node of one or more candidates for a memo group, before -/// Stage 2 chose its timing and materialization. -#[derive(Debug, Clone)] -pub struct OperatorNodeOfCandidateForMemoGroup { - /// Every candidate of the group this node can implement, ascending: - /// identical terminal readers of different candidates are factored into - /// one node, and the selected producers decide which one it implements. - /// A private node belongs to the one candidate it was lowered from. - pub candidates_for_memo_group: Vec, - /// Whether it is its candidates' root; false for an implementation's - /// private nodes. - pub is_implementation_root: bool, - /// How many equal shares of a query's accuracy budget this node's sketch sizes meet - /// (`CandidateForMemoGroup::accuracy_budget_parts`). For a factored node, the largest among its - /// candidates: equal nodes have equal sizes. 1 for an exact node. - pub accuracy_budget_parts: usize, - /// The operator node, with Stage 2's physical implementation; its - /// children are the expected inputs, which the selected producers replace. - pub physical_node: Rc, +/// One physical alternative of a logical candidate: what Stage 3 selects. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct PhysicalCandidate { + /// $e_v$ and $p^{\log}_v$. + pub logical_candidate_id: CandidateForMemoGroupId, + /// $p^{\mathrm{phys}}_v$. + pub physical_properties: PhysicalProperties, } -impl OperatorNodeOfCandidateForMemoGroup { - /// The memo group whose candidates this node implements. - /// `candidates_for_memo_group` is never empty, and all its entries are - /// candidates of this one group. - pub fn group(&self) -> MemoGroupId { - let group = self.candidates_for_memo_group[0].group; - debug_assert!(self - .candidates_for_memo_group - .iter() - .all(|candidate| candidate.group == group)); - group +impl PhysicalCandidate { + /// Whether a consumer running in `context` can read this producer (the + /// concepts doc's alignment table, section 2). True for $\ell$ = None, + /// and for `OneEvaluation` in the same context. A windowed + /// `AcrossEvaluations` producer, with $w$ read from its logical window + /// layout in `memo`, needs every evaluation time of the context on its + /// grid: $w$ divides the context's evaluation grid (interval, range step + /// and phase), each scheduled time, or a one-off query's time; false + /// when the times are not known (an interval without a phase, an + /// estimated rate, an unknown recurrence). + pub fn delivers_on(&self, memo: &Memo, context: &ExecutionContext) -> bool { + let _ = (memo, context); + todo!("#1207 PR 1: the alignment table of section 2") } } -/// One operator node with one timing and materialization: what Stage 3 -/// selects. A selection may run it more than once, each run a node of the -/// [`DagOfSelectedCandidates`](super::DagOfSelectedCandidates). -#[derive(Debug, Clone)] -pub struct PhysicalCandidate { - /// The operator node it runs. - pub operator_node: OperatorNodeOfCandidateForMemoGroupId, - /// When it runs. A node that is not materialized under an ingestion-time - /// build still runs at ingestion time, so both are kept. - pub timing: ExecutionTiming, - /// Whether and where its result is kept. - pub materialization: Materialization, - /// Per input of its operator node (in `map_children` order), the - /// candidates that may produce it. Depends on its timing. - pub producer_candidates_per_input: Vec, - /// Whether it may produce a query's output: an implementation root at - /// query time whose output matches its group's logical node, in a group - /// some query's root reads. - pub may_produce_query_output: bool, - /// One entry per workload query: whether this tumbling-window chain may be kept - /// across evaluations when that query is its only reader. Filled only - /// for a `MatAtQueryKeptAcrossEvaluations(Memory)` tumbling-window chain in a - /// deployment that keeps query-time results; empty otherwise. - pub may_keep_across_evaluations_per_sole_reader_query: Vec, +/// $\alpha^{\mathrm{phys}}_{u,c,i}$: timing and lifetime only (the concepts +/// doc's table, section 2). Alignment is checked per context in Stage 3 with +/// [`PhysicalCandidate::delivers_on`]. +pub fn producer_satisfies_physical_requirements( + producer: &PhysicalCandidate, + consumer: &PhysicalCandidate, +) -> bool { + let _ = (producer, consumer); + todo!("#1207 PR 1: the timing x lifetime table of section 2") +} + +/// $(u, c, i)$ over physical candidates. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct PhysicalEdge { + pub producer_candidate_id: PhysicalCandidateId, + pub consumer_candidate_id: PhysicalCandidateId, + pub input_index_of_consumer: usize, + /// $\alpha^{\log}_{u,c,i}$. + pub satisfies_logical_requirement: bool, + /// $\alpha^{\mathrm{phys}}_{u,c,i}$. + pub satisfies_physical_requirement: bool, } /// A physical candidate Stage 2 pruned, named by what it would have been: /// candidates are renumbered after pruning, so a pruned one has no -/// [`PhysicalCandidateId`]. Unique among a [`PhysicalASAPMemoCandidates`]' -/// `pruned_candidates`: an operator node has at most one candidate per -/// timing and materialization. +/// [`PhysicalCandidateId`]. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub struct PhysicalCandidateKey { - /// Its operator node: an index into the output's - /// `operator_nodes_of_candidates_for_memo_groups`, which pruning keeps - /// whole. - pub operator_node: OperatorNodeOfCandidateForMemoGroupId, - pub timing: ExecutionTiming, - pub materialization: Materialization, + pub logical_candidate_id: CandidateForMemoGroupId, + pub physical_properties: PhysicalProperties, } /// Stage 2's output, which Stage 3 selects from. #[derive(Debug, Clone)] pub struct PhysicalASAPMemoCandidates { - /// One per operator node of an implementation, before Stage 2 chose its - /// timing and materialization. - pub operator_nodes_of_candidates_for_memo_groups: Vec, - /// One per (operator node, timing, materialization): what Stage 3 - /// selects. Indexed by [`PhysicalCandidateId`]. - pub candidates: Vec, + /// Pass 2's output: the memo, its candidates' $e_v$, $p^{\log}_v$, + /// $r^{\log}_{c,i}$, and the logical edges. + pub logical_candidates: LogicalASAPMemoCandidates, + /// $\mathcal C$, indexed by [`PhysicalCandidateId`]. + pub physical_candidates: Vec, /// Each query's root over groups, by index in the workload, passed - /// through from Pass 2. Its group references, in order, are the output - /// references of [`Self::query_output_candidates`]. + /// through from Pass 2. pub query_roots: Vec>, - /// Legality that depends on which readers are selected. - pub reader_constraints: Vec, - /// The candidates Stage 2 pruned, each on its operator node's memo - /// group, in pruning order: configurations it does not generate, then - /// candidates no buildable input, query output or reader count can use. - /// Empty when built by [`Self::new`]. + /// The candidates Stage 2 pruned, in pruning order. pub pruned_candidates: Vec>, - /// By group index: the candidates whose operator node is an - /// implementation root of that group, ascending. - implementation_roots_of_group: Vec>, - /// Each output reference's group: [`output_references`] of - /// `query_roots`, built once. - output_reference_groups: Vec, + /// $\mathcal E$, grouped by (consumer, input); indexed by + /// [`PhysicalEdgeId`]. + edges: Vec, + /// $P_{c,i}$ after $\alpha$: the admissible edges into each (consumer, input). + admissible_edges_into_input: HashMap<(PhysicalCandidateId, usize), Vec>, + /// The admissible edges out of each candidate, by [`PhysicalCandidateId`]. + admissible_edges_out_of_candidate: Vec>, +} + +impl PhysicalASAPMemoCandidates { + /// Stage 2's output over `physical_candidates` and `edges` (grouped by + /// (consumer, input)), with the admissible-edge indexes built once. + pub fn new( + logical_candidates: LogicalASAPMemoCandidates, + physical_candidates: Vec, + query_roots: Vec>, + edges: Vec, + ) -> Self { + let _ = (logical_candidates, physical_candidates, query_roots, edges); + todo!("#1207 PR 5: index the admissible edges") + } + + /// $\mathcal E$: every edge, in (consumer, input) order. + pub fn edges(&self) -> impl Iterator + '_ { + self.edges + .iter() + .enumerate() + .map(|(id, edge)| (PhysicalEdgeId(id), *edge)) + } + + /// $P^{\mathrm{adm}}_{c,i}$: the admissible edges into input + /// `input_index_of_consumer` of `consumer` + /// ($\alpha^{\log} = \alpha^{\mathrm{phys}} = 1$). + pub fn admissible_edges_into( + &self, + consumer: PhysicalCandidateId, + input_index_of_consumer: usize, + ) -> &[PhysicalEdgeId] { + let _ = ( + consumer, + input_index_of_consumer, + &self.admissible_edges_into_input, + ); + todo!("#1207 PR 5") + } + + /// The admissible edges out of `producer`: every consumer that may share it. + pub fn admissible_edges_out_of(&self, producer: PhysicalCandidateId) -> &[PhysicalEdgeId] { + let _ = (producer, &self.admissible_edges_out_of_candidate); + todo!("#1207 PR 5") + } } /// The output references of `query_roots`: each operator reference, in @@ -145,82 +158,3 @@ pub fn output_references(query_roots: &[QueryRoot]) -> Vec<(usize, } references } - -impl PhysicalASAPMemoCandidates { - /// Builds the private per-group index over `candidates`. Stage 3's - /// component search uses it to build one smaller catalog per component. - pub fn new( - operator_nodes_of_candidates_for_memo_groups: Vec, - candidates: Vec, - query_roots: Vec>, - reader_constraints: Vec, - ) -> Self { - let mut implementation_roots_of_group: Vec> = Vec::new(); - for (id, candidate) in candidates.iter().enumerate() { - let node = &operator_nodes_of_candidates_for_memo_groups[candidate.operator_node.0]; - if node.is_implementation_root { - let group = node.group().0 as usize; - if implementation_roots_of_group.len() <= group { - implementation_roots_of_group.resize(group + 1, Vec::new()); - } - implementation_roots_of_group[group].push(PhysicalCandidateId(id)); - } - } - let output_reference_groups = output_references(&query_roots) - .into_iter() - .map(|(_, group)| group) - .collect(); - Self { - operator_nodes_of_candidates_for_memo_groups, - candidates, - query_roots, - reader_constraints, - pruned_candidates: Vec::new(), - implementation_roots_of_group, - output_reference_groups, - } - } - - /// The implementation roots of `group`, ascending. - pub fn implementation_roots(&self, group: MemoGroupId) -> &[PhysicalCandidateId] { - self.implementation_roots_of_group - .get(group.0 as usize) - .map_or(&[], Vec::as_slice) - } - - /// The candidates that may produce output reference `output_reference` - /// (each group reference of `query_roots`, in order; a scalar query has - /// several): its group's roots with `may_produce_query_output`. - pub fn query_output_candidates( - &self, - output_reference: usize, - ) -> impl Iterator + '_ { - let roots = self - .output_reference_groups - .get(output_reference) - .map_or(&[][..], |&group| self.implementation_roots(group)); - roots - .iter() - .copied() - .filter(|id| self.candidates[id.0].may_produce_query_output) - } - - /// The number of output references: group references of `query_roots`. - pub fn output_reference_count(&self) -> usize { - self.output_reference_groups.len() - } - - /// The operator node candidate `candidate` runs. - pub fn physical_node(&self, candidate: PhysicalCandidateId) -> &Rc { - &self.operator_nodes_of_candidates_for_memo_groups - [self.candidates[candidate.0].operator_node.0] - .physical_node - } - - /// The memo group whose candidates candidate `candidate` implements. - pub fn memo_group_of(&self, candidate: PhysicalCandidateId) -> MemoGroupId { - self.operator_nodes_of_candidates_for_memo_groups - [self.candidates[candidate.0].operator_node.0] - .group() - } -} diff --git a/crates/types/src/stages/physical/mod.rs b/crates/types/src/stages/physical/mod.rs index 7f029925a..b5ed3f780 100644 --- a/crates/types/src/stages/physical/mod.rs +++ b/crates/types/src/stages/physical/mod.rs @@ -1,18 +1,13 @@ -//! The Stage 2 → Stage 3 boundary: the physical candidates Stage 2 builds -//! from the memo and Stage 3 selects from (#1047, #1054). +//! The Stage 2 → Stage 3 boundary: the physical candidates and edges Stage 2 +//! builds from the memo and Stage 3 selects from (#1047, #1054, #1200, #1201). +mod execution_context; mod graph; mod memo_candidates; -mod producers; -pub use graph::{ - DagOfSelectedCandidates, PhysicalCandidateId, ReaderConstraint, SelectedEdge, SelectedNodeId, -}; +pub use execution_context::{ExecutionContext, ExecutionContextId, RefreshWindowId}; +pub use graph::{DagOfSelectedCandidates, PhysicalCandidateId, SelectedEdge, SelectedNodeId}; pub use memo_candidates::{ - output_references, CandidateForMemoGroupId, OperatorNodeOfCandidateForMemoGroup, - OperatorNodeOfCandidateForMemoGroupId, PhysicalASAPMemoCandidates, PhysicalCandidate, - PhysicalCandidateKey, + output_references, producer_satisfies_physical_requirements, PhysicalASAPMemoCandidates, + PhysicalCandidate, PhysicalCandidateKey, PhysicalEdge, PhysicalEdgeId, }; -#[cfg(any(test, feature = "test-support"))] -pub use producers::{producer_storage_metrics, ProducerStorageMetrics}; -pub use producers::{ProducerCandidates, ProducerSetInterner}; diff --git a/crates/types/src/stages/physical/producers.rs b/crates/types/src/stages/physical/producers.rs deleted file mode 100644 index 0e7547434..000000000 --- a/crates/types/src/stages/physical/producers.rs +++ /dev/null @@ -1,206 +0,0 @@ -//! Shared storage for ordered compatible producers. Sharing this storage never -//! ties together the independent selections of different consumer input slots. -use super::graph::PhysicalCandidateId; -use std::collections::HashSet; -use std::ops::{Deref, DerefMut}; -use std::rc::Rc; - -#[derive(Debug, Clone, Default, PartialEq, Eq, Hash)] -pub struct ProducerCandidates(Rc>); - -impl From> for ProducerCandidates { - fn from(values: Vec) -> Self { - Self(Rc::new(values)) - } -} - -impl FromIterator for ProducerCandidates { - fn from_iter>(values: T) -> Self { - Self::from(values.into_iter().collect::>()) - } -} - -impl Deref for ProducerCandidates { - type Target = Vec; - fn deref(&self) -> &Self::Target { - &self.0 - } -} - -impl DerefMut for ProducerCandidates { - fn deref_mut(&mut self) -> &mut Self::Target { - Rc::make_mut(&mut self.0) - } -} - -impl<'a> IntoIterator for &'a ProducerCandidates { - type Item = &'a PhysicalCandidateId; - type IntoIter = std::slice::Iter<'a, PhysicalCandidateId>; - fn into_iter(self) -> Self::IntoIter { - self.iter() - } -} - -impl<'a> IntoIterator for &'a mut ProducerCandidates { - type Item = &'a mut PhysicalCandidateId; - type IntoIter = std::slice::IterMut<'a, PhysicalCandidateId>; - fn into_iter(self) -> Self::IntoIter { - self.iter_mut() - } -} - -impl IntoIterator for ProducerCandidates { - type Item = PhysicalCandidateId; - type IntoIter = std::vec::IntoIter; - fn into_iter(self) -> Self::IntoIter { - Rc::unwrap_or_clone(self.0).into_iter() - } -} - -/// Exact ordered-content interning. No sorting, deduplication, or compatibility -/// inference occurs here; malformed input remains visible to graph validation. -#[derive(Default)] -pub struct ProducerSetInterner { - sets: HashSet, -} - -impl ProducerSetInterner { - pub fn intern(&mut self, values: impl Into) -> ProducerCandidates { - let values = values.into(); - if let Some(existing) = self.sets.get(&values) { - return existing.clone(); - } - self.sets.insert(values.clone()); - values - } -} - -/// Count physical producer-list storage separately from logical connections. -/// Distinct input positions remain distinct even when their storage is shared. -#[cfg(any(test, feature = "test-support"))] -#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] -pub struct ProducerStorageMetrics { - pub logical_connections: usize, - pub unique_producer_sets: usize, - pub stored_producer_ids: usize, -} - -#[cfg(any(test, feature = "test-support"))] -pub fn producer_storage_metrics( - physical: &super::PhysicalASAPMemoCandidates, -) -> ProducerStorageMetrics { - let mut allocations = HashSet::new(); - let mut metrics = ProducerStorageMetrics::default(); - for producer_candidates in physical - .candidates - .iter() - .flat_map(|candidate| &candidate.producer_candidates_per_input) - { - metrics.logical_connections += producer_candidates.len(); - if allocations.insert(Rc::as_ptr(&producer_candidates.0)) { - metrics.unique_producer_sets += 1; - metrics.stored_producer_ids += producer_candidates.len(); - } - } - metrics -} - -#[cfg(test)] -mod tests { - use super::*; - - // Two slots can choose different producers even when their compatible lists - // share one allocation; storage compression adds no selection equality. - #[test] - fn shared_producer_storage_keeps_input_choices_independent() { - use super::super::graph::{DagOfSelectedCandidates, SelectedEdge, SelectedNodeId}; - use super::super::memo_candidates::{ - CandidateForMemoGroupId, OperatorNodeOfCandidateForMemoGroup, - OperatorNodeOfCandidateForMemoGroupId, PhysicalASAPMemoCandidates, PhysicalCandidate, - }; - use crate::ir::properties::ExecutionTiming; - use crate::ir::QueryRoot; - use crate::physical::Materialization; - use crate::stages::MemoGroupId; - let mut interner = ProducerSetInterner::default(); - let choices = interner.intern(vec![PhysicalCandidateId(0), PhysicalCandidateId(1)]); - use crate::ir::scalar::expr_ir::ScalarValue; - use crate::ir::schema::{DataType, Field, Schema}; - use crate::ir::{NonASAPOp, Operator, OperatorNode, ScalarExpr}; - let node = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Values { - rows: vec![vec![ScalarExpr::Literal(ScalarValue::Float64(1.0))]], - schema: Schema::lifted(vec![Field::plain("value", DataType::Float64, false)], None), - })) - .unwrap(); - let physical = PhysicalASAPMemoCandidates::new( - (0..3) - .map(|group| OperatorNodeOfCandidateForMemoGroup { - candidates_for_memo_group: vec![CandidateForMemoGroupId { - group: MemoGroupId(group), - index: 0, - }], - is_implementation_root: true, - accuracy_budget_parts: 1, - physical_node: Rc::clone(&node), - }) - .collect(), - (0..3) - .map(|id| PhysicalCandidate { - operator_node: OperatorNodeOfCandidateForMemoGroupId(id), - timing: ExecutionTiming::QueryTime, - materialization: Materialization::NotMaterialized, - producer_candidates_per_input: if id == 2 { - vec![choices.clone(), choices.clone()] - } else { - vec![] - }, - may_produce_query_output: id == 2, - may_keep_across_evaluations_per_sole_reader_query: Vec::new(), - }) - .collect(), - vec![QueryRoot::Operator(MemoGroupId(2))], - Vec::new(), - ); - let selected = DagOfSelectedCandidates { - candidate_of_node: (0..3).map(PhysicalCandidateId).collect(), - edges: (0..2) - .map(|input| SelectedEdge { - producer: SelectedNodeId(input), - consumer: SelectedNodeId(2), - input, - }) - .collect(), - node_of_output_reference: vec![SelectedNodeId(2)], - }; - selected.validate(&physical).unwrap(); - assert_eq!( - producer_storage_metrics(&physical), - ProducerStorageMetrics { - logical_connections: 4, - unique_producer_sets: 1, - stored_producer_ids: 2 - } - ); - } - - // Equal ordered lists share one allocation; mutation of one input detaches - // it and cannot silently change another input or an interner key. - #[test] - fn interned_inputs_detach_on_mutation() { - let mut interner = ProducerSetInterner::default(); - let original = interner.intern(vec![PhysicalCandidateId(2), PhysicalCandidateId(1)]); - let mut changed = interner.intern(vec![PhysicalCandidateId(2), PhysicalCandidateId(1)]); - assert!(Rc::ptr_eq(&original.0, &changed.0)); - changed.retain(|id| id.0 == 1); - assert_eq!( - original.as_slice(), - &[PhysicalCandidateId(2), PhysicalCandidateId(1)] - ); - assert_eq!(changed.as_slice(), &[PhysicalCandidateId(1)]); - assert!(!Rc::ptr_eq(&original.0, &changed.0)); - let repeated = interner.intern(original.clone()); - assert!(Rc::ptr_eq(&original.0, &repeated.0)); - let reversed = interner.intern(vec![PhysicalCandidateId(1), PhysicalCandidateId(2)]); - assert!(!Rc::ptr_eq(&original.0, &reversed.0)); - } -} diff --git a/crates/types/src/stages/pruning.rs b/crates/types/src/stages/pruning.rs index f857460f4..a2f8446f4 100644 --- a/crates/types/src/stages/pruning.rs +++ b/crates/types/src/stages/pruning.rs @@ -1,8 +1,8 @@ //! What a stage removed, and why: one shape for every stage (#1047 P6). //! Each stage removes only what its own knowledge rules out (option C): -//! Pass 1 what the measure or the deployment cannot compute, Stage 2 what -//! cannot be implemented, built or kept. Stage 0 fails with an error and -//! Pass 2 only adds candidates, so neither prunes. Stage 3 prunes before +//! Stage 2 what cannot be implemented, built or materialized. Stage 0 fails with an +//! error, Pass 1 records nothing it leaves out (#1205 will show it), and +//! Pass 2 only adds candidates. Stage 3 prunes before //! its selection MILP (dominated and uncostable candidates); its //! rejections of whole plans keep their own cause (`RejectionCause` in plan //! selection). @@ -10,9 +10,9 @@ use super::{MemoGroupId, PhysicalCandidateId}; /// One candidate a stage removed from memo group `group`, and why. `C` is -/// the stage's candidate type: Pass 1's way to compute a measure -/// (`HowToComputeAggregateMeasure`), Stage 2's -/// [`PhysicalCandidateKey`](super::PhysicalCandidateKey). +/// the stage's candidate type: Stage 2's +/// [`PhysicalCandidateKey`](super::PhysicalCandidateKey), Stage 3's +/// [`PhysicalCandidateId`]. #[derive(Debug, Clone, PartialEq)] pub struct PrunedCandidate { pub group: MemoGroupId, @@ -20,54 +20,11 @@ pub struct PrunedCandidate { pub reason: PruneReason, } -/// Why a stage removed a candidate. The first seven are Pass 1's: the -/// candidate cannot compute its measure, the deployment cannot run it, or -/// no query that reads it can meet its accuracy target with it. -/// Then Stage 2's: the physical candidate cannot be built, used or -/// kept. The last two are Stage 3's, before its selection MILP. +/// Why a stage removed a candidate: first Stage 2's (the physical candidate +/// cannot be built, used or materialized), then Stage 3's, before its selection +/// MILP. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum PruneReason { - /// Pass 1: the measure requires an exact answer and the summary is - /// approximate. - RequiresExact, - /// Pass 1: `count` counts rows, so a possibly-null `avg` input changes - /// the result. - NullableAvgInput, - /// Pass 1: frequency sketches do not implement SQL row-count updates. - /// The exact counter is kept; a fused raw-row top-k has its own keyed - /// update contract. - NoRelationalCountSketchRealization, - /// Pass 1: the deployment's executor cannot build the summary's family - /// or answer the measure's readout from it (`summary_supported` in - /// Stage 1). - UnsupportedByDeployment, - /// Pass 1: the quantile's φ is outside [0, 1], where PromQL answers -Inf - /// or +Inf: no sketch estimates that, and the executor's sketches refuse - /// such a φ, so only the exact answer remains. - QuantileOutsideUnitInterval, - /// Pass 1: a PromQL top-k's heap (Count-Min or CountSketch with a heap) - /// over its own input, an instant vector with one row per ranked series - /// or group: the heap reads every row and adds nothing to it, so an - /// exact top-k over the same rows, which keeps only `k` of them, is - /// exact and no costlier. Only the whole-expression heap over the inner - /// aggregate's input, where an item's rows repeat, is offered (Pass 1's - /// fused computation). - TopKHeapOverOneRowPerItem, - /// Pass 1: for every query that reads its memo group, the query's - /// output has no error bound that can meet the query's accuracy target - /// at any summary size: guarantee propagation has no rule above it - /// (e.g. a rank error through `avg`, or a value `HAVING` above an - /// estimate), its bound stays unknown once the data workload's - /// declarations are applied (e.g. a relative error through `avg` over - /// samples not declared non-negative), its bound is in a metric the - /// target does not accept (the deployment's accuracy model decides), or - /// its failure probability is per value and the target's is over a - /// whole result whose number of values the data does not bound. Kept - /// when another aggregate over the same source offers the same family, - /// which Pass 2 may build from it. Pruned only when the deployment's - /// accuracy model and evidence cannot accept an output without its - /// guarantee. - NoErrorBoundForQueryTarget, /// Stage 2: the executor cannot run its operator node, an exact /// per-entity `Aggregate` (a per-series range reduction over raw data /// samples) with more than one measure, a measure without an exact @@ -83,14 +40,14 @@ pub enum PruneReason { /// cycle, on an input that cannot be built. InputHasNoBuildableProducer, /// Stage 2: no query output can read it, directly or through other - /// candidates, once each candidate's allowed readers are applied. + /// candidates, once each candidate's allowed consumers are applied. ReachesNoQueryOutput, /// Stage 2: it is shared within one query evaluation - /// (`MatAtQuery`), which needs at least two reading queries, and fewer + /// (`OneEvaluation`), which needs at least two reading queries, and fewer /// queries can reach it. TooFewQueryReaders, /// Stage 3: the candidate it names, in the same memo group with the - /// same producers and readers, is at least as good in every respect + /// same producers and consumers, is at least as good in every respect /// (guarantee, usage, latency, output, held state), so no optimal plan /// needs this one. DominatedBy(PhysicalCandidateId), diff --git a/crates/types/src/stages/realizer.rs b/crates/types/src/stages/realizer.rs index 8c2ff17d6..96ad555a1 100644 --- a/crates/types/src/stages/realizer.rs +++ b/crates/types/src/stages/realizer.rs @@ -33,21 +33,21 @@ impl From<&str> for PhysicalCandidateError { } /// A realized selection (was `SelectedPhysicalCandidate`): the physical -/// plan Stage 2 built for it, how long each kept tumbling-window chain of that plan +/// plan Stage 2 built for it, how long each materialized tumbling-window chain of that plan /// keeps its tumbling windows, and the operator node each selected node became. #[derive(Debug, Clone)] pub struct RealizedPlan { /// The selection rebuilt as one physical plan, one operator node per - /// selected node, each with its candidate's timing and materialization. + /// selected node, each with its candidate's timing and lifetime. pub plan: PhysicalWorkloadPlan, - /// For each tumbling-window chain of `plan` kept beyond one evaluation - /// (maintained at ingestion time, or built at query time and kept + /// For each tumbling-window chain of `plan` materialized beyond one evaluation + /// (maintained at ingestion time, or built at query time and materialized /// across evaluations), by node id in `plan.dag`: how far back from the /// newest evaluation the runtime must keep its completed tumbling windows, in /// milliseconds. It is how far its tumbling windows reach back: the offset its /// newest tumbling window ends before the evaluation (`[60m] offset 10m`: 10 min), /// plus the chain's window, plus the longest range-evaluation span of - /// its readers, so the earliest step of the next request still finds + /// its queries, so the earliest step of the next request still finds /// its tumbling windows; without an offset or range evaluations, just the window. /// Empty when the plan keeps no tumbling windows. pub tumbling_window_retention_ms: BTreeMap, @@ -68,7 +68,7 @@ pub struct RealizedPlan { /// recomposed, and every output contract (result kind and schema) met; a /// realized assignment also passes Stage 2's timing and dependency rules /// and the physical DAG validator, with every node keeping its -/// candidate's timing and materialization. +/// candidate's timing and lifetime. /// /// It checks structure and execution legality only. It does not check cost, /// and it does not check accuracy targets: Stage 3 does both. diff --git a/crates/types/src/test_support.rs b/crates/types/src/test_support.rs index d267ee3eb..71879fdc5 100644 --- a/crates/types/src/test_support.rs +++ b/crates/types/src/test_support.rs @@ -2,43 +2,34 @@ //! crates' tests), exposed under the `test-support` feature so the normal //! public API stays as narrow as its production users need. Each forwards //! to the crate-private function of the same name. -use std::rc::Rc; use crate::ir::materialized_view::RawDataWindow; +use crate::ir::operator::input::node_of_build; use crate::ir::operator::maintained_population::CurrentSeriesInput; use crate::ir::operator::node::Operator; -use crate::ir::physical_export::PhysicalASAPDAGCompilation; use crate::ir::properties::summary_coverage::SummaryCoverage; -use crate::ir::properties::timing::NodeToTimedNodeCache; -use crate::ir::properties::ExecutionDataStateError; + use crate::ir::schema::Schema; use crate::ir::{ASAPOp, NonASAPOp}; use crate::ir::{OperatorNode, SchemaDerivationError}; -use crate::physical::Materialization; +use std::rc::Rc; /// `SummaryCoverage::definition_id`. pub fn definition_id(coverage: &SummaryCoverage) -> String { coverage.definition_id() } -/// `tumbling_window_chain::whole_window_build`. -pub fn whole_window_build(chain: &OperatorNode) -> Result { - crate::ir::operator::tumbling_window_chain::whole_window_build(chain) +/// `tumbling_window_chain::whole_window_build`, as a node. +pub fn whole_window_build(chain: &Rc) -> Result { + crate::ir::operator::tumbling_window_chain::whole_window_build(chain).map(node_of_build) } -/// `tumbling_window_chain::tumbling_window`. +/// `tumbling_window_chain::tumbling_window`, as a node. pub fn tumbling_window( - chain: &OperatorNode, + chain: &Rc, index: u32, ) -> Result { - crate::ir::operator::tumbling_window_chain::tumbling_window(chain, index) -} - -/// `physical_export::compile_physical_asap_dag_with_node_ids`. -pub fn compile_physical_asap_dag_with_node_ids( - root: &Rc, -) -> Result { - crate::ir::physical_export::compile_physical_asap_dag_with_node_ids(root) + crate::ir::operator::tumbling_window_chain::tumbling_window(chain, index).map(node_of_build) } /// `Operator::output_schema`. @@ -57,24 +48,42 @@ pub fn validate_asap_inputs(op: &ASAPOp) -> Result<(), SchemaDerivationError> { } /// `CurrentSeriesInput::matches_node`. -pub fn matches_node(input: &CurrentSeriesInput, node: &OperatorNode) -> bool { +pub fn matches_node(input: &CurrentSeriesInput, node: &Rc) -> bool { input.matches_node(node) } -/// `NodeToTimedNodeCache::timed`. -pub fn timed<'a>( - cache: &'a NodeToTimedNodeCache, - input: &Rc, -) -> Option<&'a Rc> { - cache.timed(input) -} - /// `RawDataWindow::lookback_ms`. pub fn lookback_ms(window: &RawDataWindow) -> u64 { window.lookback_ms() } -/// `Materialization::is_materialized_view`. -pub fn is_materialized_view(materialization: Materialization) -> bool { - materialization.is_materialized_view() +/// `physical_export::physical_asap_dag_at_query_time_with_node_ids`: `roots` +/// exported with every node at query time, not materialized and without θ, +/// with each exported node's operator node by node id. +pub fn physical_asap_dag_at_query_time_with_node_ids( + roots: &[crate::ir::QueryRoot], +) -> Result< + ( + crate::ir::physical_export::PhysicalASAPDAG, + Vec>, + ), + crate::ir::physical_export::PhysicalASAPDAGValidationError, +> { + crate::ir::physical_export::physical_asap_dag_at_query_time_with_node_ids(roots) +} + +/// [`physical_asap_dag_at_query_time_with_node_ids`] of the single root `root`, without +/// the node ids. +pub fn physical_asap_dag_at_query_time( + root: &Rc, +) -> Result< + crate::ir::physical_export::PhysicalASAPDAG, + crate::ir::physical_export::PhysicalASAPDAGValidationError, +> { + Ok( + physical_asap_dag_at_query_time_with_node_ids(&[crate::ir::QueryRoot::Operator( + Rc::clone(root), + )])? + .0, + ) } diff --git a/crates/types/src/workload/bounds.rs b/crates/types/src/workload/bounds.rs index a50afb0ed..85b33b75c 100644 --- a/crates/types/src/workload/bounds.rs +++ b/crates/types/src/workload/bounds.rs @@ -208,7 +208,11 @@ fn declared_group_bound(node: &OperatorNode, data: &DataWorkload) -> Option ) => grouped(reduction, child), // A merge's children are states of one definition: its first // build's grouping stands for all of them. - Operator::ASAP(ASAPOp::SummaryMerge { .. }) => match node.summary_build() { + Operator::ASAP(ASAPOp::SummaryMerge { + children, + group_by: None, + .. + }) => match children.first().and_then(|first| first.summary_build()) { Some( ASAPOp::SummaryAgg { reduction, child, .. @@ -216,7 +220,7 @@ fn declared_group_bound(node: &OperatorNode, data: &DataWorkload) -> Option | ASAPOp::SummaryTumblingWindowChain { reduction, child, .. }, - ) => grouped(reduction, child), + ) => grouped(&reduction, &child), _ => None, }, Operator::ASAP(ASAPOp::SummaryEstimate { @@ -365,11 +369,11 @@ fn at_most_one_row(node: &OperatorNode) -> bool { .. }) => keys.is_empty(), // A merge gives one state per group, whatever the tumbling windows it merges. - Operator::ASAP(ASAPOp::SummaryMerge { .. }) => matches!( - node.summary_build(), + Operator::ASAP(ASAPOp::SummaryMerge { children, .. }) => matches!( + children.first().and_then(|first| first.summary_build()), Some( ASAPOp::SummaryAgg { reduction, .. } | ASAPOp::SummaryTumblingWindowChain { reduction, .. } - ) if global(reduction) + ) if global(&reduction) ), Operator::ASAP(ASAPOp::SummaryEstimate { summary_input, @@ -476,9 +480,7 @@ pub fn with_confirmed_weights_at( let rebuilt = if changed { Rc::new( OperatorNode::with_schema(operator, node.schema.clone()) - .with_guarantee(node.guarantee.clone()) - .with_timing(node.timing) - .with_materialization(node.materialization), + .with_guarantee(node.guarantee.clone()), ) } else { Rc::clone(node) diff --git a/crates/types/tests/main.rs b/crates/types/tests/main.rs index e95aeb9cf..bf56559c0 100644 --- a/crates/types/tests/main.rs +++ b/crates/types/tests/main.rs @@ -2,12 +2,10 @@ //! its own copy of every dependency, which outgrew CI's disk. //! One file: `cargo test -p asap-types --test types ::`. -mod materialized_view; mod physical_export; mod planner_vocabulary; mod schema_rebuilding; mod structure_contract; -mod summary_coverage; mod summary_merge_structure; mod tumbling_window_chain; mod workload_compatibility; diff --git a/crates/types/tests/materialized_view.rs b/crates/types/tests/materialized_view.rs deleted file mode 100644 index 0576ca187..000000000 --- a/crates/types/tests/materialized_view.rs +++ /dev/null @@ -1,512 +0,0 @@ -//! A DAG's materialized views and their retention -//! (`asap_types::ir::materialized_view`): two one-minute KLL tumbling windows of `m` -//! kept at ingestion, merged into a view refreshed on first read. - -use std::collections::HashMap; -use std::rc::Rc; -use std::time::Duration; - -use asap_types::ir::materialized_view::{ - InstallableDagError, MaterializedViewRefresh, RefreshWindow, -}; -use asap_types::ir::operator::operator_properties::{Reduction, Source}; -use asap_types::ir::operator::tumbling_window_chain::expand_tumbling_window_chains_in_workload; -use asap_types::ir::operator::TimeShift; -use asap_types::ir::physical_export::{ - compile_physical_asap_dag, compile_physical_asap_workload, PhysicalASAPDAG, -}; -use asap_types::ir::scalar::ColumnRef; -use asap_types::ir::schema::{ - DataType, Field, FieldDataType, GroupingStrategy, Schema, SketchAlgorithm, SketchKind, - SketchParams, SummaryUpdate, -}; -use asap_types::ir::{ - apply_materialization_timings, apply_materialization_timings_to_roots, ASAPOp, - MaterializationAssignment, NodeToTimedNodeCache, NonASAPOp, Operator, OperatorNode, QueryRoot, - TimeRangeKind, -}; -use asap_types::physical::{Materialization, Medium}; - -/// A KLL of `m`'s values over `(T - shift - 1 min, T - shift]`. -fn tumbling_window(scan: &Rc, shift_ms: i64) -> Rc { - let shifted = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeShift { - shift: TimeShift { - offset_ms: shift_ms, - at: None, - }, - child: Rc::clone(scan), - })) - .unwrap(); - let range = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeRange { - range: Duration::from_secs(60), - kind: TimeRangeKind::Range, - child: shifted, - })) - .unwrap(); - OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryAgg { - child: range, - family: FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: 200, - failure_probability: None, - }, - ), - Default::default(), - ), - input: SummaryUpdate::column(ColumnRef::Named("value".into())), - reduction: Reduction::by(vec![2]), - grouping: GroupingStrategy::default(), - filter: None, - })) - .unwrap() -} - -/// `m` with columns `ts`, `value` and `job`. -fn scan_m() -> Rc { - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Scan { - source: Source::TimeSeries { metric: "m".into() }, - predicates: vec![], - schema: Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("value", DataType::Float64, false), - Field::plain("job", DataType::Utf8, true), - ], - 0, - vec![], - ), - })) - .unwrap() -} - -/// The tumbling windows at offsets 0 and 1 min kept at ingestion, their merge kept -/// across evaluations, exported: `(dag, merge's node id)`. -fn tumbling_windows_and_kept_merge() -> (PhysicalASAPDAG, usize) { - let scan = scan_m(); - let tumbling_windows = [tumbling_window(&scan, 0), tumbling_window(&scan, 60_000)]; - let merge = OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryMerge { - children: tumbling_windows.to_vec(), - newest_tumbling_windows: None, - group_by: None, - })) - .unwrap(); - let mut assignment = MaterializationAssignment::default(); - for tumbling_window in &tumbling_windows { - assignment.set( - tumbling_window, - Materialization::MatAtIngestion(Medium::Memory), - ); - } - assignment.set( - &merge, - Materialization::MatAtQueryKeptAcrossEvaluations(Medium::Memory), - ); - let timed = - apply_materialization_timings(&merge, &assignment, &mut NodeToTimedNodeCache::new()) - .unwrap(); - let dag = compile_physical_asap_dag(&timed).unwrap(); - let merge_id = dag.roots[0]; - (dag, merge_id) -} - -/// A view without `retention_ms` (a DAG before installation) has no -/// materialized views to list. -#[test] -fn a_view_without_retention_is_refused() { - let (dag, _) = tumbling_windows_and_kept_merge(); - assert!(matches!( - dag.materialized_views(), - Err(InstallableDagError::ViewWithoutRetention { .. }) - )); -} - -/// The merge, refreshed on first read for each step of a one-hour range, -/// is kept for its two minutes plus the hour; each tumbling window it reads is kept -/// that long too, since the merge may be refreshed for its oldest step. -/// The tumbling windows read `m` at ingestion, so no raw data is kept for queries. -#[test] -fn retention_propagates_from_a_view_refreshed_on_first_read_to_its_inputs() { - let (mut dag, merge_id) = tumbling_windows_and_kept_merge(); - let spans = HashMap::from([(merge_id, 3_600_000)]); - dag.set_retention(&HashMap::new(), &spans).unwrap(); - let views = dag.materialized_views().unwrap(); - let (merge, tumbling_windows) = views.split_last().unwrap(); - assert_eq!(merge.node_id, merge_id); - assert_eq!(merge.refresh(), MaterializedViewRefresh::OnFirstRead); - assert_eq!(merge.retention_ms(), 120_000 + 3_600_000); - assert_eq!(tumbling_windows.len(), 2); - for tumbling_window in tumbling_windows { - assert_eq!( - tumbling_window.refresh(), - MaterializedViewRefresh::AtIngestion - ); - assert_eq!(tumbling_window.retention_ms(), merge.retention_ms()); - let node = dag - .nodes - .iter() - .find(|n| n.id == tumbling_window.node_id) - .unwrap(); - assert_eq!(node.retention_ms, Some(tumbling_window.retention_ms())); - } - assert_eq!(dag.raw_data_retention().unwrap(), []); -} - -/// A view kept for less than its data reaches back (the older tumbling window, which -/// reaches 2 minutes back, kept for 1 minute) is refused. -#[test] -fn a_view_kept_shorter_than_its_reach_is_refused() { - let (mut dag, _) = tumbling_windows_and_kept_merge(); - let views = dag.set_retention(&HashMap::new(), &HashMap::new()).unwrap(); - let older = views - .iter() - .find(|v| { - v.inputs - .raw_data_windows() - .first() - .map(|w| w.end_offset_ms()) - == Some(60_000) - }) - .unwrap() - .node_id; - dag.nodes - .iter_mut() - .find(|n| n.id == older) - .unwrap() - .retention_ms = Some(60_000); - assert_eq!( - dag.materialized_views(), - Err(InstallableDagError::ViewRetentionTooShort { node_id: older }) - ); -} - -/// A quantile sketch of `algorithm` (KLL k = 200 or DDSketch α = 0.01). -fn quantile_sketch(algorithm: SketchAlgorithm) -> FieldDataType { - let params = match algorithm { - SketchAlgorithm::Kll => SketchParams::Kll { - k: 200, - failure_probability: None, - }, - _ => SketchParams::DDSketch { alpha: 0.01 }, - }; - FieldDataType::Sketch(SketchKind::new(algorithm, params), Default::default()) -} - -/// `tumbling_window_count` tumbling windows of `tumbling_window_width_ms` of `family` per `job` over `input`. -fn chain( - input: &Rc, - family: FieldDataType, - tumbling_window_width_ms: u64, - tumbling_window_count: u32, -) -> Rc { - OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { - child: Rc::clone(input), - family, - input: SummaryUpdate::column(ColumnRef::Named("value".into())), - reduction: Reduction::by(vec![2]), - grouping: GroupingStrategy::default(), - filter: None, - tumbling_window_width_ms, - tumbling_window_count, - })) - .unwrap() -} - -/// Three queries over one `Scan` of `m`, each merging a chain kept at -/// ingestion: a KLL of six 10 s tumbling windows, a DDSketch of thirty 10 s tumbling windows, and -/// a KLL of five 60 s tumbling windows. The DAG Stage 3 costs (chains unexpanded) and -/// the installed DAG (chains expanded into tumbling windows, retention set). -fn three_chains_over_one_scan() -> (PhysicalASAPDAG, PhysicalASAPDAG) { - let scan = scan_m(); - let chains = [ - chain(&scan, quantile_sketch(SketchAlgorithm::Kll), 10_000, 6), - chain( - &scan, - quantile_sketch(SketchAlgorithm::DDSketch), - 10_000, - 30, - ), - chain(&scan, quantile_sketch(SketchAlgorithm::Kll), 60_000, 5), - ]; - let mut assignment = MaterializationAssignment::default(); - let roots: Vec = chains - .iter() - .map(|chain| { - assignment.set(chain, Materialization::MatAtIngestion(Medium::Memory)); - QueryRoot::Operator( - OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryMerge { - children: vec![Rc::clone(chain)], - newest_tumbling_windows: None, - group_by: None, - })) - .unwrap(), - ) - }) - .collect(); - let timed = apply_materialization_timings_to_roots(&roots, &assignment).unwrap(); - let unexpanded = compile_physical_asap_workload(&timed).unwrap(); - let expanded = expand_tumbling_window_chains_in_workload(&timed).unwrap(); - let mut installed = compile_physical_asap_workload(&expanded.roots).unwrap(); - installed - .set_retention(&HashMap::new(), &HashMap::new()) - .unwrap(); - (unexpanded, installed) -} - -/// The width and sketch of the tumbling windows `node` of `dag` builds. -fn tumbling_window_shape(dag: &PhysicalASAPDAG, node: usize) -> (u64, SketchAlgorithm) { - let payload = &dag.nodes.iter().find(|n| n.id == node).unwrap().payload; - let (family, width_ms) = match payload { - Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { - family, - tumbling_window_width_ms, - .. - }) => (family, *tumbling_window_width_ms), - Operator::ASAP(ASAPOp::SummaryAgg { family, child, .. }) => { - let range = &dag.nodes.iter().find(|n| n.id == *child).unwrap().payload; - let Operator::NonASAP(NonASAPOp::TimeRange { range, .. }) = range else { - panic!("a tumbling window reads a TimeRange") - }; - (family, range.as_millis() as u64) - } - other => panic!("not a tumbling window: {other:?}"), - }; - let FieldDataType::Sketch(kind, _) = family else { - panic!("a sketch") - }; - (width_ms, *kind.algorithm()) -} - -/// A chain kept at ingestion has its newest tumbling window's refresh window on the -/// DAG Stage 3 costs, and each of its expanded tumbling windows has the same one. -#[test] -fn a_chain_and_each_of_its_tumbling_windows_have_one_refresh_window() { - let (unexpanded, installed) = three_chains_over_one_scan(); - let window_of = |width_ms| RefreshWindow { - width_ms, - phase_ms: 0, - }; - let chains: Vec = unexpanded - .nodes - .iter() - .filter(|node| { - matches!( - node.payload, - Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) - ) - }) - .map(|node| node.id) - .collect(); - assert_eq!(chains.len(), 3); - for chain in chains { - let (width_ms, _) = tumbling_window_shape(&unexpanded, chain); - assert_eq!(unexpanded.refresh_window(chain), Some(window_of(width_ms))); - } - let views = installed.materialized_views().unwrap(); - assert_eq!(views.len(), 6 + 30 + 5); - for view in &views { - let (width_ms, _) = tumbling_window_shape(&installed, view.node_id); - assert_eq!( - installed.refresh_window(view.node_id), - Some(window_of(width_ms)) - ); - } -} - -/// The 10 s KLL and 10 s DDSketch tumbling windows are one refresh group, the 60 s -/// KLL tumbling windows another; the merges, at query time, are in none. -#[test] -fn tumbling_windows_of_one_width_form_one_refresh_group() { - let (_, installed) = three_chains_over_one_scan(); - let groups = installed.ingestion_refresh_groups().unwrap(); - let shapes: Vec<_> = groups - .iter() - .map(|group| { - let mut shapes: Vec<_> = group - .views - .iter() - .map(|view| tumbling_window_shape(&installed, view.node_id)) - .collect(); - shapes.sort_by_key(|(width_ms, algorithm)| (*width_ms, format!("{algorithm:?}"))); - shapes.dedup(); - (group.refresh_window, shapes) - }) - .collect(); - assert_eq!( - shapes, - [ - ( - Some(RefreshWindow { - width_ms: 10_000, - phase_ms: 0 - }), - vec![ - (10_000, SketchAlgorithm::DDSketch), - (10_000, SketchAlgorithm::Kll) - ] - ), - ( - Some(RefreshWindow { - width_ms: 60_000, - phase_ms: 0 - }), - vec![(60_000, SketchAlgorithm::Kll)] - ), - ] - ); - assert_eq!(groups[0].views.len(), 6 + 30); - assert_eq!(groups[1].views.len(), 5); -} - -/// A tumbling window whose window ends 30 s after a multiple of its 60 s width (a -/// `TimeShift(90 s)` below its range) is refreshed on another grid than a -/// tumbling window at no offset: another refresh group. -#[test] -fn a_shifted_tumbling_window_has_its_own_phase() { - let scan = scan_m(); - let tumbling_windows = [tumbling_window(&scan, 0), tumbling_window(&scan, 90_000)]; - let mut assignment = MaterializationAssignment::default(); - for tumbling_window in &tumbling_windows { - assignment.set( - tumbling_window, - Materialization::MatAtIngestion(Medium::Memory), - ); - } - let roots: Vec = tumbling_windows - .iter() - .map(|tumbling_window| QueryRoot::Operator(Rc::clone(tumbling_window))) - .collect(); - let timed = apply_materialization_timings_to_roots(&roots, &assignment).unwrap(); - let mut dag = compile_physical_asap_workload(&timed).unwrap(); - dag.set_retention(&HashMap::new(), &HashMap::new()).unwrap(); - let mut windows: Vec<(u64, u64)> = dag - .ingestion_refresh_groups() - .unwrap() - .iter() - .map(|group| { - let window = group.refresh_window.unwrap(); - (window.width_ms, window.phase_ms) - }) - .collect(); - windows.sort_unstable(); - assert_eq!(windows, [(60_000, 0), (60_000, 30_000)]); -} - -/// Two views kept at ingestion over rows with no time window (`Values`) -/// have no refresh window: each is a group of its own. -#[test] -fn views_of_unbounded_windows_are_groups_of_their_own() { - let rows = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Values { - rows: vec![], - schema: scan_m().schema.clone(), - })) - .unwrap(); - let mut assignment = MaterializationAssignment::default(); - let roots: Vec = [SketchAlgorithm::Kll, SketchAlgorithm::DDSketch] - .into_iter() - .map(|algorithm| { - let build = OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryAgg { - child: Rc::clone(&rows), - family: quantile_sketch(algorithm), - input: SummaryUpdate::column(ColumnRef::Named("value".into())), - reduction: Reduction::by(vec![2]), - grouping: GroupingStrategy::default(), - filter: None, - })) - .unwrap(); - assignment.set(&build, Materialization::MatAtIngestion(Medium::Memory)); - QueryRoot::Operator(build) - }) - .collect(); - let timed = apply_materialization_timings_to_roots(&roots, &assignment).unwrap(); - let mut dag = compile_physical_asap_workload(&timed).unwrap(); - dag.set_retention(&HashMap::new(), &HashMap::new()).unwrap(); - let groups = dag.ingestion_refresh_groups().unwrap(); - assert_eq!(groups.len(), 2); - for group in &groups { - assert_eq!(group.refresh_window, None); - assert_eq!(group.views.len(), 1); - } -} - -/// A rollup across jobs of a tumbling window, both kept at ingestion, reads the tumbling window's -/// window: one refresh group, the tumbling window listed before the rollup that reads -/// it, so a run whose roots hold both computes the tumbling window in the run. -#[test] -fn a_view_reading_a_view_of_its_group_follows_it() { - let scan = scan_m(); - let tumbling_window = tumbling_window(&scan, 0); - let rollup = OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryMerge { - children: vec![Rc::clone(&tumbling_window)], - newest_tumbling_windows: None, - group_by: Some(vec![]), - })) - .unwrap(); - let mut assignment = MaterializationAssignment::default(); - for view in [&tumbling_window, &rollup] { - assignment.set(view, Materialization::MatAtIngestion(Medium::Memory)); - } - let timed = apply_materialization_timings_to_roots( - &[QueryRoot::Operator(Rc::clone(&rollup))], - &assignment, - ) - .unwrap(); - let mut dag = compile_physical_asap_workload(&timed).unwrap(); - dag.set_retention(&HashMap::new(), &HashMap::new()).unwrap(); - let groups = dag.ingestion_refresh_groups().unwrap(); - assert_eq!(groups.len(), 1); - let [tumbling_window_view, rollup_view] = groups[0].views.as_slice() else { - panic!("{groups:#?}") - }; - assert_eq!(rollup_view.node_id, dag.roots[0]); - assert_eq!( - rollup_view.inputs.materialized_views, - [tumbling_window_view.node_id] - ); - assert_eq!( - groups[0].refresh_window, - Some(RefreshWindow { - width_ms: 60_000, - phase_ms: 0 - }) - ); -} - -/// On the DAG Stage 3 costs (chains unexpanded), a node above a chain has -/// the window of all the chain's tumbling windows: a merge of six 10 s tumbling windows, refreshed -/// at ingestion, has refresh window (60 s, 0). -#[test] -fn a_node_above_an_unexpanded_chain_has_the_window_of_its_tumbling_windows() { - let (unexpanded, _) = three_chains_over_one_scan(); - let merge_of_kll_10s = unexpanded.roots[0]; - assert_eq!( - unexpanded.refresh_window(merge_of_kll_10s), - Some(RefreshWindow { - width_ms: 60_000, - phase_ms: 0 - }) - ); -} - -/// A non-chain view (a KLL over `TimeRange(1 min)` shifted 15 s) has the -/// same refresh window, (60 s, 15 s), on a DAG of its nodes alone (no -/// edges, what Stage 3 builds from candidates): `view_windows` walks each -/// operator's inputs. -#[test] -fn a_view_has_its_refresh_window_on_a_dag_without_edges() { - let view = tumbling_window(&scan_m(), 15_000); - let mut assignment = MaterializationAssignment::default(); - assignment.set(&view, Materialization::MatAtIngestion(Medium::Memory)); - let timed = apply_materialization_timings(&view, &assignment, &mut NodeToTimedNodeCache::new()) - .unwrap(); - let mut dag = compile_physical_asap_dag(&timed).unwrap(); - let view_id = dag.roots[0]; - let window = Some(RefreshWindow { - width_ms: 60_000, - phase_ms: 15_000, - }); - assert_eq!(dag.refresh_window(view_id), window); - dag.edges.clear(); - assert_eq!(dag.refresh_window(view_id), window); -} diff --git a/crates/types/tests/physical_export.rs b/crates/types/tests/physical_export.rs index e2f701ffb..10a46370a 100644 --- a/crates/types/tests/physical_export.rs +++ b/crates/types/tests/physical_export.rs @@ -1,18 +1,13 @@ -//! A timed plan exports as a PhysicalASAPDAG that keeps timing. +//! A plan with its nodes' physical properties exports as a PhysicalASAPDAG +//! that keeps them. use asap_types::ir::operator::{Reduction, Source}; -use asap_types::ir::physical_export::{ - compile_physical_asap_dag, compile_physical_asap_workload, PhysicalASAPDAG, - PhysicalASAPDAGValidationError, -}; -use asap_types::ir::properties::{ExecutionDataStateError, ExecutionTiming}; +use asap_types::ir::physical_export::{PhysicalASAPDAG, PhysicalASAPDAGValidationError}; +use asap_types::ir::properties::ExecutionTiming; use asap_types::ir::scalar::ColumnRef; use asap_types::ir::schema::{DataType, Field, FieldDataType, Schema}; use asap_types::ir::schema::{ExactKind, ExactParams, SummaryUpdate}; -use asap_types::ir::{ - apply_materialization_timings, ASAPOp, MaterializationAssignment, NodeToTimedNodeCache, - NonASAPOp, Operator, OperatorNode, QueryRoot, -}; -use asap_types::physical::{Materialization, Medium}; +use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode, QueryRoot}; +use asap_types::physical::{Lifetime, PhysicalPropertiesError, StorageMedium}; use std::rc::Rc; /// Scan(t) → SummaryAgg(sum by key) → FinalizeExactAccumulator, untimed. @@ -43,107 +38,22 @@ fn plan() -> Rc { .unwrap() } -const ON_DISK: Materialization = Materialization::MatAtIngestion(Medium::Disk); - -/// `plan()` timed with its summary `MatAtIngestion` on disk. -fn timed_at_ingestion() -> Rc { - let root = plan(); - let mut assignment = MaterializationAssignment::default(); - assignment.set(root.children()[0], ON_DISK); - apply_materialization_timings(&root, &assignment, &mut NodeToTimedNodeCache::new()).unwrap() -} - -/// A summary materialized at ingestion time is read at query time, and the -/// exported DAG says where to store it. -#[test] -fn timed_plan_exports_with_timing() { - let timed = timed_at_ingestion(); - let dag = compile_physical_asap_dag(&timed).unwrap(); - dag.validate().unwrap(); - - let timings: Vec<_> = dag.nodes.iter().map(|n| n.output_state.timing).collect(); - assert_eq!( - timings, - vec![ - ExecutionTiming::IngestionTime, - ExecutionTiming::IngestionTime, - ExecutionTiming::QueryTime - ] - ); - let materializations: Vec<_> = dag.nodes.iter().map(|n| n.materialization).collect(); - assert_eq!( - materializations, - vec![ - Materialization::NotMaterialized, - ON_DISK, - Materialization::NotMaterialized - ] - ); - // Timing keeps what the summary covers. - assert!(timed.children()[0].coverage().is_some()); - - let decoded: PhysicalASAPDAG = - serde_json::from_str(&serde_json::to_string(&dag).unwrap()).unwrap(); - assert_eq!(decoded, dag); -} - -/// A query-time producer cannot feed an ingestion-time consumer. -#[test] -fn query_time_input_to_ingestion_is_rejected() { - let mut dag = compile_physical_asap_dag(&timed_at_ingestion()).unwrap(); - dag.nodes[0].output_state.timing = ExecutionTiming::QueryTime; - *dag.edges[0].data_state_mut() = dag.nodes[0].output_state; - assert!(matches!( - dag.validate(), - Err(PhysicalASAPDAGValidationError::QueryDependencyInIngestion { .. }) - )); -} - -/// A summary's timing must be the one its materialization implies: the -/// deployment cannot run a `MatAtIngestion` state at query time. -#[test] -fn materialization_inconsistent_with_timing_is_rejected() { - let mut dag = compile_physical_asap_dag(&timed_at_ingestion()).unwrap(); - dag.nodes[1].output_state.timing = ExecutionTiming::QueryTime; - *dag.edges[1].data_state_mut() = dag.nodes[1].output_state; - assert_eq!( - dag.validate(), - Err( - PhysicalASAPDAGValidationError::MaterializationTimingMismatch { - node: 1, - timing: ExecutionTiming::QueryTime, - materialization: ON_DISK, - } - ) - ); - dag.nodes[1].materialization = Materialization::MatAtQuery(Medium::Disk); - dag.validate().unwrap(); -} - -/// Untimed plans cannot be exported as physical plans. -#[test] -fn untimed_plan_is_rejected() { - assert!(compile_physical_asap_dag(&plan()).is_err()); -} - /// Two queries reading one summary state export once, with one root per query. #[test] fn batch_shares_the_summary_and_keeps_one_root_per_query() { - use asap_types::ir::physical_export::compile_physical_asap_workload; let first = plan(); let state = first.children()[0].clone(); let second = OperatorNode::new_shared(Operator::ASAP(ASAPOp::FinalizeExactAccumulator { child: state, })) .unwrap(); - let assignment = MaterializationAssignment::default(); - let mut cache = NodeToTimedNodeCache::new(); - let timed: Vec<_> = [first, second] - .iter() - .map(|root| apply_materialization_timings(root, &assignment, &mut cache).unwrap()) + let roots: Vec = [first, second] + .into_iter() + .map(QueryRoot::Operator) .collect(); - let roots: Vec = timed.into_iter().map(QueryRoot::Operator).collect(); - let dag = compile_physical_asap_workload(&roots).unwrap(); + let dag = asap_types::test_support::physical_asap_dag_at_query_time_with_node_ids(&roots) + .map(|(dag, _)| dag) + .unwrap(); dag.validate().unwrap(); assert_eq!(dag.roots.len(), 2); assert_eq!(dag.nodes.len(), 4, "scan and summary are exported once"); @@ -155,22 +65,18 @@ fn batch_shares_the_summary_and_keeps_one_root_per_query() { fn scalar_roots_export_in_batch_order_and_reference_nodes() { use asap_types::ir::scalar::ArithmeticOpKind; use asap_types::ir::{ExprSemantics, ScalarExpr}; - let timed = apply_materialization_timings( - &plan(), - &MaterializationAssignment::default(), - &mut NodeToTimedNodeCache::new(), - ) - .unwrap(); + let read = plan(); let scalar = ScalarExpr::Arithmetic { op: ArithmeticOpKind::Add, - left: Box::new(ScalarExpr::PromqlScalarFromVector(Rc::clone(&timed))), + left: Box::new(ScalarExpr::PromqlScalarFromVector(Rc::clone(&read))), right: Box::new(ScalarExpr::literal_f64(1.0)), semantics: ExprSemantics::Promql, }; - let dag = compile_physical_asap_workload(&[ + let dag = asap_types::test_support::physical_asap_dag_at_query_time_with_node_ids(&[ QueryRoot::Scalar(scalar), - QueryRoot::Operator(Rc::clone(&timed)), + QueryRoot::Operator(Rc::clone(&read)), ]) + .map(|(dag, _)| dag) .unwrap(); assert_eq!(dag.nodes.len(), 3, "the read plan is exported once"); assert_eq!(dag.roots.len(), 1); @@ -187,7 +93,7 @@ fn scalar_roots_export_in_batch_order_and_reference_nodes() { scalar_only.roots.clear(); scalar_only.validate().unwrap(); let valid_scalar_only = scalar_only.clone(); - // Materialization is checked on the nodes a scalar reads, too. + // Physical properties are checked on the nodes a scalar reads, too. let state = scalar_only .nodes .iter() @@ -200,16 +106,14 @@ fn scalar_roots_export_in_batch_order_and_reference_nodes() { ) }) .unwrap(); - scalar_only.nodes[state].materialization = ON_DISK; + scalar_only.nodes[state].output_state.timing = ExecutionTiming::IngestionTime; + scalar_only.nodes[state].lifetime = Lifetime::OneEvaluation(StorageMedium::Disk); assert_eq!( scalar_only.validate(), - Err( - PhysicalASAPDAGValidationError::MaterializationTimingMismatch { - node: scalar_only.nodes[state].id, - timing: ExecutionTiming::QueryTime, - materialization: ON_DISK, - } - ) + Err(PhysicalASAPDAGValidationError::PhysicalProperties { + node: scalar_only.nodes[state].id, + reason: PhysicalPropertiesError::OneEvaluationAtIngestion, + }) ); let mut missing = dag.clone(); @@ -276,13 +180,7 @@ fn a_reference_read_twice_is_one_scalar_ref_edge() { child: series("a"), })) .unwrap(); - let timed = apply_materialization_timings( - &project, - &MaterializationAssignment::default(), - &mut NodeToTimedNodeCache::new(), - ) - .unwrap(); - let dag = compile_physical_asap_dag(&timed).unwrap(); + let dag = asap_types::test_support::physical_asap_dag_at_query_time(&project).unwrap(); dag.validate().unwrap(); let roles: Vec<_> = dag .edges @@ -318,13 +216,7 @@ fn swapped_edge_roles_are_rejected() { right: scan("b"), })) .unwrap(); - let timed = apply_materialization_timings( - &join, - &MaterializationAssignment::default(), - &mut NodeToTimedNodeCache::new(), - ) - .unwrap(); - let mut dag = compile_physical_asap_dag(&timed).unwrap(); + let mut dag = asap_types::test_support::physical_asap_dag_at_query_time(&join).unwrap(); dag.validate().unwrap(); for edge in &mut dag.edges { edge.role = match edge.role { @@ -345,141 +237,12 @@ fn swapped_edge_roles_are_rejected() { #[test] fn an_empty_batch_is_an_error() { assert!(matches!( - compile_physical_asap_workload(&[]), - Err(ExecutionDataStateError::InvalidPhysicalDAG( - PhysicalASAPDAGValidationError::NoRoots - )) + asap_types::test_support::physical_asap_dag_at_query_time_with_node_ids(&[]) + .map(|(dag, _)| dag), + Err(PhysicalASAPDAGValidationError::NoRoots) )); } -/// A summary over another summary exports a typed maintenance edge between -/// them, and the document round-trips and rejects unknown fields and versions. -#[test] -fn exports_summary_over_summary_as_typed_precompute_edges() { - use asap_types::ir::physical_export::{GroupingEdgeCompatibility, WindowEdgeCompatibility}; - use asap_types::ir::properties::ExecutionDataState; - use asap_types::test_support::compile_physical_asap_dag_with_node_ids; - let scan = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Scan { - source: Source::TimeSeries { metric: "m".into() }, - predicates: vec![], - schema: Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("value", DataType::Float64, false), - ], - 0, - vec![], - ), - })) - .unwrap(); - let sum = |child, column| { - OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryAgg { - child, - family: FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), - input: SummaryUpdate::column(column), - reduction: Reduction::by(vec![]), - grouping: Default::default(), - filter: None, - })) - .unwrap() - }; - let inner = sum(scan, ColumnRef::SampleValue); - let outer = sum(Rc::clone(&inner), ColumnRef::Named("state".into())); - let root = OperatorNode::new_shared(Operator::ASAP(ASAPOp::FinalizeExactAccumulator { - child: outer, - })) - .unwrap(); - let mut cache = NodeToTimedNodeCache::new(); - let at_ingestion = - MaterializationAssignment::with_default(Materialization::MatAtIngestion(Medium::Memory)); - let timed = apply_materialization_timings(&root, &at_ingestion, &mut cache).unwrap(); - let compiled = compile_physical_asap_dag_with_node_ids(&timed).unwrap(); - assert_eq!(compiled.node_ids.node_id(&timed), Some(3)); - assert_eq!( - compiled - .node_ids - .node_id(asap_types::test_support::timed(&cache, &inner).unwrap()), - Some(1) - ); - let dag = compiled.dag; - assert_eq!(dag.roots, vec![3]); - assert_eq!( - dag.nodes[1].output_state, - ExecutionDataState::INGESTION_SUMMARY - ); - assert_eq!( - dag.nodes[2].output_state, - ExecutionDataState::INGESTION_SUMMARY - ); - let dependency = dag - .edges - .iter() - .find(|e| e.producer == 1 && e.consumer == 2) - .unwrap(); - assert_eq!( - dependency.data_state(), - ExecutionDataState::INGESTION_SUMMARY - ); - assert_eq!(dependency.grouping(), GroupingEdgeCompatibility::Identical); - assert_eq!( - dependency.window(), - WindowEdgeCompatibility::RequiresAlignedTumblingWindowPhaseOrExactWindowEdgeResidual - ); - assert!(matches!( - dependency.intermediate_schema().fields[0].dtype, - FieldDataType::ExactAggregate(ExactKind::Sum, _) - )); - assert!(matches!( - dag.nodes[2].payload, - Operator::ASAP(ASAPOp::SummaryAgg { - family: FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), - reduction: Reduction::Reduce(_), - .. - }) - )); - - let decoded: PhysicalASAPDAG = - serde_json::from_str(&serde_json::to_string(&dag).unwrap()).unwrap(); - assert_eq!(decoded, dag); - decoded.validate().unwrap(); - let mut unknown = serde_json::to_value(&dag).unwrap(); - unknown["unexpected"] = serde_json::json!(true); - assert!(serde_json::from_value::(unknown).is_err()); -} - -/// `retention_ms` is accepted on a materialized view (a node kept across -/// evaluations), survives a round trip, and is rejected on a node that is -/// neither a view nor a query-time scan (here an ingestion-time node). -#[test] -fn retention_is_only_on_materialized_views_and_query_time_scans() { - let mut dag = compile_physical_asap_dag(&timed_at_ingestion()).unwrap(); - let view = dag - .nodes - .iter() - .position(|node| asap_types::test_support::is_materialized_view(node.materialization)) - .expect("a node built at ingestion"); - let other = dag - .nodes - .iter() - .position(|node| { - !asap_types::test_support::is_materialized_view(node.materialization) - && node.output_state.timing == ExecutionTiming::IngestionTime - }) - .expect("an ingestion-time node that is not a view"); - dag.nodes[view].retention_ms = Some(60_000); - dag.validate().unwrap(); - let decoded: PhysicalASAPDAG = - serde_json::from_str(&serde_json::to_string(&dag).unwrap()).unwrap(); - assert_eq!(decoded, dag); - dag.nodes[other].retention_ms = Some(60_000); - assert_eq!( - dag.validate(), - Err(PhysicalASAPDAGValidationError::UnexpectedRetention( - dag.nodes[other].id - )) - ); -} - /// Node-id-keyed bindings serialize in numeric id order. #[test] fn physical_node_ids_serialize_in_deterministic_binding_order() { @@ -493,41 +256,6 @@ fn physical_node_ids_serialize_in_deterministic_binding_order() { ); } -/// The plan with every node at query time is the same operators with -/// nothing materialized: what the default materialization exports. The -/// shared summary stays one node, and the scalar root that reads it keeps -/// its batch position. -#[test] -fn every_node_at_query_time_keeps_the_operators_and_stores_nothing() { - use asap_types::ir::scalar::ArithmeticOpKind; - use asap_types::ir::{ExprSemantics, ScalarExpr}; - let workload = |root: Rc| { - let scalar = ScalarExpr::Arithmetic { - op: ArithmeticOpKind::Add, - left: Box::new(ScalarExpr::PromqlScalarFromVector(Rc::clone(&root))), - right: Box::new(ScalarExpr::literal_f64(1.0)), - semantics: ExprSemantics::Promql, - }; - compile_physical_asap_workload(&[ - QueryRoot::Operator(Rc::clone(&root)), - QueryRoot::Scalar(scalar), - QueryRoot::Operator(root), - ]) - .unwrap() - }; - let materialized = workload(timed_at_ingestion()); - let at_query_time = apply_materialization_timings( - &plan(), - &MaterializationAssignment::default(), - &mut NodeToTimedNodeCache::new(), - ) - .unwrap(); - assert_eq!( - materialized.with_every_node_at_query_time().unwrap(), - workload(at_query_time) - ); -} - /// A window 600,000 ms wide ending 300,000 ms before T = 3,600,000 covers /// `(2,700,000, 3,300,000]`, and reaches 900,000 ms back. #[test] @@ -572,17 +300,10 @@ fn exported_with_scan( asap_types::ir::physical_export::PhysicalASAPDAG, asap_types::ir::physical_export::PhysicalASAPNodeId, ) { - // One timing cache, so the roots keep sharing their `Scan`. - let mut cache = NodeToTimedNodeCache::new(); - let timed: Vec<_> = roots - .into_iter() - .map(|root| { - apply_materialization_timings(&root, &MaterializationAssignment::default(), &mut cache) - .unwrap() - }) - .map(QueryRoot::Operator) - .collect(); - let dag = compile_physical_asap_workload(&timed).unwrap(); + let roots: Vec = roots.into_iter().map(QueryRoot::Operator).collect(); + let dag = asap_types::test_support::physical_asap_dag_at_query_time_with_node_ids(&roots) + .map(|(dag, _)| dag) + .unwrap(); let scans: Vec<_> = dag .nodes .iter() @@ -712,7 +433,8 @@ fn high_fan_in_edges_preserve_multiplicity_without_order_dependence() { schema: schema.clone(), }), output_state: ExecutionDataState::QUERY_ROWS, - materialization: Materialization::NotMaterialized, + lifetime: Lifetime::None, + parameters: None, output_schema: schema.clone(), guarantee: None, retention_ms: None, @@ -725,7 +447,8 @@ fn high_fan_in_edges_preserve_multiplicity_without_order_dependence() { children: inputs.clone(), }), output_state: ExecutionDataState::QUERY_ROWS, - materialization: Materialization::NotMaterialized, + lifetime: Lifetime::None, + parameters: None, output_schema: schema.clone(), guarantee: None, retention_ms: None, @@ -764,16 +487,3 @@ fn high_fan_in_edges_preserve_multiplicity_without_order_dependence() { Err(PhysicalASAPDAGValidationError::EdgeRoleMismatch { .. }) )); } - -/// A workload export of one timed root is that root's export, and an -/// untimed root is invalid. -#[test] -fn workload_export_matches_the_single_root_export() { - use asap_types::ir::physical_export::compile_physical_asap_workload_with_node_ids; - let timed = timed_at_ingestion(); - let roots = [QueryRoot::Operator(Rc::clone(&timed))]; - let exported = compile_physical_asap_workload_with_node_ids(&roots).unwrap(); - assert_eq!(exported.dag, compile_physical_asap_dag(&timed).unwrap()); - let untimed = [QueryRoot::Operator(plan())]; - assert!(compile_physical_asap_workload_with_node_ids(&untimed).is_err()); -} diff --git a/crates/types/tests/schema_rebuilding.rs b/crates/types/tests/schema_rebuilding.rs index 6e61144a8..611b0b316 100644 --- a/crates/types/tests/schema_rebuilding.rs +++ b/crates/types/tests/schema_rebuilding.rs @@ -1,5 +1,5 @@ use asap_types::ir::operator::{AggIntent, Reduction, Source}; -use asap_types::ir::properties::{AccuracyGuarantee, ExecutionTiming}; +use asap_types::ir::properties::AccuracyGuarantee; use asap_types::ir::scalar::ColumnRef; use asap_types::ir::schema::{DataType, Field, FieldDataType, Schema}; use asap_types::ir::schema::{ExactKind, ExactParams, GroupingStrategy, SummaryUpdate}; @@ -67,8 +67,7 @@ fn rebuilding_preserves_only_explicit_naming_overrides() { schema.fields[0].name = "alias".into(); schema.fields[0].table = Some("result".into()); let renamed = OperatorNode::with_schema(original.operator.clone(), schema) - .with_guarantee(Some(AccuracyGuarantee::exact("fixture"))) - .with_timing(Some(ExecutionTiming::QueryTime)); + .with_guarantee(Some(AccuracyGuarantee::exact("fixture"))); let original = Rc::new(renamed); original.validate_structure().unwrap(); let replacement = scan(DataType::Utf8, "new_key"); @@ -80,7 +79,6 @@ fn rebuilding_preserves_only_explicit_naming_overrides() { Some(&DataType::Utf8) ); assert!(rebuilt.guarantee.is_none()); - assert!(rebuilt.timing.is_none()); rebuilt.validate_structure().unwrap(); } } @@ -229,50 +227,6 @@ fn series_selector() -> Rc { .unwrap() } -/// `topk(5, a)` read from a current-series population of `a`, built by hand. -fn current_series_topk() -> Rc { - use asap_types::ir::operator::maintained_population::*; - let selector = series_selector(); - let population = OperatorNode::new_shared(Operator::ASAP(ASAPOp::MaintainPopulation { - child: selector, - population: MaintainedPopulation { - input: PopulationInput::CurrentSeries(CurrentSeriesInput { - metric: "a".into(), - matchers: vec![], - grouping: vec![], - without: false, - lookback_ms: 1_000, - }), - max_k: 5, - quantiles: false, - }, - })) - .unwrap(); - OperatorNode::new_shared(Operator::ASAP(ASAPOp::EvaluatePopulation { - child: population, - evaluation: PopulationStatistic::TopK { k: 5 }, - })) - .unwrap() -} - -/// Time `root` under the default materialization assignment, which runs the -/// data-state and population-contract validation, and export it. -fn compile_physical_asap_dag(root: &Rc) -> Result<(), String> { - use asap_types::ir::{ - apply_materialization_timings, MaterializationAssignment, NodeToTimedNodeCache, - }; - root.validate_structure().map_err(|e| e.to_string())?; - let timed = apply_materialization_timings( - root, - &MaterializationAssignment::default(), - &mut NodeToTimedNodeCache::new(), - ) - .map_err(|e| format!("{e:?}"))?; - asap_types::ir::physical_export::compile_physical_asap_dag(&timed) - .map_err(|e| format!("{e:?}"))?; - Ok(()) -} - /// A table-row population maintains a numeric value column: one that names /// a label column fails validation. #[test] @@ -331,80 +285,3 @@ fn a_current_series_population_matches_only_its_selectors_lookback() { }; assert!(!asap_types::test_support::matches_node(&wrong, &selector)); } - -/// Population timing is a materialization choice: the population follows its -/// assignment (`NotMaterialized` by default), while its evaluation always -/// runs at query time. -#[test] -fn population_timing_follows_its_materialization() { - use asap_types::ir::physical_export::compile_physical_asap_dag as export_timed; - use asap_types::ir::properties::ExecutionTiming::{IngestionTime, QueryTime}; - use asap_types::ir::{ - apply_materialization_timings, MaterializationAssignment, NodeToTimedNodeCache, - }; - use asap_types::physical::{Materialization, Medium}; - let candidate = current_series_topk(); - let Some(ASAPOp::EvaluatePopulation { - child: population, .. - }) = candidate.asap() - else { - unreachable!() - }; - let on_disk = Materialization::MatAtIngestion(Medium::Disk); - let mut at_ingestion = MaterializationAssignment::default(); - at_ingestion.set(population, on_disk); - for (assignment, timing, materialization) in [ - ( - MaterializationAssignment::default(), - QueryTime, - Materialization::NotMaterialized, - ), - (at_ingestion, IngestionTime, on_disk), - ] { - let timed = apply_materialization_timings( - &candidate, - &assignment, - &mut NodeToTimedNodeCache::new(), - ) - .unwrap(); - let dag = export_timed(&timed).unwrap(); - let [population, evaluation] = &dag.nodes[dag.nodes.len() - 2..] else { - unreachable!() - }; - assert_eq!(population.output_state.timing, timing); - assert_eq!(population.materialization, materialization); - assert_eq!(evaluation.output_state.timing, QueryTime); - } - let mut evaluation_at_ingestion = (*candidate).clone(); - evaluation_at_ingestion.timing = Some(IngestionTime); - assert!(compile_physical_asap_dag(&Rc::new(evaluation_at_ingestion)).is_err()); -} - -/// An evaluation cannot exceed its population's contract (`max_k`), and a -/// population cannot claim a source it does not read. -#[test] -fn malformed_population_dags_fail_closed() { - use asap_types::ir::operator::maintained_population::{PopulationInput, PopulationStatistic}; - let candidate = current_series_topk(); - compile_physical_asap_dag(&candidate).expect("the unmodified candidate is legal"); - let mut bad = (*candidate).clone(); - let Operator::ASAP(ASAPOp::EvaluatePopulation { evaluation, .. }) = &mut bad.operator else { - unreachable!() - }; - *evaluation = PopulationStatistic::TopK { k: 6 }; - assert!(compile_physical_asap_dag(&Rc::new(bad.clone())).is_err()); - let Operator::ASAP(ASAPOp::EvaluatePopulation { child, evaluation }) = &mut bad.operator else { - unreachable!() - }; - *evaluation = PopulationStatistic::TopK { k: 5 }; - let producer = Rc::make_mut(child); - let Operator::ASAP(ASAPOp::MaintainPopulation { population, .. }) = &mut producer.operator - else { - unreachable!() - }; - let PopulationInput::CurrentSeries(spec) = &mut population.input else { - unreachable!() - }; - spec.metric = "b".into(); - assert!(compile_physical_asap_dag(&Rc::new(bad)).is_err()); -} diff --git a/crates/types/tests/structure_contract.rs b/crates/types/tests/structure_contract.rs index 0c16614cf..47ce849bc 100644 --- a/crates/types/tests/structure_contract.rs +++ b/crates/types/tests/structure_contract.rs @@ -90,20 +90,10 @@ fn state_evaluations_and_passthrough_keep_their_contracts() { use asap_types::ir::operator::Reduction; use asap_types::ir::scalar::ColumnRef; use asap_types::ir::schema::{ - FieldDataType, GroupingStrategy, SketchAlgorithm, SketchKind, SketchParams, - SketchStatistic, SummaryUpdate, + FieldDataType, GroupingStrategy, SketchAlgorithm, SketchStatistic, SummaryUpdate, }; use asap_types::ir::{ASAPOp, Operator, ProjectItem}; - let family = FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: 100, - failure_probability: None, - }, - ), - GroupingStrategy::default(), - ); + let family = FieldDataType::Sketch(SketchAlgorithm::Kll, GroupingStrategy::default()); let state = Rc::new( OperatorNode::new(Operator::ASAP(ASAPOp::SummaryAgg { child: scan(), @@ -127,7 +117,9 @@ fn state_evaluations_and_passthrough_keep_their_contracts() { .unwrap(); pass.validate_structure().unwrap(); assert_eq!(pass.result_kind, OperatorResultKind::State); - assert!(ScalarExpr::Column(0).scalar_type(&pass.schema).is_err()); + assert!(ScalarExpr::>::Column(0) + .scalar_type(&pass.schema) + .is_err()); assert!(validate_asap_inputs(&ASAPOp::SummaryEstimate { summary_input: state.clone(), query: SketchStatistic::Cardinality @@ -141,36 +133,6 @@ fn state_evaluations_and_passthrough_keep_their_contracts() { assert!(validate_asap_inputs(&ASAPOp::FinalizeExactAccumulator { child: state }).is_err()); } -/// Phase validation checks dependencies, without declaring a computation query-only. -#[test] -fn execution_timing_checks_edges_not_function_names() { - use asap_types::ir::properties::ExecutionTiming::{IngestionTime, QueryTime}; - use asap_types::ir::ProjectItem; - let input = Rc::new((*scan()).clone().with_timing(Some(IngestionTime))); - let mut project = OperatorNode::new(asap_types::ir::Operator::NonASAP(NonASAPOp::Project { - child: input, - cols: vec![ProjectItem { - expr: ScalarExpr::FunctionCall { - name: "promql_abs".into(), - args: vec![ScalarExpr::Column(0)], - }, - alias: None, - }], - qualifier: None, - })) - .unwrap() - .with_timing(Some(IngestionTime)); - Rc::new(project.clone()) - .validate_execution_timing() - .unwrap(); - if let asap_types::ir::Operator::NonASAP(NonASAPOp::Project { child, .. }) = - &mut project.operator - { - *child = Rc::new(child.as_ref().clone().with_timing(Some(QueryTime))); - } - assert!(Rc::new(project).validate_execution_timing().is_err()); -} - /// Both operator categories use the same fallible schema-deriving constructor. #[test] fn shared_construction_derives_both_operator_categories() { @@ -195,7 +157,6 @@ fn shared_construction_derives_both_operator_categories() { assert_eq!(state.result_kind, OperatorResultKind::State); assert!(!state.schema.fields.last().unwrap().is_plain()); assert!(state.guarantee.is_none()); - assert!(state.timing.is_none()); state.validate_structure().unwrap(); assert!( diff --git a/crates/types/tests/summary_coverage.rs b/crates/types/tests/summary_coverage.rs deleted file mode 100644 index 07357264b..000000000 --- a/crates/types/tests/summary_coverage.rs +++ /dev/null @@ -1,1096 +0,0 @@ -//! Summary coverage is `definition + selection`, derived from the sub-DAG -//! (`docs/design_docs/proposals/asap-primitive-schema.md` §4.2). Merges need -//! the same definition and disjoint selections. -use std::ops::Bound; -use std::rc::Rc; -use std::time::Duration; - -use asap_types::ir::operator::operator_properties::{Reduction, Source}; -use asap_types::ir::operator::TimeShift; -use asap_types::ir::properties::summary_coverage::{ - ColumnIdentity, Constraint, CoverageError, SelectionBox, -}; -use asap_types::ir::properties::ExecutionTiming; -use asap_types::ir::scalar::expr_ir::{ArithmeticOpKind, CompareOpKind, ScalarValue}; -use asap_types::ir::scalar::ColumnRef; -use asap_types::ir::schema::{DataType, Field, FieldDataType, Schema}; -use asap_types::ir::schema::{ - ExactKind, ExactParams, GroupingStrategy, MergeInputRule, SketchAlgorithm, SketchKind, - SketchParams, SummaryUpdate, -}; -use asap_types::ir::{ - ASAPOp, ExprSemantics, NonASAPOp, Operator, OperatorNode, OperatorResultKind, Predicate, - ProjectItem, ScalarExpr, SchemaDerivationError, TimeRangeKind, -}; -use asap_types::test_support::definition_id; - -const JOB: usize = 0; -const REGION: usize = 1; -const LATENCY: usize = 2; - -fn table() -> Rc { - table_with(vec![ - Field::plain("job", DataType::Utf8, false), - Field::plain("region", DataType::Utf8, false), - Field::plain("latency", DataType::Float64, false), - Field::plain("size", DataType::Float64, false), - ]) -} - -fn table_with(fields: Vec) -> Rc { - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Scan { - source: Source::Table { - table_ref: "t".into(), - }, - predicates: vec![], - schema: Schema::new(fields), - })) - .unwrap() -} - -/// PromQL leaf `m{predicates}` with columns `[ts, value, job]`. -fn series(predicates: Vec) -> Rc { - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Scan { - source: Source::TimeSeries { metric: "m".into() }, - predicates, - schema: Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("value", DataType::Float64, false), - Field::plain("job", DataType::Utf8, true), - ], - 0, - vec![], - ), - })) - .unwrap() -} - -fn compare(column: usize, op: CompareOpKind, value: ScalarValue) -> ScalarExpr { - ScalarExpr::Compare { - left: Box::new(ScalarExpr::Column(column)), - op, - right: Box::new(ScalarExpr::Literal(value)), - semantics: ExprSemantics::Sql, - } -} - -fn eq(column: usize, value: &str) -> ScalarExpr { - compare(column, CompareOpKind::Eq, ScalarValue::Utf8(value.into())) -} - -fn filter(child: Rc, pred: ScalarExpr) -> Rc { - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Filter { - pred: Predicate(pred), - child, - })) - .unwrap() -} - -fn kll(child: Rc, column: &str) -> Rc { - summary( - child, - column, - FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: 200, - failure_probability: None, - }, - ), - Default::default(), - ), - ) -} - -fn summary(child: Rc, column: &str, family: FieldDataType) -> Rc { - OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryAgg { - child, - family, - input: SummaryUpdate::column(ColumnRef::Named(column.into())), - reduction: Reduction::by(vec![JOB]), - grouping: GroupingStrategy::default(), - filter: None, - })) - .unwrap() -} - -/// KLL of latency by job over `t` restricted by `pred`. -fn latency_where(pred: ScalarExpr) -> Rc { - kll(filter(table(), pred), "latency") -} - -/// Tumbling window: `TimeRange(1m)` over `TimeShift(shift)` over `m`. -fn tumbling_window(shift_ms: i64, predicates: Vec) -> Rc { - kll(tumbling_window_rows(shift_ms, predicates), "value") -} - -/// Exact counter-increase tumbling window over the same rows as [`tumbling_window`]. -fn increase_tumbling_window(shift_ms: i64, predicates: Vec) -> Rc { - summary( - tumbling_window_rows(shift_ms, predicates), - "value", - FieldDataType::ExactAggregate(ExactKind::Increase, ExactParams::Increase), - ) -} - -/// KLL over `TimeRange(width_ms)` over `TimeShift(shift_ms)` over `m`: the -/// window `(-(shift + width), -shift]`. -fn window(shift_ms: i64, width_ms: u64) -> Rc { - let shifted = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeShift { - shift: TimeShift { - offset_ms: shift_ms, - at: None, - }, - child: series(vec![]), - })) - .unwrap(); - let range = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeRange { - range: Duration::from_millis(width_ms), - kind: TimeRangeKind::Range, - child: shifted, - })) - .unwrap(); - kll(range, "value") -} - -fn tumbling_window_rows(shift_ms: i64, predicates: Vec) -> Rc { - let shifted = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeShift { - shift: TimeShift { - offset_ms: shift_ms, - at: None, - }, - child: series(predicates), - })) - .unwrap(); - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeRange { - range: Duration::from_millis(60_000), - kind: TimeRangeKind::Range, - child: shifted, - })) - .unwrap() -} - -fn merge(children: Vec>) -> Result, SchemaDerivationError> { - OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryMerge { - children, - newest_tumbling_windows: None, - group_by: None, - })) -} - -fn rejected(children: Vec>) -> CoverageError { - match merge(children) { - Err(SchemaDerivationError::Coverage(error)) => error, - other => panic!("expected a coverage error, got {other:?}"), - } -} - -fn column(name: &str) -> ColumnIdentity { - ColumnIdentity::new(None, name) -} - -fn utf8(values: &[&str]) -> Constraint { - Constraint::In( - values - .iter() - .map(|v| ScalarValue::Utf8((*v).into())) - .collect(), - ) -} - -fn relative(from_ms: i64, to_ms: i64) -> Option<(Bound, Bound)> { - Some((Bound::Excluded(from_ms), Bound::Included(to_ms))) -} - -/// A lifted filter leaves the definition over the bare scan. -#[test] -fn filters_lift_into_the_selection() { - let scan = table(); - let us = kll(filter(Rc::clone(&scan), eq(REGION, "us")), "latency"); - let coverage = us.coverage().unwrap(); - assert!(Rc::ptr_eq(coverage.definition().children()[0], &scan)); - assert_eq!( - coverage.selection(), - vec![SelectionBox::new( - [(column("region"), utf8(&["us"]))].into(), - None - )] - ); -} - -/// Different populations merge into one box; the same population twice -/// would count every row twice. -#[test] -fn populations_merge_when_disjoint() { - let merged = merge(vec![ - latency_where(eq(REGION, "us")), - latency_where(eq(REGION, "eu")), - ]) - .unwrap(); - assert_eq!( - merged.coverage().unwrap().selection()[0].columns()[&column("region")], - utf8(&["us", "eu"]) - ); - assert_eq!( - rejected(vec![ - latency_where(eq(REGION, "us")), - latency_where(eq(REGION, "us")), - ]), - CoverageError::InputsOverlap { - required: MergeInputRule::Disjoint, - } - ); -} - -/// Equal schemas do not make a KLL over latency and one over size mergeable. -#[test] -fn different_inputs_do_not_merge() { - assert_eq!( - rejected(vec![ - kll(filter(table(), eq(REGION, "us")), "latency"), - kll(filter(table(), eq(REGION, "eu")), "size"), - ]), - CoverageError::DefinitionMismatch - ); -} - -/// The same update over scans of different columns is a different computation. -#[test] -fn the_same_update_over_different_scan_schemas_does_not_merge() { - let scan = |measure: &str| { - table_with(vec![ - Field::plain("job", DataType::Utf8, false), - Field::plain("region", DataType::Utf8, false), - Field::plain(measure, DataType::Float64, false), - Field::plain("value", DataType::Float64, false), - ]) - }; - assert_eq!( - rejected(vec![ - kll(filter(scan("latency"), eq(REGION, "us")), "value"), - kll(filter(scan("size"), eq(REGION, "eu")), "value"), - ]), - CoverageError::DefinitionMismatch - ); -} - -/// `shipping.region` and `billing.region` are different columns, so -/// restricting each says nothing about overlap. -#[test] -fn qualified_columns_stay_distinct() { - let scan = || { - let qualified = |table: &str| Field { - table: Some(table.into()), - ..Field::plain("region", DataType::Utf8, false) - }; - table_with(vec![ - Field::plain("job", DataType::Utf8, false), - qualified("shipping"), - qualified("billing"), - Field::plain("latency", DataType::Float64, false), - ]) - }; - assert_eq!( - rejected(vec![ - kll(filter(scan(), eq(1, "us")), "latency"), - kll(filter(scan(), eq(2, "eu")), "latency"), - ]), - CoverageError::InputsOverlap { - required: MergeInputRule::Disjoint, - } - ); -} - -/// Tumbling windows take their time from `TimeRange` over `TimeShift`: adjacent tumbling windows -/// join, a gap stays two boxes, and the same tumbling window twice overlaps. -#[test] -fn time_tumbling_windows_merge_and_keep_gaps() { - let first = tumbling_window(0, vec![]); - assert_eq!( - first.coverage().unwrap().selection()[0].relative_time(), - relative(-60_000, 0) - ); - assert!(Rc::ptr_eq( - first.coverage().unwrap().definition().children()[0], - first.children()[0].children()[0].children()[0] - )); - let joined = merge(vec![first.clone(), tumbling_window(60_000, vec![])]).unwrap(); - assert_eq!( - joined.coverage().unwrap().selection(), - vec![SelectionBox::new(Default::default(), relative(-120_000, 0))] - ); - let gapped = merge(vec![first.clone(), tumbling_window(120_000, vec![])]).unwrap(); - assert_eq!(gapped.coverage().unwrap().selection().len(), 2); - assert_eq!( - rejected(vec![first.clone(), first]), - CoverageError::InputsOverlap { - required: MergeInputRule::Disjoint, - } - ); -} - -/// Counter tumbling-window merge only when their time windows are disjoint: states -/// disjoint by a label but over the same window would interleave in time. -#[test] -fn counter_tumbling_windows_merge_only_disjoint_in_time() { - let joined = merge(vec![ - increase_tumbling_window(0, vec![]), - increase_tumbling_window(60_000, vec![]), - ]) - .unwrap(); - assert_eq!( - joined.coverage().unwrap().selection()[0].relative_time(), - relative(-120_000, 0) - ); - let by_job = |job| increase_tumbling_window(0, vec![Predicate(eq(2, job))]); - assert_eq!( - rejected(vec![by_job("api"), by_job("web")]), - CoverageError::InputsOverlap { - required: MergeInputRule::DisjointInTime, - } - ); -} - -/// A day of one-minute tumbling windows (1,440 inputs) merges into one window, in -/// either order: the inputs' windows are sorted and only neighbours are -/// compared, so the check is O(n log n), not all pairs. A tumbling window listed -/// twice still overlaps. -#[test] -fn a_day_of_minute_tumbling_windows_merges_into_one_window() { - const TUMBLING_WINDOW_COUNT: i64 = 1_440; - let day = relative(-TUMBLING_WINDOW_COUNT * 60_000, 0); - let newest_first: Vec<_> = (0..TUMBLING_WINDOW_COUNT) - .map(|i| tumbling_window(i * 60_000, vec![])) - .collect(); - let oldest_first: Vec<_> = newest_first.iter().rev().cloned().collect(); - for tumbling_windows in [newest_first.clone(), oldest_first] { - let merged = merge(tumbling_windows).unwrap(); - merged.validate_structure().unwrap(); - let coverage = merged.coverage().unwrap(); - let selection = coverage.selection(); - assert_eq!(selection.len(), 1); - assert_eq!(selection[0].relative_time(), day); - } - let counters: Vec<_> = (0..TUMBLING_WINDOW_COUNT) - .map(|i| increase_tumbling_window(i * 60_000, vec![])) - .collect(); - assert_eq!( - merge(counters).unwrap().coverage().unwrap().selection()[0].relative_time(), - day - ); - let mut repeated = newest_first; - repeated.push(tumbling_window(30 * 60_000, vec![])); - assert_eq!( - rejected(repeated), - CoverageError::InputsOverlap { - required: MergeInputRule::Disjoint, - } - ); -} - -/// A long window that spans two short ones overlaps both, though the -/// short ones are apart: sorted by start, the long window's neighbour -/// overlaps it, so the all-pairs check runs and rejects the merge. The -/// short windows alone merge as two boxes. -#[test] -fn a_long_window_spanning_others_is_rejected() { - let long = window(0, 100_000); - let (near, far) = (window(10_000, 10_000), window(50_000, 10_000)); - assert_eq!( - rejected(vec![long.clone(), near.clone(), far.clone()]), - CoverageError::InputsOverlap { - required: MergeInputRule::Disjoint, - } - ); - assert_eq!( - rejected(vec![near.clone(), far.clone(), long]), - CoverageError::InputsOverlap { - required: MergeInputRule::Disjoint, - } - ); - let apart = merge(vec![far, near]).unwrap(); - assert_eq!( - apart - .coverage() - .unwrap() - .selection() - .iter() - .map(|b| b.relative_time()) - .collect::>(), - [relative(-60_000, -50_000), relative(-20_000, -10_000)] - ); -} - -/// Label matchers on the scan lift like any filter. -#[test] -fn scan_predicates_lift() { - let api = tumbling_window(0, vec![Predicate(eq(2, "api"))]); - let coverage = api.coverage().unwrap(); - assert_eq!( - coverage.selection()[0].columns()[&column("job")], - utf8(&["api"]) - ); - let scan = coverage.definition().children()[0]; - assert!(matches!( - scan.non_asap(), - Some(NonASAPOp::Scan { predicates, .. }) if predicates.is_empty() - )); - merge(vec![api, tumbling_window(0, vec![Predicate(eq(2, "web"))])]).unwrap(); -} - -/// Value ranges are selections: `latency < 100` and `latency >= 100` are -/// disjoint, `latency <= 100` overlaps `latency >= 100`. -#[test] -fn value_ranges_merge_when_disjoint() { - let latency = |op| latency_where(compare(LATENCY, op, ScalarValue::Float64(100.0))); - merge(vec![latency(CompareOpKind::Lt), latency(CompareOpKind::Ge)]).unwrap(); - assert_eq!( - rejected(vec![latency(CompareOpKind::Le), latency(CompareOpKind::Ge)]), - CoverageError::InputsOverlap { - required: MergeInputRule::Disjoint, - } - ); -} - -/// Touching value ranges join like touching time windows; the joined range -/// stays explicit, since neither input took NULL rows. A gap stays apart. -#[test] -fn touching_value_ranges_join() { - let latency = |op, v| latency_where(compare(LATENCY, op, ScalarValue::Float64(v))); - let joined = merge(vec![ - latency(CompareOpKind::Lt, 100.0), - latency(CompareOpKind::Ge, 100.0), - ]) - .unwrap(); - assert_eq!( - joined.coverage().unwrap().selection(), - vec![SelectionBox::new( - [( - column("latency"), - Constraint::Interval { - lower: Bound::Unbounded, - upper: Bound::Unbounded, - }, - )] - .into(), - None - )] - ); - let gap = merge(vec![ - latency(CompareOpKind::Lt, 100.0), - latency(CompareOpKind::Gt, 100.0), - ]) - .unwrap(); - assert_eq!(gap.coverage().unwrap().selection().len(), 2); -} - -/// A predicate that is not a value set or interval on one column stays in -/// the definition, so states with different residuals do not merge. -#[test] -fn residual_predicates_stay_in_the_definition() { - let doubled_above = |threshold: f64, region: &str| { - let doubled = ScalarExpr::Arithmetic { - op: ArithmeticOpKind::Mul, - left: Box::new(ScalarExpr::Column(LATENCY)), - right: Box::new(ScalarExpr::Literal(ScalarValue::Float64(2.0))), - semantics: ExprSemantics::Sql, - }; - latency_where(ScalarExpr::BoolAnd(vec![ - ScalarExpr::Compare { - left: Box::new(doubled), - op: CompareOpKind::Gt, - right: Box::new(ScalarExpr::Literal(ScalarValue::Float64(threshold))), - semantics: ExprSemantics::Sql, - }, - eq(REGION, region), - ])) - }; - let state = doubled_above(10.0, "us"); - let coverage = state.coverage().unwrap(); - let definition = coverage.definition(); - assert!(matches!( - definition.children()[0].non_asap(), - Some(NonASAPOp::Filter { .. }) - )); - merge(vec![state.clone(), doubled_above(10.0, "eu")]).unwrap(); - assert_eq!( - rejected(vec![state, doubled_above(20.0, "eu")]), - CoverageError::DefinitionMismatch - ); -} - -/// An instant selector picks the latest sample per series, which is not a -/// selection of rows, so it stays in the definition. -#[test] -fn instant_selectors_stay_in_the_definition() { - let instant = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeRange { - range: Duration::from_millis(300_000), - kind: TimeRangeKind::Instant, - child: series(vec![]), - })) - .unwrap(); - let state = kll(Rc::clone(&instant), "value"); - let coverage = state.coverage().unwrap(); - assert_eq!(coverage.selection()[0].relative_time(), None); - assert!(Rc::ptr_eq(coverage.definition().children()[0], &instant)); -} - -/// A merge is a state with coverage, so merges nest. -#[test] -fn merges_nest() { - let us_eu = merge(vec![ - latency_where(eq(REGION, "us")), - latency_where(eq(REGION, "eu")), - ]) - .unwrap(); - merge(vec![us_eu.clone(), latency_where(eq(REGION, "apac"))]).unwrap(); - assert_eq!( - rejected(vec![us_eu, latency_where(eq(REGION, "eu"))]), - CoverageError::InputsOverlap { - required: MergeInputRule::Disjoint, - } - ); -} - -/// When a state is computed is not what it computes. -#[test] -fn timing_does_not_block_merges() { - let ingested = Rc::new( - (*tumbling_window(60_000, vec![])) - .clone() - .with_timing(Some(ExecutionTiming::IngestionTime)), - ); - let queried = Rc::new( - (*tumbling_window(0, vec![])) - .clone() - .with_timing(Some(ExecutionTiming::QueryTime)), - ); - merge(vec![ingested, queried]).unwrap(); -} - -/// A merge built around `new` is still checked by `validate_structure`. -#[test] -fn forged_merges_fail_validation() { - let us = latency_where(eq(REGION, "us")); - let schema = us.schema.clone(); - let forged = Rc::new(OperatorNode::with_schema( - Operator::ASAP(ASAPOp::SummaryMerge { - children: vec![us.clone(), us], - newest_tumbling_windows: None, - group_by: None, - }), - schema, - )); - assert!(forged.coverage().is_none()); - assert!(matches!( - forged.validate_structure(), - Err(SchemaDerivationError::Coverage( - CoverageError::InputsOverlap { - required: MergeInputRule::Disjoint, - } - )) - )); -} - -/// Only summary states have coverage. -#[test] -fn non_summary_nodes_have_no_coverage() { - assert!(table().coverage().is_none()); - assert!(filter(table(), eq(REGION, "us")).coverage().is_none()); -} - -/// Two output columns with the same `(table, name)` cannot be told apart in -/// a selection, so restrictions on them stay in the definition. -#[test] -fn ambiguous_column_names_do_not_lift() { - let renamed = |restricted: usize, value: &str| { - let item = |column: usize, alias: Option<&str>| ProjectItem { - alias: alias.map(Into::into), - expr: ScalarExpr::Column(column), - }; - let project = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Project { - cols: vec![ - item(JOB, None), - item(REGION, Some("k")), - item(LATENCY, Some("k")), - item(LATENCY, None), - ], - qualifier: None, - child: table(), - })) - .unwrap(); - kll(filter(project, eq(restricted, value)), "latency") - }; - assert_eq!( - rejected(vec![renamed(1, "us"), renamed(2, "eu")]), - CoverageError::DefinitionMismatch - ); -} - -/// A rename outside a `Project` stays in the definition: here B's `Filter` -/// swaps the names of `region` and `tier`, so B's `tier = 'eu'` must not -/// read as `region = 'eu'` and look disjoint from A's `region = 'us'`. -#[test] -fn renaming_filters_stay_in_the_definition() { - let scan = || { - table_with(vec![ - Field::plain("job", DataType::Utf8, false), - Field::plain("region", DataType::Utf8, false), - Field::plain("tier", DataType::Utf8, false), - Field::plain("latency", DataType::Float64, false), - ]) - }; - let us = kll(filter(scan(), eq(1, "us")), "latency"); - let swapped = filter(scan(), eq(2, "eu")); - let mut schema = swapped.schema.clone(); - schema.fields[1].name = "tier".into(); - schema.fields[2].name = "region".into(); - let swapped = OperatorNode::with_schema(swapped.operator.clone(), schema); - let eu_tier = kll(Rc::new(swapped), "latency"); - assert_eq!( - rejected(vec![us, eu_tier]), - CoverageError::DefinitionMismatch - ); -} - -/// `latency = 1` and `latency = 1.0` select the same rows of a Float64 column. -#[test] -fn literals_of_different_types_are_not_proven_different() { - let one = |value| latency_where(compare(LATENCY, CompareOpKind::Eq, value)); - assert!(merge(vec![ - one(ScalarValue::Int64(1)), - one(ScalarValue::Float64(1.0)) - ]) - .is_err()); -} - -/// A scan predicate whose conjuncts only partly lift keeps just the rest. -#[test] -fn partly_lifted_scan_predicates_keep_only_the_rest() { - let doubled_positive = ScalarExpr::Compare { - left: Box::new(ScalarExpr::Arithmetic { - op: ArithmeticOpKind::Mul, - left: Box::new(ScalarExpr::Column(1)), - right: Box::new(ScalarExpr::Literal(ScalarValue::Float64(2.0))), - semantics: ExprSemantics::Promql, - }), - op: CompareOpKind::Gt, - right: Box::new(ScalarExpr::Literal(ScalarValue::Float64(0.0))), - semantics: ExprSemantics::Promql, - }; - let job = |name: &str| { - tumbling_window( - 0, - vec![Predicate(ScalarExpr::BoolAnd(vec![ - eq(2, name), - doubled_positive.clone(), - ]))], - ) - }; - merge(vec![job("api"), job("web")]).unwrap(); -} - -/// The worked example of the design doc (§4.2.2), as the SQL frontend -/// lowers it: the `FILTER` condition and `region = 'us'` (in -/// `Scan.predicates`, through a renaming `Project`) move into the -/// selection; the condition on an expression stays. -#[test] -fn design_doc_worked_example() { - let doubled_above_10 = ScalarExpr::Compare { - left: Box::new(ScalarExpr::Arithmetic { - op: ArithmeticOpKind::Mul, - left: Box::new(ScalarExpr::Column(LATENCY)), - right: Box::new(ScalarExpr::Literal(ScalarValue::Int64(2))), - semantics: ExprSemantics::Sql, - }), - op: CompareOpKind::Gt, - right: Box::new(ScalarExpr::Literal(ScalarValue::Int64(10))), - semantics: ExprSemantics::Sql, - }; - let scan = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Scan { - source: Source::Table { - table_ref: "t".into(), - }, - predicates: vec![Predicate(ScalarExpr::BoolAnd(vec![ - eq(REGION, "us"), - doubled_above_10.clone(), - ]))], - schema: Schema::new(vec![ - Field::plain("job", DataType::Utf8, false), - Field::plain("region", DataType::Utf8, false), - Field::plain("latency", DataType::Float64, false), - ]), - })) - .unwrap(); - let item = |column: usize, alias: Option<&str>| ProjectItem { - alias: alias.map(Into::into), - expr: ScalarExpr::Column(column), - }; - let project = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Project { - cols: vec![ - item(JOB, None), - item(REGION, Some("r")), - item(LATENCY, None), - ], - qualifier: None, - child: scan, - })) - .unwrap(); - let below_100 = compare(2, CompareOpKind::Lt, ScalarValue::Int64(100)); - let state = OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryAgg { - child: project, - family: FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: 200, - failure_probability: None, - }, - ), - Default::default(), - ), - input: SummaryUpdate::column(ColumnRef::Named("latency".into())), - reduction: Reduction::by(vec![0]), - grouping: GroupingStrategy::default(), - filter: Some(Predicate(below_100)), - })) - .unwrap(); - - let coverage = state.coverage().unwrap(); - assert_eq!( - coverage.selection(), - vec![SelectionBox::new( - [ - (column("r"), utf8(&["us"])), - ( - column("latency"), - Constraint::Interval { - lower: Bound::Unbounded, - upper: Bound::Excluded(ScalarValue::Int64(100)), - }, - ), - ] - .into(), - None - )] - ); - // definition: SummaryAgg (no filter) over Project over - // Scan t with only `latency * 2 > 10` left in its predicates. - let Some(ASAPOp::SummaryAgg { filter, child, .. }) = coverage.definition().asap() else { - panic!("definition is a SummaryAgg"); - }; - assert!(filter.is_none()); - assert!(matches!(child.non_asap(), Some(NonASAPOp::Project { .. }))); - assert!(matches!( - child.children()[0].non_asap(), - Some(NonASAPOp::Scan { predicates, .. }) if *predicates == vec![Predicate(doubled_above_10)] - )); -} - -/// The coverage of `node`, which must have one. -fn coverage_of( - node: &OperatorNode, -) -> asap_types::ir::properties::summary_coverage::SummaryCoverage { - node.coverage().expect("a summary state") -} - -/// Equal definitions get equal ids whatever the node, its selection, timing -/// or guarantee; a different computation (another metric, another column, -/// another sketch) gets another id. -#[test] -fn equal_definitions_get_equal_ids() { - let id = |node: &OperatorNode| definition_id(&coverage_of(node)); - let us = latency_where(eq(REGION, "us")); - let eu = latency_where(eq(REGION, "eu")); - assert_eq!(id(&us), id(&eu)); - let ingested = (*tumbling_window(0, vec![])) - .clone() - .with_timing(Some(ExecutionTiming::IngestionTime)); - assert_eq!(id(&ingested), id(&tumbling_window(60_000, vec![]))); - assert_eq!(id(&ingested).len(), 32); - let size = kll(filter(table(), eq(REGION, "eu")), "size"); - assert_ne!(id(&size), id(&eu)); - assert_ne!( - id(&tumbling_window(0, vec![])), - id(&increase_tumbling_window(0, vec![])) - ); - assert_ne!(id(&tumbling_window(0, vec![])), id(&us)); -} - -/// Six 10-minute tumbling windows at offsets 0 to 50 minutes share one definition id, -/// each with its own time selection `(-(i + 1) · 10m, -i · 10m]`; their -/// merge has the same id and the hour as its selection. -#[test] -fn tumbling_windows_of_an_hour_share_one_definition_with_distinct_times() { - let tumbling_windows: Vec> = - (0..6).map(|i| window(i * 600_000, 600_000)).collect(); - let coverages: Vec<_> = tumbling_windows - .iter() - .map(|tumbling_window| coverage_of(tumbling_window)) - .collect(); - for (i, coverage) in coverages.iter().enumerate() { - let i = i as i64; - assert_eq!(definition_id(coverage), definition_id(&coverages[0])); - assert_eq!( - coverage.selection(), - [SelectionBox::new( - Default::default(), - relative(-(i + 1) * 600_000, -i * 600_000) - )] - ); - } - let hour = coverage_of(&merge(tumbling_windows).unwrap()); - assert_eq!(definition_id(&hour), definition_id(&coverages[0])); - assert_eq!( - hour.selection(), - [SelectionBox::new( - Default::default(), - relative(-3_600_000, 0) - )] - ); -} - -/// A selection round-trips through JSON; columns are a list and bounds -/// name their kind. -#[test] -fn selections_round_trip_through_json() { - let selection = SelectionBox::new( - [ - (column("region"), utf8(&["eu", "us"])), - ( - column("latency"), - Constraint::Interval { - lower: Bound::Unbounded, - upper: Bound::Excluded(ScalarValue::Float64(100.0)), - }, - ), - ] - .into(), - relative(-120_000, -60_000), - ); - let json = serde_json::to_value(&selection).unwrap(); - assert_eq!( - json["relative_time"], - serde_json::json!([{"excluded": -120000}, {"included": -60000}]) - ); - assert_eq!(json["columns"].as_array().unwrap().len(), 2); - assert_eq!( - serde_json::from_value::(json).unwrap(), - selection - ); -} - -/// A `family` build over `child`'s `latency` (a table) or `value` (a -/// series), grouped by `reduction`. -fn grouped( - child: Rc, - family: FieldDataType, - reduction: Reduction, -) -> Rc { - let column = match child.schema.column_id("latency") { - Some(_) => "latency", - None => "value", - }; - OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryAgg { - child, - family, - input: SummaryUpdate::column(ColumnRef::Named(column.into())), - reduction, - grouping: GroupingStrategy::default(), - filter: None, - })) - .unwrap() -} - -fn kll_family() -> FieldDataType { - FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: 200, - failure_probability: None, - }, - ), - GroupingStrategy::default(), - ) -} - -fn rollup(state: Rc, keys: Vec) -> Result, String> { - OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryMerge { - children: vec![state], - group_by: Some(keys), - newest_tumbling_windows: None, - })) - .map_err(|error| error.to_string()) -} - -/// States by (job, region) over `region ∈ {us, eu}`, rolled up to region, -/// cover what a direct build by region over those rows does: the same -/// definition (and id) and the same selection. -#[test] -fn a_rollup_covers_the_coarser_build() { - let rows = filter( - table(), - ScalarExpr::InList { - expr: Box::new(ScalarExpr::Column(REGION)), - list: vec![ - ScalarExpr::Literal(ScalarValue::Utf8("us".into())), - ScalarExpr::Literal(ScalarValue::Utf8("eu".into())), - ], - negated: false, - }, - ); - let finer = grouped(rows.clone(), kll_family(), Reduction::by(vec![JOB, REGION])); - let coarse = grouped(rows, kll_family(), Reduction::by(vec![REGION])); - let rolled = rollup(finer, vec![1]).unwrap(); - assert_eq!(rolled.schema, coarse.schema); - let (rolled, coarse) = (rolled.coverage().unwrap(), coarse.coverage().unwrap()); - assert_eq!(definition_id(&rolled), definition_id(&coarse)); - assert_eq!(rolled.selection(), coarse.selection()); - assert_eq!( - rolled.selection()[0].columns()[&column("region")], - utf8(&["us", "eu"]) - ); -} - -/// Per-series states keep their input's columns in place, so a rollup by -/// the `job` label covers a build by `job` over the same series. -#[test] -fn a_rollup_of_per_series_states_covers_the_build_by_its_labels() { - let rows = tumbling_window_rows(0, vec![]); - let job = rows.schema.column_id("job").unwrap(); - let per_series = grouped(rows.clone(), kll_family(), Reduction::PerEntity); - let state_job = per_series.schema.column_id("job").unwrap(); - let rolled = rollup(per_series, vec![state_job]).unwrap(); - let by_job = grouped(rows, kll_family(), Reduction::by(vec![job])); - assert_eq!( - definition_id(&rolled.coverage().unwrap()), - definition_id(&by_job.coverage().unwrap()) - ); -} - -/// A rollup is rejected when a key is no group key of the input's build, -/// and when its family cannot merge different groups (counter states merge -/// consecutive windows of one series, IRate has no merge). -#[test] -fn invalid_rollups_fail_construction() { - let without = grouped( - table(), - kll_family(), - Reduction::Reduce( - asap_types::ir::operator::operator_properties::GroupKeys::without(vec![REGION]), - ), - ); - assert!(rollup(without, vec![0]) - .unwrap_err() - .contains(&CoverageError::NotAGroupKey { key: 0 }.to_string())); - for (kind, params) in [ - (ExactKind::Increase, ExactParams::Increase), - (ExactKind::Rate, ExactParams::Rate), - (ExactKind::IRate, ExactParams::IRate), - ] { - let state = grouped( - tumbling_window_rows(0, vec![]), - FieldDataType::ExactAggregate(kind, params), - Reduction::by(vec![2]), - ); - assert!( - rollup(state, vec![]) - .unwrap_err() - .contains(&CoverageError::FamilyCannotRegroup.to_string()), - "{kind:?}" - ); - } -} - -/// A rollup to no keys of a SQL relation's states covers the global build -/// over the same rows: a global SQL aggregate, one row even over no input -/// rows, as the rollup's one state. -#[test] -fn a_rollup_to_no_keys_of_a_relation_covers_the_global_build() { - let by_job = grouped(table(), kll_family(), Reduction::by(vec![JOB])); - let global = grouped(table(), kll_family(), Reduction::by(vec![])); - let rolled = rollup(by_job, vec![]).unwrap(); - assert_eq!(rolled.schema, global.schema); - assert_eq!( - definition_id(&rolled.coverage().unwrap()), - definition_id(&global.coverage().unwrap()) - ); -} - -/// `group_by` is left out of JSON when unset, so a merge that keeps its -/// groups serializes as before; a rollup's keys round-trip. -#[test] -fn group_by_is_serialized_only_for_a_rollup() { - let state = grouped(table(), kll_family(), Reduction::by(vec![JOB, REGION])); - let merge = ASAPOp::SummaryMerge { - children: vec![0u32], - group_by: None, - newest_tumbling_windows: None, - }; - assert_eq!( - serde_json::to_value(&merge).unwrap(), - serde_json::json!({"SummaryMerge": {"children": [0]}}) - ); - let rollup_op: ASAPOp = ASAPOp::SummaryMerge { - children: vec![0], - group_by: Some(vec![1]), - newest_tumbling_windows: None, - }; - let json = serde_json::to_string(&rollup_op).unwrap(); - assert_eq!( - serde_json::from_str::>(&json).unwrap(), - rollup_op - ); - assert!(rollup(state, vec![1]).is_ok()); -} - -/// An exact sum rolled up from (job, region) to region finalizes to the -/// direct build's column, type and nullability. -#[test] -fn a_finalized_rollup_has_the_direct_builds_schema() { - let sum = FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum); - let finer = grouped(table(), sum.clone(), Reduction::by(vec![JOB, REGION])); - let coarse = grouped(table(), sum, Reduction::by(vec![REGION])); - let finalize = |state: Rc| { - OperatorNode::new_shared(Operator::ASAP(ASAPOp::FinalizeExactAccumulator { - child: state, - })) - .unwrap() - }; - assert_eq!( - finalize(rollup(finer, vec![1]).unwrap()).schema, - finalize(coarse).schema - ); -} - -/// A filter on the group keys of per-(job, region) states keeps whole -/// states, so it is still state: merging the kept states covers the global -/// build over the filtered rows. A filter reading the state column is not a -/// selection of states and is rejected. -#[test] -fn a_key_filter_over_states_selects_states() { - let states = grouped(table(), kll_family(), Reduction::by(vec![JOB, REGION])); - let kept = filter(Rc::clone(&states), eq(JOB, "a")); - assert_eq!(kept.result_kind, OperatorResultKind::State); - let rolled = rollup(kept, vec![]).unwrap(); - let direct = grouped( - filter(table(), eq(JOB, "a")), - kll_family(), - Reduction::by(vec![]), - ); - assert_eq!(rolled.schema, direct.schema); - assert_eq!( - definition_id(&rolled.coverage().unwrap()), - definition_id(&direct.coverage().unwrap()) - ); - let on_state = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Filter { - pred: Predicate(ScalarExpr::IsNull(Box::new(ScalarExpr::Column(2)))), - child: states, - })) - .unwrap(); - assert_eq!(on_state.result_kind, OperatorResultKind::Relation); - assert!(on_state.validate_structure().is_err()); -} diff --git a/crates/types/tests/summary_merge_structure.rs b/crates/types/tests/summary_merge_structure.rs index b76ad5cc7..afb075fa9 100644 --- a/crates/types/tests/summary_merge_structure.rs +++ b/crates/types/tests/summary_merge_structure.rs @@ -1,34 +1,22 @@ //! Window composition merges compatible summary states without consuming raw rows. use asap_types::ir::operator::operator_properties::{Reduction, Source}; -use asap_types::ir::properties::ExecutionTiming; use asap_types::ir::scalar::expr_ir::{CompareOpKind, ScalarValue}; use asap_types::ir::scalar::ColumnRef; use asap_types::ir::schema::{ DataType, ExactKind, ExactParams, FamilyCapability, Field, FieldDataType, GroupingStrategy, HydraKind, HydraParams, MergeInputRule, MergedError, SamplingKind, SamplingParams, Schema, - SketchAlgorithm, SketchKind, SketchParams, SketchStatistic, StatModelKind, StatModelParams, - StatisticKind, SummaryUpdate, WaveletKind, WaveletParams, + SketchAlgorithm, SketchParams, SketchStatistic, StatModelKind, StatModelParams, StatisticKind, + SummaryUpdate, WaveletKind, WaveletParams, }; use asap_types::ir::{ - apply_materialization_timings, ASAPOp, ExprSemantics, MaterializationAssignment, - NodeToTimedNodeCache, NonASAPOp, Operator, OperatorNode, Predicate, ScalarExpr, + ASAPOp, ExprSemantics, NonASAPOp, Operator, OperatorNode, Predicate, ScalarExpr, }; -use asap_types::physical::{Materialization, Medium}; use std::rc::Rc; /// KLL over `value` for one `region`, so states of different regions are /// disjoint. -fn state(k: u32, region: &str) -> Rc { +fn state(_k: u32, region: &str) -> Rc { family_state( - FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k, - failure_probability: None, - }, - ), - Default::default(), - ), + FieldDataType::Sketch(SketchAlgorithm::Kll, Default::default()), region, ) } @@ -77,14 +65,12 @@ fn compatible_tumbling_windows_merge_structurally() { root.validate_structure().unwrap(); assert_eq!(root.schema.fields.len(), 1); } -/// An empty merge, raw rows and differently sized state cannot masquerade as compatible tumbling windows. +/// An empty merge and raw rows cannot masquerade as compatible tumbling +/// windows. (State sizes are no longer part of the state's type: θ is a +/// candidate's `LogicalProperties::parameters`.) #[test] fn incompatible_merge_inputs_fail() { - for children in [ - vec![], - vec![state(200, "us"), state(300, "eu")], - vec![state(200, "us").children()[0].clone()], - ] { + for children in [vec![], vec![state(200, "us").children()[0].clone()]] { assert!( OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryMerge { children, @@ -96,21 +82,18 @@ fn incompatible_merge_inputs_fail() { } } -fn sketch(algorithm: SketchAlgorithm, params: SketchParams) -> FieldDataType { - FieldDataType::Sketch(SketchKind::new(algorithm, params), Default::default()) +fn sketch(algorithm: SketchAlgorithm, _params: SketchParams) -> FieldDataType { + FieldDataType::Sketch(algorithm, Default::default()) } fn hydra( algorithm: SketchAlgorithm, - params: SketchParams, + _params: SketchParams, kind: HydraKind, - hydra_params: HydraParams, + _hydra_params: HydraParams, ) -> FieldDataType { FieldDataType::Sketch( - SketchKind::new(algorithm, params), - GroupingStrategy::SharedMultiSubpopulation { - kind, - params: hydra_params, - }, + algorithm, + GroupingStrategy::SharedMultiSubpopulation { kind }, ) } fn exact(kind: ExactKind, params: ExactParams) -> FieldDataType { @@ -577,6 +560,7 @@ fn estimate_validation_matches_the_category_match_except_the_fixed_rows() { SketchParams::CmsWithHeap { .. } => A::CmsWithHeap, SketchParams::CountSketchWithHeap { .. } => A::CountSketchWithHeap, SketchParams::UnivMon { .. } => A::UnivMon, + SketchParams::Hydra(_) => unreachable!("no Hydra parameters in this list"), }; sketch(algorithm, params) }) @@ -638,7 +622,7 @@ fn estimate_validation_matches_the_category_match_except_the_fixed_rows() { }; for query in &statistics { let fixed = matches!( - (kind.algorithm(), query.kind()), + (kind, query.kind()), ( SketchAlgorithm::UnivMon, StatisticKind::ItemFrequency | StatisticKind::TopK @@ -648,7 +632,7 @@ fn estimate_validation_matches_the_category_match_except_the_fixed_rows() { StatisticKind::ItemFrequency, ) ); - let before = accepted_by_category(kind.algorithm(), query); + let before = accepted_by_category(kind, query); let after = estimate(family.clone(), query.clone()).is_ok(); assert_eq!(after, before != fixed, "{family:?} {query:?}"); } @@ -669,78 +653,3 @@ fn summary_merge_is_permissive_about_the_family() { merged.validate_structure().unwrap(); assert_eq!(FamilyCapability::of(&irate()).merge_inputs, None); } - -/// Idempotent families (HLL, exact Min/Max) merge states that may share -/// rows, and the union of identical selections is that selection; counting -/// families (exact Sum, KLL) still need disjoint selections. -#[test] -fn idempotent_families_merge_overlapping_states() { - let same_region = |family: FieldDataType| { - merge(vec![ - family_state(family.clone(), "us"), - family_state(family, "us"), - ]) - }; - for family in [hll(), exact(ExactKind::Max, ExactParams::Max)] { - let merged = same_region(family.clone()).unwrap_or_else(|e| panic!("{family:?}: {e}")); - merged.validate_structure().unwrap(); - assert_eq!( - merged.coverage().unwrap().selection(), - family_state(family, "us").coverage().unwrap().selection() - ); - } - for family in [exact(ExactKind::Sum, ExactParams::Sum), kll()] { - let error = same_region(family).unwrap_err(); - assert!(error.contains("not proven disjoint"), "{error}"); - } -} - -/// `SummaryMerge` takes the timing of its consumer, and reads tumbling-window states -/// stored at ingestion time (#509 Example 3/4 Pattern B1) or built at query -/// time (B2). Merging at ingestion time needs every tumbling window at ingestion time. -#[test] -fn summary_merge_timing() { - let tumbling_windows = [state(200, "us"), state(200, "eu")]; - let root = merge(tumbling_windows.to_vec()).unwrap(); - let timings = |assignment: &MaterializationAssignment| { - let timed = - apply_materialization_timings(&root, assignment, &mut NodeToTimedNodeCache::new()) - .unwrap(); - let children = timed.children(); - ( - timed.timing.unwrap(), - children - .iter() - .map(|c| c.timing.unwrap()) - .collect::>(), - ) - }; - use ExecutionTiming::{IngestionTime, QueryTime}; - assert_eq!( - timings(&MaterializationAssignment::default()), - (QueryTime, vec![QueryTime, QueryTime]) - ); - let at_ingestion = Materialization::MatAtIngestion(Medium::Memory); - let mut stored_tumbling_windows = MaterializationAssignment::default(); - for tumbling_window in &tumbling_windows { - stored_tumbling_windows.set(tumbling_window, at_ingestion); - } - assert_eq!( - timings(&stored_tumbling_windows), - (QueryTime, vec![IngestionTime, IngestionTime]) - ); - // A `SummaryAgg` without an assignment runs at query time, so a merge - // stored at ingestion time needs its tumbling windows there too. - let mut only_merge = MaterializationAssignment::default(); - only_merge.set(&root, at_ingestion); - assert!( - apply_materialization_timings(&root, &only_merge, &mut NodeToTimedNodeCache::new()) - .is_err() - ); - let mut stored_merge = MaterializationAssignment::with_default(at_ingestion); - stored_merge.set(&root, at_ingestion); - assert_eq!( - timings(&stored_merge), - (IngestionTime, vec![IngestionTime, IngestionTime]) - ); -} diff --git a/crates/types/tests/tumbling_window_chain.rs b/crates/types/tests/tumbling_window_chain.rs index b37d77385..376ab310f 100644 --- a/crates/types/tests/tumbling_window_chain.rs +++ b/crates/types/tests/tumbling_window_chain.rs @@ -1,7 +1,6 @@ //! A `SummaryTumblingWindowChain` of tumbling windows stands for its tumbling windows: it covers what one //! build over its whole window does, and expands into the `SummaryAgg` //! tumbling windows the executor runs (#509 Pass 2, window-composition rule). -use std::ops::Bound; use std::rc::Rc; use std::time::Duration; @@ -11,9 +10,7 @@ use asap_types::ir::operator::tumbling_window_chain::{ }; use asap_types::ir::scalar::ColumnRef; use asap_types::ir::schema::{DataType, Field, FieldDataType, Schema}; -use asap_types::ir::schema::{ - GroupingStrategy, SketchAlgorithm, SketchKind, SketchParams, SketchStatistic, SummaryUpdate, -}; +use asap_types::ir::schema::{GroupingStrategy, SketchAlgorithm, SketchStatistic, SummaryUpdate}; use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode, TimeRangeKind}; use asap_types::test_support::{tumbling_window, whole_window_build}; @@ -47,16 +44,7 @@ fn shifted(child: Rc, offset_ms: i64) -> Rc { } fn kll() -> FieldDataType { - FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: 200, - failure_probability: None, - }, - ), - GroupingStrategy::default(), - ) + FieldDataType::Sketch(SketchAlgorithm::Kll, GroupingStrategy::default()) } /// Five 1-min KLL tumbling windows per series over `input`. @@ -83,22 +71,14 @@ fn merge(children: Vec>) -> Rc { .unwrap() } -fn relative_time(node: &OperatorNode) -> (Bound, Bound) { - let coverage = node.coverage().expect("coverage"); - let [selection] = coverage.selection() else { - panic!("one box: {coverage:?}") - }; - selection.relative_time().expect("a time window") -} - /// The merged chain is a state of the build over the whole 5-min window: -/// the same schema and coverage `(-5m, 0]`, and it validates. +/// the same schema, and it validates. #[test] fn merged_tumbling_windows_cover_the_whole_window() { let chain = chain(scan()); let merged = merge(vec![Rc::clone(&chain)]); merged.validate_structure().unwrap(); - assert!(merged_chain(&merged).is_some_and(|c| std::ptr::eq(c, &*chain))); + assert!(merged_chain(&merged).is_some_and(|c| Rc::ptr_eq(&c, &chain))); let whole = whole_window_build(&chain).unwrap(); let Some(NonASAPOp::TimeRange { range, kind, .. }) = whole.children()[0].non_asap() else { panic!("{:?}", whole.operator) @@ -108,11 +88,6 @@ fn merged_tumbling_windows_cover_the_whole_window() { (Duration::from_millis(5 * MINUTE), TimeRangeKind::Range) ); assert_eq!(merged.schema, whole.schema); - assert_eq!(merged.coverage(), whole.coverage()); - assert_eq!( - relative_time(&merged), - (Bound::Excluded(-5 * MINUTE as i64), Bound::Included(0)) - ); } /// Tumbling window `i` is the build over `(-(i+1)·1m, -i·1m]`, the query's own offset @@ -124,14 +99,6 @@ fn each_tumbling_window_is_one_shifted_minute() { for i in 0..5 { let tumbling_window = tumbling_window(&chain, i).unwrap(); let back = offset + i64::from(i) * MINUTE as i64; - assert_eq!( - relative_time(&tumbling_window), - ( - Bound::Excluded(-(back + MINUTE as i64)), - Bound::Included(-back) - ), - "tumbling window {i}" - ); let range = tumbling_window.children()[0]; let Some(NonASAPOp::TimeShift { shift, child }) = range.children()[0].non_asap() else { panic!("tumbling window {i} reads a shift") @@ -142,7 +109,7 @@ fn each_tumbling_window_is_one_shifted_minute() { } /// Expansion gives the executor a merge over the five tumbling windows, newest first, -/// which covers what the merged chain does; a chain read without a merge +/// newest first; a chain read without a merge /// does not expand. #[test] fn expansion_merges_the_tumbling_windows() { @@ -170,53 +137,9 @@ fn expansion_merges_the_tumbling_windows() { "tumbling window {i}" ); } - assert_eq!(summary_input.coverage(), merged.coverage()); assert!(expand_tumbling_window_chains(&chain).is_err()); } -/// Expanding a timed chain carries its timing and materialization onto its -/// tumbling windows: a chain materialized at ingestion time expands into tumbling windows each -/// built and stored at ingestion time, over ranges and shifts that run at -/// ingestion time too, while the merge reading them stays at query time. -/// An unmaterialized chain expands into query-time tumbling windows. -#[test] -fn expansion_carries_the_chains_timing_onto_its_tumbling_windows() { - use asap_types::ir::properties::ExecutionTiming; - use asap_types::ir::{ - apply_materialization_timings, MaterializationAssignment, NodeToTimedNodeCache, - }; - use asap_types::physical::{Materialization, Medium}; - let at_ingestion = Materialization::MatAtIngestion(Medium::Memory); - for (materialization, timing) in [ - (at_ingestion, ExecutionTiming::IngestionTime), - (Materialization::NotMaterialized, ExecutionTiming::QueryTime), - ] { - let chain = chain(shifted(scan(), MINUTE as i64)); - let estimate = OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryEstimate { - summary_input: merge(vec![Rc::clone(&chain)]), - query: SketchStatistic::Quantile { q: 0.99 }, - })) - .unwrap(); - let mut assignment = MaterializationAssignment::default(); - assignment.set(&chain, materialization); - let timed = - apply_materialization_timings(&estimate, &assignment, &mut NodeToTimedNodeCache::new()) - .unwrap(); - let expanded = expand_tumbling_window_chains(&timed).unwrap(); - let merged = expanded.children()[0]; - assert_eq!(merged.timing, Some(ExecutionTiming::QueryTime)); - assert_eq!(merged.children().len(), 5); - for tumbling_window in merged.children() { - assert_eq!(tumbling_window.timing, Some(timing), "{materialization:?}"); - assert_eq!(tumbling_window.materialization, materialization); - for below in OperatorNode::reachable(tumbling_window).iter().skip(1) { - assert_eq!(below.timing, Some(timing), "{}", below.operator.kind_name()); - assert_eq!(below.materialization, Materialization::NotMaterialized); - } - } - } -} - /// A chain needs a positive width and count, and reads the instant vector /// a range selector would. #[test] @@ -245,63 +168,6 @@ fn a_chain_reads_an_instant_vector() { assert!(empty.validate_structure().is_err()); } -/// Two merges over one chain (two queries reading the same tumbling windows) expand -/// into one set of tumbling windows: the same five `SummaryAgg` nodes, not ten. -#[test] -fn merges_over_one_chain_share_its_tumbling_windows() { - use asap_types::ir::operator::tumbling_window_chain::expand_tumbling_window_chains_in_workload; - use asap_types::ir::QueryRoot; - let shared = chain(scan()); - let roots = [ - QueryRoot::Operator(merge(vec![Rc::clone(&shared)])), - QueryRoot::Operator(merge(vec![shared])), - ]; - let expansion = expand_tumbling_window_chains_in_workload(&roots).unwrap(); - assert_eq!(expansion.chains.len(), 1); - assert_eq!(expansion.chains[0].tumbling_windows.len(), 5); - let expanded = expansion.roots; - let tumbling_windows = |root: &QueryRoot| match root { - QueryRoot::Operator(node) => node.children().into_iter().cloned().collect::>(), - QueryRoot::Scalar(_) => unreachable!(), - }; - let (first, second) = ( - tumbling_windows(&expanded[0]), - tumbling_windows(&expanded[1]), - ); - assert_eq!(first.len(), 5); - assert!(first.iter().zip(&second).all(|(a, b)| Rc::ptr_eq(a, b))); -} - -fn newest(chain: Rc, newest_tumbling_windows: u32) -> Rc { - OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryMerge { - children: vec![chain], - newest_tumbling_windows: Some(newest_tumbling_windows), - group_by: None, - })) - .unwrap() -} - -/// A merge of the newest 2 of 5 tumbling windows is a state of one build over the -/// newest 2 minutes: coverage `(-2m, 0]`, the window build's schema, and -/// a valid structure. -#[test] -fn a_merge_of_the_newest_tumbling_windows_covers_their_window() { - use asap_types::ir::operator::tumbling_window_chain::merged_window_build; - let partial = newest(chain(scan()), 2); - partial.validate_structure().unwrap(); - let build = merged_window_build(&partial).unwrap(); - let Some(NonASAPOp::TimeRange { range, .. }) = build.children()[0].non_asap() else { - panic!("{:?}", build.operator) - }; - assert_eq!(*range, Duration::from_millis(2 * MINUTE)); - assert_eq!(partial.schema, build.schema); - assert_eq!(partial.coverage(), build.coverage()); - assert_eq!( - relative_time(&partial), - (Bound::Excluded(-2 * MINUTE as i64), Bound::Included(0)) - ); -} - /// `newest_tumbling_windows` must name between 1 and fewer than all tumbling windows of the one /// chain the merge reads; anything else is rejected. #[test] @@ -324,42 +190,6 @@ fn a_merge_of_the_newest_tumbling_windows_reads_one_chain_and_fewer_than_all_tum } } -/// A merge of the newest 2 tumbling windows and a merge of all 5 over one chain -/// expand into one set of tumbling windows: the partial merge reads the first 2 of -/// the full merge's 5 nodes, newest first, and the chain lists all 5. -#[test] -fn a_merge_of_the_newest_tumbling_windows_reads_the_first_tumbling_windows_of_the_chain() { - use asap_types::ir::operator::tumbling_window_chain::expand_tumbling_window_chains_in_workload; - use asap_types::ir::QueryRoot; - let shared = chain(scan()); - let roots = [ - QueryRoot::Operator(newest(Rc::clone(&shared), 2)), - QueryRoot::Operator(merge(vec![shared])), - ]; - let expansion = expand_tumbling_window_chains_in_workload(&roots).unwrap(); - assert_eq!(expansion.chains.len(), 1); - assert_eq!(expansion.chains[0].tumbling_windows.len(), 5); - let tumbling_windows = |root: &QueryRoot| match root { - QueryRoot::Operator(node) => node.children().into_iter().cloned().collect::>(), - QueryRoot::Scalar(_) => unreachable!(), - }; - let (partial, full) = ( - tumbling_windows(&expansion.roots[0]), - tumbling_windows(&expansion.roots[1]), - ); - assert_eq!((partial.len(), full.len()), (2, 5)); - assert!(partial.iter().zip(&full).all(|(a, b)| Rc::ptr_eq(a, b))); - for (root, window) in expansion.roots.iter().zip([2, 5]) { - let QueryRoot::Operator(node) = root else { - unreachable!() - }; - assert_eq!( - relative_time(node), - (Bound::Excluded(-window * MINUTE as i64), Bound::Included(0)) - ); - } -} - /// `newest_tumbling_windows` is left out of the JSON when `None`, and JSON without it /// reads as `None`; `Some(n)` round-trips. #[test] @@ -383,9 +213,7 @@ fn newest_tumbling_windows_serializes_only_when_set() { } /// A rollup of a chain's tumbling windows to no keys (every series' tumbling windows into one -/// state) is no tumbling-window merge, and expansion keeps its keys: the expanded -/// rollup covers what the planned one does, one build over the whole -/// window and every series. +/// state) is no tumbling-window merge, and expansion keeps its keys. #[test] fn expansion_keeps_a_rollup_of_the_tumbling_windows() { let chain = chain(scan()); @@ -404,12 +232,6 @@ fn expansion_keeps_a_rollup_of_the_tumbling_windows() { panic!("{:?}", expanded.operator) }; assert_eq!((children.len(), group_by), (5, &Some(vec![]))); - let coverage = rollup.coverage().unwrap(); - assert_eq!(expanded.coverage(), Some(coverage.clone())); - let Some(ASAPOp::SummaryAgg { reduction, .. }) = coverage.definition().asap() else { - panic!("{:?}", coverage.definition().operator) - }; - assert_eq!(*reduction, Reduction::by(vec![])); // A rollup merges every input state: it reads a merge of the newest // tumbling windows, it is not one. assert!(OperatorNode::new(Operator::ASAP(ASAPOp::SummaryMerge { @@ -420,56 +242,6 @@ fn expansion_keeps_a_rollup_of_the_tumbling_windows() { .is_err()); } -/// A rollup reads a merge of the newest tumbling windows: every series' newest 2 of 5 -/// tumbling windows into one state covers one build by no keys over the newest 2 -/// minutes. -#[test] -fn a_rollup_of_the_newest_tumbling_windows_covers_their_window() { - let rollup = OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryMerge { - children: vec![newest(chain(scan()), 2)], - newest_tumbling_windows: None, - group_by: Some(vec![]), - })) - .unwrap(); - rollup.validate_structure().unwrap(); - let coverage = rollup.coverage().unwrap(); - let Some(ASAPOp::SummaryAgg { reduction, .. }) = coverage.definition().asap() else { - panic!("{:?}", coverage.definition().operator) - }; - assert_eq!(*reduction, Reduction::by(vec![])); - assert_eq!( - coverage.selection()[0].relative_time(), - Some((Bound::Excluded(-2 * MINUTE as i64), Bound::Included(0))) - ); -} - -/// Expansion keeps every tumbling window and the identity of a chain two -/// readers share. -#[test] -fn expansion_preserves_every_tumbling_window_and_shared_reader() { - use asap_types::ir::operator::tumbling_window_chain::expand_tumbling_window_chains_in_workload; - use asap_types::ir::QueryRoot; - let shared = chain(scan()); - let roots = [ - QueryRoot::Operator(merge(vec![shared.clone()])), - QueryRoot::Operator(merge(vec![shared])), - ]; - let result = expand_tumbling_window_chains_in_workload(&roots).unwrap(); - assert_eq!(result.chains.len(), 1); - assert_eq!(result.chains[0].tumbling_windows.len(), 5); - let QueryRoot::Operator(first) = &result.roots[0] else { - unreachable!() - }; - let QueryRoot::Operator(second) = &result.roots[1] else { - unreachable!() - }; - assert!(first - .children() - .iter() - .zip(second.children()) - .all(|(a, b)| Rc::ptr_eq(a, b))); -} - /// JSON written before #1163 names the chain `PaneChain`, its fields /// `tumbling_window_width_ms` and `tumbling_window_count`, and a merge's `newest_tumbling_windows`; it reads /// as today's operators. diff --git a/docs/design_docs/architecture/input-output-workflow.md b/docs/design_docs/architecture/input-output-workflow.md index 400b73675..a3fc422c1 100644 --- a/docs/design_docs/architecture/input-output-workflow.md +++ b/docs/design_docs/architecture/input-output-workflow.md @@ -305,7 +305,7 @@ alternative stays visible in `PlanningRun`. PlanningWorkload + frontend dependencies + PlanningModels -> Stage 0: frontend lowering, one root per normalized query entry -> Stage 1: Pass 1 per-computation candidates, Pass 2 sharing, tumbling windows, reuse - -> Stage 2: one physical candidate per materialization assignment + -> Stage 2: one physical candidate per physical properties assignment -> Stage 3: check, measure and select -> PlanOutput ``` diff --git a/docs/design_docs/architecture/physical-plan-integration.md b/docs/design_docs/architecture/physical-plan-integration.md index a9961b710..4d42aaab9 100644 --- a/docs/design_docs/architecture/physical-plan-integration.md +++ b/docs/design_docs/architecture/physical-plan-integration.md @@ -461,9 +461,9 @@ a numeric entity key is not a score. Exact accumulator inputs are explicitly finalized before row operators consume them. None of these operations proves candidate completeness; that evidence belongs to the semi-join's pruning step. -The logical DAG carries no execution layout: `apply_materialization_timings` -writes each node's timing and materialization from a -`MaterializationAssignment` before export (every sub-DAG `NotMaterialized` by -default, so all query time). The exported DAG carries both, and the deployment +The logical DAG carries no execution layout: `apply_physical_properties` +writes each node's timing and lifetime from a +`PhysicalPropertiesAssignment` before export (no sub-DAG materialized by default, so all +query time). The exported DAG carries both, and the deployment executes it as given; this is not a claim that every deployment has implemented every placement. diff --git a/docs/design_docs/architecture/planner-runtime-contract.md b/docs/design_docs/architecture/planner-runtime-contract.md index 69813bc5f..dbab5b3cc 100644 --- a/docs/design_docs/architecture/planner-runtime-contract.md +++ b/docs/design_docs/architecture/planner-runtime-contract.md @@ -14,7 +14,7 @@ execution remain downstream. The | Layer | Owner | Examples | |---|---|---| | Logical candidate semantics | ASAPPlanner | Query rewrite; summary family and parameters; grouping; accuracy guarantees when established. | -| Summary materialization and realization selection | ASAPPlanner Stage 2 materialization (#509) | `NotMaterialized`, `MatAtIngestion(medium)` or `MatAtQuery(medium)` per sub-DAG, and how long a stored output is kept; direct build or incremental update; window implementations compared using provider evidence. | +| Summary materialization and realization selection | ASAPPlanner Stage 2 materialization (#509) | a timing and a lifetime per sub-DAG (`None`, `OneEvaluation(medium)` or `AcrossEvaluations(medium)`), and how much a materialized output retains; direct build or incremental update; window implementations compared using provider evidence. | | Concrete implementation and deployment | ASAPQuery-backend and its workload optimizer | Library and data-structure implementation, exact tumbling-window layout, placement, sharding, storage, transmission, materialization IDs, executor configuration, and workload-wide assignment. | ASAPCollector and the ASAPQuery data plane execute the compiled downstream diff --git a/docs/design_docs/architecture/updated_interface_with_pluggable_optimization.md b/docs/design_docs/architecture/updated_interface_with_pluggable_optimization.md index 3011064f3..d8756480f 100644 --- a/docs/design_docs/architecture/updated_interface_with_pluggable_optimization.md +++ b/docs/design_docs/architecture/updated_interface_with_pluggable_optimization.md @@ -101,7 +101,7 @@ pub struct QueryPlan { ``` `Stage1ToStage3Pass` returns the selected plan's roots, as Stage 2's realizer -built them with each node's timing and materialization, and with the weight +built them with each node's timing and lifetime, and with the weight proofs Stage 3 made applied; `PlanOutput::physical_asap_dag()` exports them. Physical ASAP-aware optimization decides per sub-DAG whether to materialize, and whether at ingestion or query time, from #604 on. @@ -125,7 +125,7 @@ tracked in #580. | Step | Call | |---|---| | Prepare roots | Entries in `entries()` order, the accuracy target from each entry's `requirements`; PromQL operator roots carry the series identity (`with_promql_series_identity`) where it has a realization | -| Stage 1: logical ASAP-aware optimization | `generate_per_computation_logical_memo_candidates` (Pass 1), then `generate_logical_asap_memo_candidates` (Pass 2) over every tumbling-window width | +| Stage 1: logical ASAP-aware optimization | `pass1_memo` (Pass 1), then `generate_logical_asap_memo_candidates` (Pass 2) over every tumbling-window width | | Stage 2: physical ASAP-aware optimization | `generate_physical_asap_memo_candidates` | | Stage 3: plan selection | `select_plan`: check accuracy and latency, price under the cost model, select one plan; the facade builds it with Stage 2's `AssignmentRealizer::realize` | diff --git a/docs/design_docs/concepts/glossary.md b/docs/design_docs/concepts/glossary.md index c27400a31..8b52e6ef6 100644 --- a/docs/design_docs/concepts/glossary.md +++ b/docs/design_docs/concepts/glossary.md @@ -38,7 +38,7 @@ A concrete execution and deployment choice: runtime topology of data lifecycle s ## Materialized view -A node of the physical DAG whose output is kept across evaluations (`MatAtIngestion` or `MatAtQueryKeptAcrossEvaluations`), with how long it is kept (`retention_ms` on the node). A `MatAtQuery` node is not a view: its output is discarded after one evaluation. Code: `MaterializedView`, `PhysicalASAPDAG::materialized_views`. +A node of the physical DAG whose output is materialized across evaluations (lifetime `AcrossEvaluations`, at ingestion time or at query time), with how long it is materialized (`retention_ms` on the node). A node materialized for `OneEvaluation` is not a view: its output is discarded after one evaluation. Code: `MaterializedView`, `PhysicalASAPDAG::materialized_views`. ## Raw data sample diff --git a/docs/design_docs/concepts/post-asap-ir.md b/docs/design_docs/concepts/post-asap-ir.md index ea9d9e57b..6caa8b0a8 100644 --- a/docs/design_docs/concepts/post-asap-ir.md +++ b/docs/design_docs/concepts/post-asap-ir.md @@ -81,7 +81,7 @@ Exact work is represented by the ordinary operators, unchanged: - `BinaryOp` combines independently planned operands. Summary planning may set its typed division guards (`checked_finite_division`, `checked_relative_division`); the operator's timing comes from the - materialization assignment, not from the operator. + physical properties assignment, not from the operator. - Aggregate, projection, filter, sort and limit over an evaluation are the ordinary `Aggregate`, `Project`, `Filter`, `Sort` and `Limit` operators reading an ASAP node. Exact-accumulator state may pass through the projection-like @@ -108,60 +108,63 @@ happens: **ingestion time** or **query time**. Operator identity must not imply one of these phases. Backend capability restrictions are implementation gaps, not definitions of the operator. -The logical DAG carries no timing: `OperatorNode::timing` is `None` on every -front-end node and every candidate, and `map_children` clears it. +Operator nodes carry no physical properties: the logical DAG has no timing +or lifetime. Materialization ([#509 §Materialization](../proposals/planner-layering.md#materialization)) -gives each sub-DAG one of three options: `NotMaterialized`, -`MatAtIngestion(medium)` or `MatAtQuery(medium)`, with `Medium::Memory` or -`Medium::Disk`. A -[`MaterializationAssignment`](../../../crates/types/src/ir/properties/timing.rs) records -the option per sub-DAG, named by the node whose output would be stored. Any -node may be named: a summary state, a population, or, as in #509 Example 4, -the tumbling windows a query reads. `set(node, materialization)` assigns -one, and every other sub-DAG is `NotMaterialized`. Choosing the options is -Stage 2's job (#604); until then the planner materializes nothing. -`apply_materialization_timings(root, &assignment, &mut NodeToTimedNodeCache)` then writes a -timing and a materialization into every node, top-down: +decides, for each sub-DAG, whether its output is materialized, and if so with which +[`PhysicalProperties`](../../../crates/types/src/physical/properties.rs): a +timing and a `Lifetime` (#1200). The lifetime is `None` (discarded after its +reader), `OneEvaluation(medium)` (materialized until every reader of one evaluation +finishes) or `AcrossEvaluations(medium)` (materialized for later evaluations), with +`StorageMedium::Memory` or `StorageMedium::Disk`; an ingestion-time output cannot be materialized +for `OneEvaluation`. A +[`PhysicalPropertiesAssignment`](../../../crates/types/src/ir/properties/timing.rs) records, +per sub-DAG, the properties of its materialized output, named by the node whose output +would be materialized. Any node may be named: a summary state, a population, or, as in +#509 Example 4, the tumbling windows a query reads. `set(node, properties)` +lists one; an entry whose lifetime is `None` fixes only the node's timing, and every +sub-DAG the assignment does not list is not materialized. Stage 2 chooses the +assignment. Operator nodes carry no timing or lifetime (#1200): physical +stages read them from the assignment. +`apply_physical_properties(root, &assignment)` then completes it into a timing and a +lifetime for every node, top-down, and returns the completed assignment: - `SummaryEstimate` and `EvaluatePopulation` run at query time; -- a node assigned `MatAtIngestion` runs at ingestion time, and one assigned - `MatAtQuery` at query time; a `SummaryAgg` or `MaintainPopulation` that is - `NotMaterialized` runs at query time. Every node upstream of an - ingestion-time node also runs at ingestion time, so `MatAtIngestion` over - something that can only exist at query time (an evaluation, or a state - that is not `MatAtIngestion`) fails with +- a node the assignment lists runs at its entry's timing; a `SummaryAgg` + or `MaintainPopulation` that is not materialized runs at query time. Every node + upstream of an ingestion-time node also runs at ingestion time, so a materialized + ingestion-time output over something that can only exist at query time (an + evaluation, or a state not materialized at ingestion time) fails with `ExecutionDataStateError::UpstreamNotAtIngestion` ("upstream of ingestion-time node not at ingestion time"); -- every other node is not stored and runs when its consumer runs: everything +- every other node is not materialized and runs when its consumer runs: everything that feeds an ingestion-time node runs at ingestion time, everything above - an evaluation at query time. A query-time consumer reads a `MatAtIngestion` - node's stored output. - -A node whose timing a candidate builder fixed (the query-time -`FinalizeExactAccumulator` read boundary of `finalize_query_candidate`) keeps -it: an explicit assignment that agrees with that timing is kept, and one that -needs the other timing fails with -`ExecutionDataStateError::MaterializationConflictsWithFixedTiming`; an -explicit `MatAtIngestion` is never silently moved or dropped. A root may be -`MatAtIngestion`, and the query reads its stored output; a root of rows that + an evaluation at query time. A query-time consumer reads the stored output + of a node materialized at ingestion time (`AcrossEvaluations`). + +Applying a completed assignment again gives every node the same properties: the +facade re-applies Stage 3's selection this way for export +(`PlanOutput::physical_properties`). A root may be materialized +at ingestion time, and the query reads its stored output; a root of rows that runs at ingestion time without being stored there fails with `ExecutionDataStateError::UnstoredIngestionRoot`. The pass then validates every edge (rows or exact-accumulator state into a -`SummaryAgg`, state into an evaluation, a timed `MaintainPopulation` under +`SummaryAgg`, state into an evaluation, a `MaintainPopulation` under an `EvaluatePopulation`, no ingestion work reading a query-time value) and rejects a node reached from two consumers that need different timings; `split_shared_by_timing` copies such a sub-DAG for one side before the -assignment is applied. How long a materialized output is kept (retention) is -also Stage 2's decision and is not in the assignment. - -`MaterializationAssignment::with_default(materialization)` gives every sub-DAG -that is not set explicitly `materialization` where the DAG allows it; a -`MatAtIngestion` default over a sub-DAG that can only run at query time leaves -that sub-DAG `NotMaterialized` instead of failing (the default applies to -`SummaryAgg` and `MaintainPopulation` nodes). The test helpers `validate_at_ingestion` and -`planned_data_state` (feature `test-support`) use it to check, at planning time and without keeping -anything, that a candidate stays valid when every sub-DAG is `MatAtIngestion`. +assignment is applied. How much a materialized output retains (retention) is decided +later and is not in the assignment. + +A node the assignment does not set is not materialized. The test helper +`PhysicalPropertiesAssignment::materialized_at_ingestion_where_allowed(roots)` +(feature `test-support`) sets every `SummaryAgg`, `SummaryTumblingWindowChain` +and `MaintainPopulation` materialized at ingestion time where the DAG allows it (not +filtered, nothing below it that can only run at query time). The test +helpers `validate_at_ingestion` and `planned_data_state` use it to check, at +planning time and without materializing anything, that a candidate stays valid +when every sub-DAG is materialized at ingestion time. ## Exported DAG @@ -188,17 +191,20 @@ children as edges and no embedded sub-DAGs: expressions (`scalar(v)`). Every edge records the intermediate schema, the producer's data state and grouping/window compatibility. - Each node records `output_state` (timing plus `Raw` or `SummaryState`), - `output_schema` and `guarantee`. Export reads the timing written by - `apply_materialization_timings` and rejects an untimed node + `output_schema` and `guarantee`. Export reads each node's timing and lifetime + from the assignment `apply_physical_properties` completed and rejects a node + it does not list (`ExecutionDataStateError::UntimedNode`); it does not re-run data-state validation. -Each node also records its `materialization`, including the medium, so the -deployment knows which outputs to store and where; validation rejects a -`SummaryAgg` or `MaintainPopulation` whose timing differs from the one its -materialization implies, and any `MatAtIngestion` or `MatAtQuery` node at the -other timing. Ingestion work cannot depend on a future query result. The -deployment executes the plan as given and does not change its materialization +Each node also records its `lifetime`, including the medium, so the +deployment knows which outputs to materialize, for how long and where; validation +rejects a timing and lifetime that cannot go together (an ingestion-time +output materialized for `OneEvaluation`) and a `SummaryAgg` or `MaintainPopulation` +that is not materialized outside query time +(`PhysicalASAPDAGValidationError::PhysicalProperties`). Ingestion +work cannot depend on a future query result. The +deployment executes the plan as given and does not change its lifetime or timing ([#509 Assumption 8](../proposals/planner-layering.md#assumptions)); it must separately check that it has an implementation and a valid data source for each placement. @@ -231,7 +237,7 @@ failure probabilities are combined, and the score guarantee remains in the membership guarantee's child provenance. An exact request does not accept this approximate output path merely because its selected identities are certified. -The materialization assignment decides whether these summaries run at +The physical properties assignment decides whether these summaries run at ingestion time or at query time, and the exported DAG carries that placement. Either placement must give each evaluation a complete rate window and an isolated summary state, or maintain an equivalent replacement diff --git a/docs/design_docs/physical-planning-and-deployment.md b/docs/design_docs/physical-planning-and-deployment.md index cf597bf16..d0da2d084 100644 --- a/docs/design_docs/physical-planning-and-deployment.md +++ b/docs/design_docs/physical-planning-and-deployment.md @@ -39,22 +39,21 @@ The Logical Post-ASAP DAG is preceded by the Pre-ASAP DAG, the language-independent query semantics before summary selection. Both are logical `OperatorNode` DAGs. `compile_physical_asap_dag` exports the selected DAG as a `PhysicalASAPDAG`, which is the Physical Plan Compiler's input. Its per-node execution phase (ingestion or query time) is -decided by a `MaterializationAssignment`, as the layer contract below states. +decided by a `PhysicalPropertiesAssignment`, as the layer contract below states. ### Layer contract 1. **Logical Post-ASAP** (`CandidateLogicalASAPDAGs`) decides what to compute: summary families, readouts and sharing. It does not decide placement; timing that a realization strategy writes while building a candidate is provisional. -2. **Materialization** (Stage 2, #509) chooses `NotMaterialized`, - `MatAtIngestion(medium)`, `MatAtQuery(medium)` or - `MatAtQueryKeptAcrossEvaluations(medium)` for each sub-DAG whose - output may be stored (a `SummaryAgg` or a `MaintainPopulation`) and records - it in a `MaterializationAssignment`. `apply_materialization_timings` writes - every node's `ExecutionTiming` and `Materialization`: a `MatAtIngestion` - state and all of its inputs run at ingestion time; readouts, other - consumers, and the other states run at query time. The default assignment - is `NotMaterialized` everywhere; `PlanOutput::physical_asap_dag` applies it +2. **Materialization** (Stage 2, #509) chooses, for each sub-DAG whose + output may be stored (a `SummaryAgg` or a `MaintainPopulation`), whether + it is materialized and with which timing and `Lifetime` (`None`, + `OneEvaluation(medium)` or `AcrossEvaluations(medium)`), and records it in + a `PhysicalPropertiesAssignment`. `apply_physical_properties` writes every + node's `ExecutionTiming` and `Lifetime`: a state materialized at ingestion time and + all of its inputs run at ingestion time; readouts, other consumers, and the + other states run at query time. The default assignment materializes nothing; `PlanOutput::physical_asap_dag` applies it only to nodes the optimization pass left untimed (a pass's own timing, such as Stage 3's ingestion-time tumbling windows, is kept). 3. **Physical compile** (executor) reads the plan's materialized views: each @@ -97,7 +96,7 @@ and a feasible candidate that loses on cost. Absence is not a cost comparison. For `sum by(job)(rate(m[1m]))`, Rate remains per series before grouped Sum. `CandidateLogicalASAPDAGs` offers one such candidate, with a per-series Rate state and a grouped -Sum state. Its materialization assignment places it: an ingestion-time Sum state +Sum state. Its physical properties assignment places it: an ingestion-time Sum state finalizes Rate and builds Sum within a bounded precompute run; a query-time Sum over an ingestion-time Rate state leaves the Rate readout and Sum in the query DAG. Storing a value requires its exact evaluation window, revision, readiness and serving @@ -113,7 +112,7 @@ Planner's candidate space decides what to compute, not placement. For an instant-vector PromQL TopK, Planner resolves rows that carry the complete series identity and lists the current-series heap realizations per root with the other candidates, unranked. Precompute or query-time placement of Rate and grouped Sum -is not a separate Planner candidate: the materialization assignment sets +is not a separate Planner candidate: the physical properties assignment sets each node's timing, and the physical compiler reads it. This is the target ownership contract. A backend path that still reconstructs @@ -473,8 +472,8 @@ These derivations are methods on `PhysicalASAPDAG` in `asap-types`, so a runtime needs no dependency on the planner crate. A **materialized view** is a node whose output is kept across evaluations -(`MatAtIngestion` or `MatAtQueryKeptAcrossEvaluations`). A `MatAtQuery` node is -not a view: its output is discarded after one evaluation. Everything a runtime +(lifetime `AcrossEvaluations`, at ingestion or at query time). A node materialized for +`OneEvaluation` is not a view: its output is discarded after one evaluation. Everything a runtime needs that the DAG determines is derived from the DAG, so the plan file has no second copy to keep consistent. Stage 3's selection report is returned beside the plan, not inside it. diff --git a/docs/design_docs/proposals/asap-aware-mapping/maintained-populations.md b/docs/design_docs/proposals/asap-aware-mapping/maintained-populations.md index 0975ba578..ffb87c859 100644 --- a/docs/design_docs/proposals/asap-aware-mapping/maintained-populations.md +++ b/docs/design_docs/proposals/asap-aware-mapping/maintained-populations.md @@ -79,9 +79,9 @@ input `EvaluatePopulation` runs at query time. `MaintainPopulation` has no fixed timing: like a `SummaryAgg`, it follows its materialization -(`NotMaterialized`, `MatAtIngestion(medium)` or `MatAtQuery(medium)`, #509 +(`Lifetime::None`, `(IngestionTime, AcrossEvaluations(medium))` or `(QueryTime, OneEvaluation(medium))`, #509 §Materialization), which Stage 2 chooses; it runs at ingestion time only when -it is `MatAtIngestion`. +it is ingestion-time `AcrossEvaluations`. The rule examines compatible workload roots, sets `max_k` to the largest requested k and enables quantile readout if any consumer needs it. It emits a candidate for diff --git a/docs/design_docs/proposals/asap-primitive-schema.md b/docs/design_docs/proposals/asap-primitive-schema.md index ee15ead49..4bf59e9ea 100644 --- a/docs/design_docs/proposals/asap-primitive-schema.md +++ b/docs/design_docs/proposals/asap-primitive-schema.md @@ -296,7 +296,7 @@ In the worked example, `latency < 100` and `region = 'us'` move into `selection` **When two definitions are equal.** Merging (§5.6) requires equal definitions. -- They must have the same structure. Planning details are ignored: `timing`, `guarantee` and `coverage_cache`. So a tumbling window built at ingestion time can merge with one built at query time. +- They must have the same structure. Planning details are ignored: `guarantee` and `coverage_cache`. Operator nodes carry no timing (#1200), so a tumbling window built at ingestion time can merge with one built at query time. - `SummaryUpdate.weight_domain` is compared too. It is computed from the rest, so it differs only if something is wrong. - States over different tables never merge: a KLL over `m1` and one over `m2` have different definitions. To combine tables, put a `UNION ALL` with a column that marks the source table below one `SummaryAgg`; that column can then be used in `selection` or in the grouping. @@ -921,9 +921,6 @@ pub struct OperatorNode { /// The accuracy statement of the output, once known. `None` does not /// mean exact. pub guarantee: Option, - /// When the node runs: `IngestionTime` or `QueryTime`. `None` until - /// planning assigns it. - pub timing: Option, /// Cache for `coverage()`. Filled on first use, never serialized, /// ignored by equality, emptied on clone. Not a source of truth: /// coverage can always be derived again. diff --git a/docs/design_docs/proposals/joint-planning-implementation-plan.md b/docs/design_docs/proposals/joint-planning-implementation-plan.md index e561406ac..89b8dde7b 100644 --- a/docs/design_docs/proposals/joint-planning-implementation-plan.md +++ b/docs/design_docs/proposals/joint-planning-implementation-plan.md @@ -12,7 +12,7 @@ Preserve supported accuracy semantics, physical eligibility, cost objectives, sc | Scope | Current implementation | Responsibility | | --- | --- | --- | -| Logical identity | `logical-asap-optimization/src/stage0.rs`; `types/src/stages/` | `DagOfLogicalMemoGroups`, `MemoGroup` (`asap_types::stages`; built by `generate_dag_of_logical_memo_groups`), stable initial query/group references. | +| Logical identity | `logical-asap-optimization/src/stage0.rs`; `types/src/stages/` | `DagOfLogicalMemoGroups`, `MemoGroup` (`asap_types::stages`; built by `memo_of_queries`), stable initial query/group references. | | Computation and sharing rules | `logical-asap-optimization/src/pass2/memo/{candidates,rules,state_groups,shared_configurations,window_candidates}` | Populate groups, retain local candidates, expose shared states and compatible rewrites. | | Physical legality | `physical-asap-optimization` | Executable operators, timing, materialization eligibility, and `PhysicalWorkloadPlan`. | | Candidate dependency inventory | `physical-asap-optimization/src/catalog`; `types/src/stages/physical` | Local `PhysicalCandidate`, `ProducerRequirement` (`asap_types::stages`), compatible producers, structural pruning, the selected DAG (`DagOfSelectedCandidates`, one node per run of a candidate), and reconstruction. Stage 2 makes no execution-scope copies; today's Stage 3 engine makes its own (`plan-selection/src/search/candidates/scopes.rs`). | @@ -23,7 +23,7 @@ Preserve supported accuracy semantics, physical eligibility, cost objectives, sc `PhysicalCandidate` is a local selectable computation. `PhysicalWorkloadPlan` is a complete Stage 2 workload DAG; they are not interchangeable. Physical candidate configuration in selection invokes the authoritative Stage 2 legality code rather than duplicating its rules. Logical rules do not depend on the solver crate. -Pass 1 has one entry, `generate_per_computation_logical_memo_candidates`, which populates the already-created memo groups (its target list, `PerComputationCandidates`, is a crate-private working form). This is not a second workload selector. Pass 2 can add intermediate state groups and consumer candidates; sharing a component does not merge distinct requested results: +Pass 1 has one entry, `pass1_memo`, which populates the already-created memo groups (its target list, `PerComputationCandidates`, is a crate-private working form). This is not a second workload selector. Pass 2 can add intermediate state groups and consumer candidates; sharing a component does not merge distinct requested results: ```text G95 = { ExactP95, LocalKLLQuantileP95, EstimateP95(G_KLL) } diff --git a/docs/design_docs/proposals/joint-planning-search.md b/docs/design_docs/proposals/joint-planning-search.md index 713a7e356..2558a5341 100644 --- a/docs/design_docs/proposals/joint-planning-search.md +++ b/docs/design_docs/proposals/joint-planning-search.md @@ -125,17 +125,17 @@ G_KLL += MergeLatest15(G_tumbling_windows) Non-summary query-time materialization must be a query output or have a selected immediate consumer used by only one query; the materialized node itself must have multiple query readers. These connection-dependent constraints match Stage 2's highest-shared-node rule and are enforced in MILP. Summary-state eligibility remains separate. -Physical rules provide executable operators, configurations, and materialization assignments for the generated computations. For example, a supported tumbling window implementation can maintain state at ingestion and merge it at evaluation. This introduces two timing decisions within the same query, rather than one materialization choice for the whole query. Eligibility that depends on consumers is checked against the selected connections; considering every potential consumer at once could incorrectly reject a useful candidate. +Physical rules provide executable operators, configurations, and physical properties assignments for the generated computations. For example, a supported tumbling window implementation can maintain state at ingestion and merge it at evaluation. This introduces two timing decisions within the same query, rather than one materialization choice for the whole query. Eligibility that depends on consumers is checked against the selected connections; considering every potential consumer at once could incorrectly reject a useful candidate. For the running example, assume evaluations occur at aligned minute boundaries and the deployment supports ingestion-time tumbling window maintenance. Stage 2 can expand the state groups as follows; `s` is a generated KLL capacity: ```text -Group Physical candidate Computation Materialization -G_tumbling_windows P_ingest(s) Maintain aligned 1m tumbling windows MatAtIngestion(Memory) -G_KLL K_merge(s) MergeLatest15(G_tumbling_windows) MatAtQuery(Memory) -G_KLL K_build(s) BuildKLL(G_input, 15m) MatAtQuery(Memory) -G95 E95 EstimateQuantile(.95, G_KLL) NotMaterialized -G99 E99 EstimateQuantile(.99, G_KLL) NotMaterialized +Group Physical candidate Computation (timing, lifetime) +G_tumbling_windows P_ingest(s) Maintain aligned 1m tumbling windows (IngestionTime, AcrossEvaluations(Memory)) +G_KLL K_merge(s) MergeLatest15(G_tumbling_windows) (QueryTime, OneEvaluation(Memory)) +G_KLL K_build(s) BuildKLL(G_input, 15m) (QueryTime, OneEvaluation(Memory)) +G95 E95 EstimateQuantile(.95, G_KLL) (QueryTime, None) +G99 E99 EstimateQuantile(.99, G_KLL) (QueryTime, None) ``` `P_ingest(s)` retains the tumbling windows needed to cover the 15m window. `K_merge(s)` constructs window state from those tumbling windows at evaluation; `K_build(s)` instead builds window state from available raw input. Each candidate records its input requirements and costs. The estimators can use either compatible state candidate. Exact and local KLL candidates from Pass 1 also receive supported physical implementations and remain available. Stage 2 has generated choices; it has not yet selected a build or connected an estimator to it. @@ -189,7 +189,7 @@ The following cases make the accounting concrete. Let `B` be the cost of one who | Repeated common evaluations | Both queries evaluate together `n` times; state is discarded after each evaluation. | `n × (B + E95 + E99)`. Selecting one candidate does not mean building only once over the entire horizon. | | Different evaluation timestamps | Q1 evaluates every minute; Q2 every five minutes on the same grid. Both use a 15m window. | Over 15 evaluation ticks, a shareable query-local build executes 15 times, with 15 P95 extractions and 3 P99 extractions: `15B + 15E95 + 3E99`. This assumes sharing at the three common timestamps; an earlier window state cannot automatically answer a later window. | | Ingestion-maintained tumbling windows | One tumbling-window chain feeds a shared query-time merge and both estimators. | Ingestion updates and tumbling-window retention are charged once for the chain. Add `n × (M + E95 + E99)` for `n` common evaluations, plus query-local merge memory. | -| State retained across evaluations | An eligible tumbling-window-chain candidate uses `MatAtQueryKeptAcrossEvaluations(Memory)`. | Charge initial construction, required refresh/update work, retained memory/storage over its lifetime, and evaluation work. Reuse does not eliminate freshness maintenance; generate this case only when supported. | +| State retained across evaluations | An eligible tumbling-window-chain candidate uses `(QueryTime, AcrossEvaluations(Memory))`. | Charge initial construction, required refresh/update work, retained memory/storage over its lifetime, and evaluation work. Reuse does not eliminate freshness maintenance; generate this case only when supported. | | Multiple selected candidates in one group | Some consumers use a small KLL state and others a separate large state. | Charge each selected state's build/update and retention, then each consumer's local work. Equal group identity does not collapse two physical candidates into one charge. | | Shared raw-data retention | Two consumers need 5m and 15m of the same raw source and compatible retention scope. | Both consumers connect to `Retain15m` in `G_raw[d]`. Charge that candidate once; reads and intermediate KLL state have separate owners. | | Per-request overhead | Two requests share computation but still require separate response handling. | Report fixed overhead as `Σ_q N_q × h_q`; omit this constant from the optimization objective but retain its applicable resource usage. Decision-dependent overhead stays in the optimization model. | diff --git a/docs/design_docs/proposals/operator-sharing.md b/docs/design_docs/proposals/operator-sharing.md index 5f9deb85f..cb822d7a6 100644 --- a/docs/design_docs/proposals/operator-sharing.md +++ b/docs/design_docs/proposals/operator-sharing.md @@ -1,6 +1,9 @@ # Sharing Operators Between Pre-ASAP IR and Post-ASAP IR > Status: proposal, not implemented. Audience: planner designers and architects. +> Note: since #1200, operator nodes carry no `timing`; physical stages keep each +> node's timing and lifetime in a `PhysicalPropertiesAssignment`, and +> `validate_execution_timing` no longer exists. The `timing` text below predates that. > Addresses [#468](https://github.com/ProjectASAP/ASAPPlanner/issues/468). > Companion: [Decoupling operators from scalar expressions](decoupling_op_and_expr.md). diff --git a/docs/design_docs/proposals/planner-layering-example1-acceptance.md b/docs/design_docs/proposals/planner-layering-example1-acceptance.md index c66cd3aa6..0dd10ff90 100644 --- a/docs/design_docs/proposals/planner-layering-example1-acceptance.md +++ b/docs/design_docs/proposals/planner-layering-example1-acceptance.md @@ -19,7 +19,7 @@ whose expectation needs a feature that is not in the stack yet is |---|---|---| | Query-language frontends | 1 workload `LogicalDAG` | same | | Logical ASAP-aware optimization | Pass 1 (3) × identical-expression rule (2) × window-composition rule (3 × 3) = **54** | Pass 1 (3), each with 2 identical-expression candidates and 6 window-composition candidates (Q1's rate and Q2's option in six 10-s tumbling windows, alone or both, each also with a shared scan) = **27**. Sliding windows are not implemented. | -| Physical ASAP-aware optimization | Materialization options per window form = **156** | Physical operator implementation, plus materialization: each logical candidate with every sub-DAG `NotMaterialized` (27: the doc's Raw plans, with tumbling windows rebuilt at every refresh), each tumbling-window chain also `MatAtIngestion(Memory)` (30 more), and each shared input built once per evaluation, `MatAtQuery(Memory)`, with every tumbling-window chain at query time (15 more) = **72** | +| Physical ASAP-aware optimization | Physical properties options per window form = **156** | Physical operator implementation, plus materialization: each logical candidate with every sub-DAG `Lifetime::None` (27: the doc's Raw plans, with tumbling windows rebuilt at every refresh), each tumbling-window chain also `(IngestionTime, AcrossEvaluations(Memory))` (30 more), and each shared input built once per evaluation, `(QueryTime, OneEvaluation(Memory))`, with every tumbling-window chain at query time (15 more) = **72** | | Plan selection | 1 plan | 1 plan | Every MVP candidate is one of the doc's candidates: in logical ASAP-aware @@ -100,26 +100,26 @@ The three whole-window candidates without a shared input: | Id | From | Q2 physical operators | Materialization | |---|---|---|---| -| P1 | L1 | sort (partition by job) → limit 10 | all `NotMaterialized` | -| P10 | L10 | CMS + heap build → top-10 estimate | all `NotMaterialized` | -| P19 | L19 | CountSketch + heap build → top-10 estimate | all `NotMaterialized` | +| P1 | L1 | sort (partition by job) → limit 10 | all `Lifetime::None` | +| P10 | L10 | CMS + heap build → top-10 estimate | all `Lifetime::None` | +| P19 | L19 | CountSketch + heap build → top-10 estimate | all `Lifetime::None` | Q1 is the same in all three: scan → range → rate accumulator → sum accumulator, each finalized. Invariants: -* Every logical candidate keeps its all-`NotMaterialized` physical candidate, +* Every logical candidate keeps its all-`Lifetime::None` physical candidate, and none is pruned, so no candidate is dropped before plan selection. A logical candidate with `n` tumbling-window chains has 2ⁿ physical candidates: each - chain `NotMaterialized` or `MatAtIngestion(Memory)` (ids `P-m`). + chain `Lifetime::None` or `(IngestionTime, AcrossEvaluations(Memory))` (ids `P-m`). Only tumbling-window chains are placed at ingestion time; Q1's rate tumbling windows may be, because continuously ingesting PromQL data is taken to arrive in time order per series. A logical candidate with a shared input has one more: the highest node both queries read (the scan, or the time range above - it) `MatAtQuery(Memory)`, with every chain at query time. A chain at + it) `(QueryTime, OneEvaluation(Memory))`, with every chain at query time. A chain at ingestion time leaves one query reading the input at query time, so - the input stays `NotMaterialized` there. + the input stays `Lifetime::None` there. * Each physical candidate keeps its logical Q2 option and input sharing. * Exact top-*k* is implemented as a sort followed by a limit: each job's series sorted by the ranked value, descending, then a `Project` that @@ -128,7 +128,7 @@ Invariants: returns each job's `k` heaviest series, and a `Project` to `[job, topk_10]` followed by each ranked series' identity. There is no merge node, because there are no window summaries. -* In the whole-window candidates, every node is `NotMaterialized` and runs +* In the whole-window candidates, every node is `Lifetime::None` and runs at query time. ## Plan selection: 1 plan @@ -137,7 +137,7 @@ Invariants: |---|---|---| | P3-m1 | selected | all exact over one time range built once per evaluation for both queries, at 4.38 per second (cost model 1) | | P2-m1 | valid · costlier | all exact over one scan built once per evaluation, at 4.78 per second: each query still reads its own 1 m time range | -| P1, P2, P3 | valid · costlier | all exact at 7.01 per second, with separate inputs (P1) or a `NotMaterialized` shared scan (P2) or scan and range (P3), charged once per query; they tie with each other up to rounding | +| P1, P2, P3 | valid · costlier | all exact at 7.01 per second, with separate inputs (P1) or a `Lifetime::None` shared scan (P2) or scan and range (P3), charged once per query; they tie with each other up to rounding | | P10 | invalid | Count-Min + heap: a top-*k* output has no guarantee (a heap's bound is per item frequency, not top-*k* membership), and the deployment has no evidence that it meets the target | | P19 | invalid | CountSketch + heap: the same reason | @@ -155,7 +155,7 @@ Invariants: reason, and marked invalid or valid but costlier. * Only valid candidates are priced. Each valid candidate's cost has one entry per DAG node, and `cost_per_second` is their sum. A materialized node is charged - once. A `NotMaterialized` node is charged once per consumer that reads it + once. A `Lifetime::None` node is charged once per consumer that reads it (its multiplier), where a consumer is a query or a materialized node and a read stops at materialized nodes. `stage3_plan_selection_charging_helper_*` check the helper on a hand-built candidate with a @@ -185,13 +185,13 @@ per evaluation, within its 100 ms. deployment evidence every heap candidate is invalid and the exact plan is selected. 4. **Physical ASAP-aware optimization is materialization in the doc's - example.** Example 1 lists only window-form and materialization options. + example.** Example 1 lists only window-form and physical properties options. The planner also reads "physical operator implementation" from the physical ASAP-aware optimization section ("TopK as a sort followed by a limit"), and enumerates materialization for summary builds and shared - inputs: each tumbling-window chain is `NotMaterialized` or `MatAtIngestion(Memory)`, - and the highest node both queries read is `NotMaterialized` or - `MatAtQuery(Memory)`. + inputs: each tumbling-window chain is `Lifetime::None` or `(IngestionTime, AcrossEvaluations(Memory))`, + and the highest node both queries read is `Lifetime::None` or + `(QueryTime, OneEvaluation(Memory))`. 5. **Where exact top-*k* becomes sort + limit.** The Pass 1 diagram draws exact Q2 as "sort + limit 10 per job", but the frontend emits `Aggregate{[TopK]}` and the physical ASAP-aware optimization section calls @@ -201,9 +201,9 @@ per evaluation, within its 100 ms. column" without naming the inner sketch. The test assumes `HydraCms`. 7. **"Shared summary charged once"** does not apply literally: the doc says no summary is shared in Example 1. Plan selection charges a - `NotMaterialized` node once per consuming query and a materialized one - once; the tests check that rule. A `NotMaterialized` shared input - therefore saves nothing here; built once per evaluation (`MatAtQuery`), + `Lifetime::None` node once per consuming query and a materialized one + once; the tests check that rule. A `Lifetime::None` shared input + therefore saves nothing here; built once per evaluation (`OneEvaluation`), it is read once, and that candidate (the doc's "Raw with a shared input") is selected. 8. **Latency.** The doc says an exact top 10 rebuilt at every refresh "may diff --git a/docs/design_docs/proposals/planner-layering-example2-acceptance.md b/docs/design_docs/proposals/planner-layering-example2-acceptance.md index cbb2214e7..0ec29bd09 100644 --- a/docs/design_docs/proposals/planner-layering-example2-acceptance.md +++ b/docs/design_docs/proposals/planner-layering-example2-acceptance.md @@ -48,8 +48,8 @@ workload itself. | Query-language frontends | One summary-free DAG: `Distinct`, `Entropy`, `L2` over `flows.src_ip`, last 1 m | Same. Q1 is a distinct count; Q2 is `Entropy` and Q3 is `L2` over the per-source counts, each with an exact population count for the empty-input NULL. | | Logical ASAP-aware optimization, Pass 1 | Each computation: exact, a specialized summary, UnivMon. 3 × 3 × 3 = **27** | Q1: exact, HLL, KMV, UnivMon (no Theta: `asap_sketchlib` has no Theta sketch). Q2, Q3: exact, UnivMon (no specialized entropy or norm summary). 4 × 2 × 2 = **16** | | Logical ASAP-aware optimization, Pass 2 | Summary-capability rule: + 1 (one UnivMon for all three, sized for ε = 0.01) + 9 (3 pairs share a UnivMon × 3 options of the third). Independent kept. **37**. Window composition left out. | Identical-expression rule (Q2 and Q3's shared per-source count, and the shared scan) and summary-capability rule (one UnivMon for two of the queries or all three, sized with each parameter's largest value among its readers): **82** | -| Physical ASAP-aware optimization | Same options as Example 1: no window form → rebuilt from raw samples each refresh. | Each logical candidate with every sub-DAG `NotMaterialized`, plus `MatAtQuery(Memory)` for each summary build two or more queries read: **184** | -| Plan selection | One UnivMon sized for ε = 0.01 vs three separate summaries; the shared one usually wins (one update per flow record instead of three). | 164 invalid: HLL or UnivMon for Q1 (120), whose bound has failure probability None without a distinct bound for `flows`, and UnivMon for Q2 (44), whose entropy has no guarantee. 20 valid. Under cost model 1, KMV for Q1 with the other two exact (P33) costs 6.097 per second and P1, all exact, 6.104, the exact distinct count's values held while Q1 runs; the shared per-source count costs 7.46 rebuilt for each query and 7.06 kept `MatAtQuery`; with the shared scan also built once per evaluation, KMV for Q1 (P35-m3) is selected at 5.745, and all exact costs 5.752; UnivMon for Q3 costs 8.09. | +| Physical ASAP-aware optimization | Same options as Example 1: no window form → rebuilt from raw samples each refresh. | Each logical candidate with every sub-DAG `Lifetime::None`, plus `(QueryTime, OneEvaluation(Memory))` for each summary build two or more queries read: **184** | +| Plan selection | One UnivMon sized for ε = 0.01 vs three separate summaries; the shared one usually wins (one update per flow record instead of three). | 164 invalid: HLL or UnivMon for Q1 (120), whose bound has failure probability None without a distinct bound for `flows`, and UnivMon for Q2 (44), whose entropy has no guarantee. 20 valid. Under cost model 1, KMV for Q1 with the other two exact (P33) costs 6.097 per second and P1, all exact, 6.104, the exact distinct count's values held while Q1 runs; the shared per-source count costs 7.46 rebuilt for each query and 7.06 materialized `OneEvaluation`; with the shared scan also built once per evaluation, KMV for Q1 (P35-m3) is selected at 5.745, and all exact costs 5.752; UnivMon for Q3 costs 8.09. | ## Invariants and tests @@ -84,12 +84,12 @@ workload itself. | Invariant | Test | Status | |---|---|---| -| Each logical candidate keeps one physical candidate with nothing materialized, plus one per other materialization assignment; none pruned | `stage2_physical_asap_optimization_keeps_every_logical_candidate` | passes | +| Each logical candidate keeps one physical candidate with nothing materialized, plus one per other physical properties assignment; none pruned | `stage2_physical_asap_optimization_keeps_every_logical_candidate` | passes | | The shared UnivMon stays one build read by all three queries | `stage2_physical_asap_optimization_keeps_the_shared_univmon_as_one_build` | passes | Window-composition candidates are out of scope: the doc leaves them out for this example, and a summary with no window form is rebuilt at every refresh -(Example 1's table) unless a shared build is kept `MatAtQuery` within one +(Example 1's table) unless a shared build is materialized `OneEvaluation` within one evaluation. ### Plan selection @@ -98,7 +98,7 @@ evaluation. |---|---|---| | One selected id; every other candidate rejected once with a reason | `stage3_plan_selection_selects_one_and_explains_the_rest` | passes | | The selected plan is the cheapest valid one | `stage3_plan_selection_selects_cheapest_valid` | passes | -| Each node of a valid candidate is charged once per consuming query while `NotMaterialized`, once when materialized | `stage3_plan_selection_charges_each_node_per_consuming_query` | passes | +| Each node of a valid candidate is charged once per consuming query while `Lifetime::None`, once when materialized | `stage3_plan_selection_charges_each_node_per_consuming_query` | passes | | No candidate builds a Theta sketch | `stage3_plan_selection_has_no_theta_candidate` | passes | | A UnivMon estimate has a guarantee, so it is judged against its target instead of rejected for having none | `stage3_plan_selection_judges_univmon_with_an_accuracy_model` | ignored: needs a UnivMon entropy accuracy model (#788); L2 and the distinct count have one | | Given a distinct bound for the source, classic HLL's failure probability is bound and Q1's HLL is accepted; a larger bound enlarges its precision, also in a tumbling-window chain | `stage3_plan_selection_accepts_hll_given_a_distinct_bound`, `…_enlarges_hll_for_a_larger_distinct_bound`, `…_enlarges_tumbling_window_hll` | passes | @@ -112,8 +112,8 @@ The doc's "the shared candidate usually wins" depends on the cost model, so it is tested only as the dominance above: one build sized for the strictest consumer costs no more than three builds that include one of that size. Whether it also beats the exact plans is the model's call. The dominance needs the -shared UnivMon materialized: plan selection charges a `NotMaterialized` -node once per consuming query, so unmaterialized it costs three builds of the +shared UnivMon materialized: plan selection charges a `Lifetime::None` +node once per consuming query, so not_materialized it costs three builds of the strictest size. ## Ambiguities and conflicts with the planner diff --git a/docs/design_docs/proposals/planner-layering-example3-acceptance.md b/docs/design_docs/proposals/planner-layering-example3-acceptance.md index f6f253b9d..8966edd70 100644 --- a/docs/design_docs/proposals/planner-layering-example3-acceptance.md +++ b/docs/design_docs/proposals/planner-layering-example3-acceptance.md @@ -47,7 +47,7 @@ Shared data workload, except Pattern A's `arrival` is `Mixed`. | Query-language frontends | A: five scan → range → quantile chains. B: one. | Same; q3–q5 also have a `time_shift`. | | Logical ASAP-aware optimization, Pass 1 | A: exact or KLL over its own interval per query, 2⁵ = 32. B: exact, KLL. | A: exact, KLL (k = 547), DDSketch per query, 3⁵ = **243**. B: exact, KLL (k = 269), DDSketch, **3**. | | Logical ASAP-aware optimization, Pass 2 | A: window composition adds one EH of KLLs over [T − 5 y, T], one merge + p99 estimate per query, plus a candidate for every grouping of two or more queries onto shared window summaries ("hundreds"). B: each Pass 1 option × {none, sliding L = 5 min s = 1 min (5 active windows), 1-min tumbling (merge 5)} = 6. Independent kept. | Identical-expression rule: each Pass 1 candidate also gets its shared-input candidates (A: 3 per Pass 1 candidate; B: none). Window-composition rule, tumbling windows only: B's KLL and DDSketch each also come as 1-min tumbling windows (a `SummaryTumblingWindowChain` of 5, merged before the p99 estimate), **5** in all; A is run once, so it gets none. No sliding windows or EHs. | -| Physical ASAP-aware optimization | See Example 4. | Each logical candidate with every sub-DAG `NotMaterialized` (A: 972; B: 5), plus, for A, each highest shared input (a scan or a shift) also built once per evaluation, `MatAtQuery(Memory)` (A: 2,187 in all), and, for B, each tumbling-window chain maintained `MatAtIngestion(Memory)` (B: 7 in all). | +| Physical ASAP-aware optimization | See Example 4. | Each logical candidate with every sub-DAG `Lifetime::None` (A: 972; B: 5), plus, for A, each highest shared input (a scan or a shift) also built once per evaluation, `(QueryTime, OneEvaluation(Memory))` (A: 2,187 in all), and, for B, each tumbling-window chain maintained `(IngestionTime, AcrossEvaluations(Memory))` (B: 7 in all). | | Plan selection | Not stated for Example 3. | A: 288 valid, 1,899 invalid: every candidate with a DDSketch misses the rank target (its bound is relative). KLL for all five queries (P485) is selected at 207,189.43 per second (cost model 1); all exact costs 327,132,784.05, about 1,600 times as much, its values held for the whole of each long query. B: KLL (P2, 2.363 per second); the 1-min KLL tumbling windows rebuilt at every evaluation cost 3.611 (the same rows and updates plus merging 5 tumbling windows per series, and the tumbling windows held while the query runs); maintained at ingestion time they keep six tumbling windows of 1,000,000 per-series KLLs for 654.54; exact costs 2.458; DDSketch, whole or in tumbling windows, misses the rank target. Every valid B plan meets 200 ms: 126 ms for the whole-window KLL, 131 ms for the rebuilt tumbling windows, 7 ms for the maintained tumbling windows, 144 ms exact. | ## Invariants and tests @@ -66,7 +66,7 @@ Shared data workload, except Pattern A's `arrival` is `Mixed`. | The EH candidate coexists with the five independent KLLs | `stage1_logical_asap_optimization_a_keeps_independent_and_shared_window_summaries` | ignored: #628, #634 | | Every pair of queries shares a window summary in some candidate | `stage1_logical_asap_optimization_a_window_composition_groups_every_pair` | ignored: #628, #634 | | One selected id, the rest explained; the selected plan is the cheapest valid one | `stage3_plan_selection_a_selects_cheapest_valid` | passes | -| Each node of a valid candidate is charged once per consuming query while `NotMaterialized`, once when materialized | `stage3_plan_selection_a_charges_each_node_per_consuming_query` | passes | +| Each node of a valid candidate is charged once per consuming query while `Lifetime::None`, once when materialized | `stage3_plan_selection_a_charges_each_node_per_consuming_query` | passes | | For the same per-computation choices, sharing the scan costs no more than separate scans | `stage3_plan_selection_a_shared_scan_is_not_costlier` | ignored: #632 (see ambiguity 7) | ### Pattern B diff --git a/docs/design_docs/proposals/planner-layering-example4-acceptance.md b/docs/design_docs/proposals/planner-layering-example4-acceptance.md index ae9371317..3565c9d27 100644 --- a/docs/design_docs/proposals/planner-layering-example4-acceptance.md +++ b/docs/design_docs/proposals/planner-layering-example4-acceptance.md @@ -50,12 +50,12 @@ time, both kept. Not materializing it is the no-window plan, so it is not a separate option. **Today:** Pass 2 builds tumbling windows (`SummaryTumblingWindowChain`), and physical -ASAP-aware optimization enumerates `NotMaterialized`, `MatAtIngestion` and -`MatAtQuery` (state reused within one evaluation) for every summary build. +ASAP-aware optimization enumerates `Lifetime::None`, ingestion-time `AcrossEvaluations` and +`OneEvaluation` (state reused within one evaluation) for every summary build. B1 and B2 exist. B3 (tumbling windows built at query time and kept across evaluations) is offered when the deployment declares `query_time_results_retained` in its `deployment_capabilities`: Stage 2 then -adds `MatAtQueryKeptAcrossEvaluations` for a tumbling-window chain that builds +adds query-time `AcrossEvaluations` for a tumbling-window chain that builds one new tumbling window per evaluation. Pattern A's shared EH needs window segments shared across queries (#628, #634), so A1–A3 do not exist yet. B3's 4 kept tumbling windows plus the newest cost 436.59 per second under cost model 1, @@ -63,12 +63,12 @@ between B2 (3.611) and B1 (654.54). ## Invariants and tests -**Reading materialization.** The physical DAG records each node's -`Materialization` (#667). Tests read it through -`planner_layering_common::materialization(p, node)` without the storage -medium (`MatAtIngestion`, `MatAtQuery`, `NotMaterialized`), and the kept +**Reading physical properties.** The physical DAG records each node's +timing and `Lifetime` (#667, #1200). Tests read them through +`planner_layering_common::materializing(p, node)` without the lifetime length or +storage medium (`MaterializedAtIngestion`, `MaterializedAtQueryTime`, `NotMaterialized`), and the materialized event-time span through `retention_ms(p, node)`. The IR does not record -retention, so the adapter derives it the way plan selection prices kept +retention, so the adapter derives it the way plan selection prices materialized state: a chain of `N` tumbling windows of width `w` keeps `N·w`, plus one tumbling window while maintained at ingestion time; a whole-window build keeps its window, plus the next one at ingestion time; each plus the offset of the shifts below @@ -89,7 +89,7 @@ it. | The shared EH yields exactly A1, A2, A3 | `stage2_physical_asap_optimization_a_shared_eh_has_three_materialization_options` | ignored: the shared EH (#628, #634) | | At rest, A2 is not generated; A1 and A3 remain | `stage2_physical_asap_optimization_a_at_rest_drops_the_ingestion_time_option` | ignored: the shared EH (#628, #634) | | A materialized EH is one build node read by all five queries, priced once | `stage2_physical_asap_optimization_a_materialized_eh_is_built_once_for_all_consumers` | ignored: the shared EH (#628, #634) | -| A3 costs strictly more than A1 (five builds instead of one: a `NotMaterialized` node is charged per consuming query) | `stage3_plan_selection_a_rebuilding_per_query_costs_more_than_building_once` | ignored: the shared EH (#628, #634) | +| A3 costs strictly more than A1 (five builds instead of one: a `Lifetime::None` node is charged per consuming query) | `stage3_plan_selection_a_rebuilding_per_query_costs_more_than_building_once` | ignored: the shared EH (#628, #634) | | As given, A1 is the cheapest of the three | `stage3_plan_selection_a_once_adhoc_prefers_the_query_time_eh` | ignored: the shared EH (#628, #634) | | Monthly, A2's cost relative to A1 drops: cost(A2)/cost(A1) monthly < as given | `stage3_plan_selection_a_monthly_amortizes_ingestion_time_maintenance` | ignored: the shared EH (#628, #634) | @@ -127,11 +127,11 @@ it. 1. **Charging A3 (decided).** A3 is one logical EH node that is not materialized, so it is rebuilt by each of its five consumers. Plan - selection charges a `NotMaterialized` node once per consuming query + selection charges a `Lifetime::None` node once per consuming query (`NodeCost::multiplier`) and a materialized node once; physical ASAP-aware optimization does not duplicate the node. So A3 costs five EH builds and A1 one, as the tests require. The A3 tests stay ignored only because the - shared EH and its materialization options do not exist yet. + shared EH and its physical properties options do not exist yet. 2. **Recurrence input (resolved).** Materialization depends on `recurrence`, `predictability` and `arrival`: `optimize_physical_asap` takes the workload entries (`QueryWorkloadEntry`: recurrence, diff --git a/docs/design_docs/proposals/planner-layering.md b/docs/design_docs/proposals/planner-layering.md index cb93a3acb..3c0678a33 100644 --- a/docs/design_docs/proposals/planner-layering.md +++ b/docs/design_docs/proposals/planner-layering.md @@ -281,7 +281,7 @@ lists each stage's input and output types. one. It makes no implementation choice, so this section counts it as Stage 0 (its output); the code runs it as Stage 1 starts, which is why [Stage inputs and outputs](#stage-inputs-and-outputs-whole-workload-candidates-and-memo) -lists `generate_dag_of_logical_memo_groups` with Pass 1. `generate_dag_of_logical_memo_groups` runs +lists `memo_of_queries` with Pass 1. `memo_of_queries` runs `share_common_sub_dags` (`crates/logical-asap-optimization/src/stage0/cse.rs`) over the frontends' roots and numbers the shared nodes as groups. Two structurally equal nodes become one group only when sharing them is legal, that is, when every reader @@ -345,7 +345,7 @@ One query, evaluated every minute, with `max_error 0.01` and avg by (job) (quantile_over_time(0.5, data[5m]) / quantile_over_time(0.5, data[5m]) + quantile_over_time(0.9, data[5m])) ``` -**Step 1, Stage 0.** `generate_dag_of_logical_memo_groups` makes 7 groups: +**Step 1, Stage 0.** `memo_of_queries` makes 7 groups: | Group | Logical sub-expression | Reads | |---|---|---| @@ -523,12 +523,12 @@ candidates is in `crates/plan-selection/src/` (`select.rs`, `groups/`, `milp/`). | Step | Entry point | Path | |---|---|---| -| Stage 0 | `generate_dag_of_logical_memo_groups` → `DagOfLogicalMemoGroups`, `MemoGroup`, `MemoGroupId` | `stage0.rs` (the types: `asap_types::stages`); hash-consing is `share_common_sub_dags` in `stage0/cse.rs` | -| Pass 1 | `generate_per_computation_logical_memo_candidates` → `PerComputationLogicalMemoCandidates` (its `Memo` and its `PrunedCandidate`s; `ComputationTarget` is Pass 1's working type, in `asap_types::stages`), the one entry: inside it, the computation targets (walking the memo groups), the summaries the deployment does not support and those no query can use pruned, the candidates attached to the groups (`try_populate_pass1`), then the accuracy-scoped sizes | `pass1/logical_memo_candidates.rs`; `pass1/per_computation_candidates.rs`; `pass1/memo_population.rs` | +| Stage 0 | `memo_of_queries` → `DagOfLogicalMemoGroups`, `MemoGroup`, `MemoGroupId` | `stage0.rs` (the types: `asap_types::stages`); hash-consing is `share_common_sub_dags` in `stage0/cse.rs` | +| Pass 1 | `pass1_memo` → `PerComputationLogicalMemoCandidates` (its `Memo` and its `PrunedCandidate`s; `ComputationTarget` is Pass 1's working type, in `asap_types::stages`), the one entry: inside it, the computation targets (walking the memo groups), the summaries the deployment does not support and those no query can use pruned, the candidates attached to the groups (`try_populate_pass1`), then the accuracy-scoped sizes | `pass1/logical_memo_candidates.rs`; `pass1/per_computation_candidates.rs`; `pass1/memo_population.rs` | | Memo types | `DagOfLogicalMemoGroups`, `PerComputationLogicalMemoCandidates`, `HowToComputeAggregateMeasure`, Pass 1's working types (`ComputationTarget`, `FusedComputation`, `HavingEntry`, `HavingPlacement`), `Memo`, `CandidateForMemoGroup`, `OriginOfCandidateForMemoGroup`, `LogicalASAPMemoCandidates`, `CandidateGenerationTimeout`, `root_groups`, `queries_rooted_at` | `crates/types/src/stages/` (`asap_types::stages`) | | Pass 2 | `generate_logical_asap_memo_candidates` → `LogicalASAPMemoCandidates`, from Pass 1's output. It calls in this order: 1. window composition for every legal tumbling-window width (`add_local_window_composition`); 2–9. the rules in `pass2/memo/rules/`: scan widening, summary inputs, top-k reuse, finer aggregate, rollup, window rollup, summary results, state configurations; 10. `add_shared_summary_state_groups`, the shared builds | `pass2/logical_memo_candidates.rs` (`LogicalASAPMemoCandidates` itself: `asap_types::stages`); `pass2/memo/window_candidates.rs`, `pass2/memo/rules/`, `pass2/memo/state_groups.rs`, `pass2/memo/shared_configurations.rs` | | Memo to candidate DAGs | `enumerate`, `build_logical_asap_dags_from_chosen_candidates`, `ChosenCandidatePerMemoGroup`, `AllChosenCandidatesPerMemoGroup`, `ErrorBuildingLogicalASAPDagsFromChosenCandidates` | `pass2/memo/enumeration.rs` (re-exported at the crate root) | -| Production memo build and search | The facade's `run_planning` builds the DAG of logical memo groups and runs Pass 1 (`generate_per_computation_logical_memo_candidates`) once; then `generate_logical_asap_memo_candidates`, `generate_physical_asap_memo_candidates` and Stage 3's `select_plan`, once each, and builds the selected plan with Stage 2's `AssignmentRealizer::realize`. Behind `select_plan`: the group selection MILP (#1095), `groups/` and `milp/` | `crates/planner/src/run.rs`; `crates/plan-selection/src/select.rs` | +| Production memo build and search | The facade's `run_planning` builds the DAG of logical memo groups and runs Pass 1 (`pass1_memo`) once; then `generate_logical_asap_memo_candidates`, `generate_physical_asap_memo_candidates` and Stage 3's `select_plan`, once each, and builds the selected plan with Stage 2's `AssignmentRealizer::realize`. Behind `select_plan`: the group selection MILP (#1095), `groups/` and `milp/` | `crates/planner/src/run.rs`; `crates/plan-selection/src/select.rs` | ## ASAPPlanner Detailed Design: Planning Stages and their decisions/Strategies @@ -567,14 +567,14 @@ The sections below describe each stage. | 0. Frontends | nothing: a query that cannot be lowered fails planning with an error | — | — | | 1, Pass 1 | what the measure or the deployment cannot compute: `RequiresExact`, `NullableAvgInput`, `NoRelationalCountSketchRealization`, `UnsupportedByDeployment`, `QuantileOutsideUnitInterval`, `TopKHeapOverOneRowPerItem`; and, only when the deployment's accuracy model and evidence cannot accept an output without its guarantee, a summary no query reading it can use: `NoErrorBoundForQueryTarget` | `PerComputationLogicalMemoCandidates::pruned_candidates`, each on its aggregate's memo group | the way to compute the measure (`HowToComputeAggregateMeasure`) | | 1, Pass 2 | nothing: it only adds candidates | — | — | -| 2 | what cannot be implemented, built or kept: `PerEntityAggregateNotExecutable` (an exact per-entity `Aggregate` the executor has no kernel for), `RawDataNotRetained` (a query-time scan of raw data the deployment does not keep), `InputHasNoBuildableProducer`, `ReachesNoQueryOutput`, `TooFewQueryReaders` (shared within one evaluation with fewer than two reading queries) | `PhysicalASAPMemoCandidates::pruned_candidates`, each on its operator node's memo group | `PhysicalCandidateKey { operator_node, timing, materialization }` | +| 2 | what cannot be implemented, built or materialized: `PerEntityAggregateNotExecutable` (an exact per-entity `Aggregate` the executor has no kernel for), `RawDataNotRetained` (a query-time scan of raw data the deployment does not keep), `InputHasNoBuildableProducer`, `ReachesNoQueryOutput`, `TooFewQueryReaders` (shared within one evaluation with fewer than two reading queries) | `PhysicalASAPMemoCandidates::pruned_candidates`, each on its operator node's memo group | `PhysicalCandidateKey { operator_node, properties }` | | 3 | valid candidates that lose, and the checks only a whole plan can make (accuracy, latency, cluster capacity) | its own `RejectionCause` | — | Stage 2's key names what a pruned candidate would have been, since kept candidates are renumbered after pruning and a pruned one has no `PhysicalCandidateId`. Stage 2 emits one candidate per operator node, -timing and materialization, and makes no copies of it, so (`operator_node`, -`timing`, `materialization`) is unique. Operator +timing and lifetime, and makes no copies of it, so (`operator_node`, +`properties`) is unique. Operator nodes are kept whole, so `operator_node` indexes the output's `operator_nodes_of_candidates_for_memo_groups`. Pass 1's accuracy pruning (`NoErrorBoundForQueryTarget`) runs inside Pass 1's entry, before the @@ -597,10 +597,10 @@ the memo path's entry points. | Stage | Whole-workload candidates: input → output | Memo: input → output | Entry point (memo path) | |---|---|---|---| | 0. Frontends | Query text, language → one logical root per query, carrying its accuracy target | Same: a `QueryRoot` per query | `lower_promql_query_workload`, `lower_sql_dialect` (`asap-frontend-*`); `plan_installable` (`asap_planner`) runs the frontends and Stages 1–3 on the user input | -| 1, Pass 1 | Logical roots; the deployment's supported summaries → the Pass 1 inventory, every computation's candidates (`PerComputationCandidates`, now Pass 1's crate-private working form) | Logical roots → `DagOfLogicalMemoGroups` (hash-consed `MemoGroup`s); with the Pass 1 inventory, less the summaries the deployment does not support and those no query reading them can use → `PerComputationLogicalMemoCandidates`: the `Memo` (each group's `CandidateForMemoGroup`s, with the accuracy-scoped sizes) and the pruned candidates (`PrunedCandidate`) | `generate_dag_of_logical_memo_groups`, `generate_per_computation_logical_memo_candidates` | +| 1, Pass 1 | Logical roots; the deployment's supported summaries → the Pass 1 inventory, every computation's candidates (`PerComputationCandidates`, now Pass 1's crate-private working form) | Logical roots → `DagOfLogicalMemoGroups` (hash-consed `MemoGroup`s); with the Pass 1 inventory, less the summaries the deployment does not support and those no query reading them can use → `PerComputationLogicalMemoCandidates`: the `Memo` (each group's `CandidateForMemoGroup`s, with the accuracy-scoped sizes) and the pruned candidates (`PrunedCandidate`) | `memo_of_queries`, `pass1_memo` | | 1, Pass 2 | One Pass 1 choice per computation → that choice rewritten by each sharing rule, every subset of sharing choices (no longer in the code) | `Memo`; the entries' recurrence and range steps → the same memo with each rule's candidates added to the groups they implement | the memo rules (`pass2/memo/rules/`), `pass2::memo::window_candidates` | | 1, concrete summary-state groups | (part of Pass 2) | `Memo` → the memo with concrete summary-state groups: one group per shared size or tumbling-window layout that readers can read | `add_shared_summary_state_groups`, Pass 2's last step; the Pass 2 output is `LogicalASAPMemoCandidates` (`generate_logical_asap_memo_candidates`) | -| 2 | One whole-workload logical candidate; entries (recurrence, predictability), `DataWorkload`, query-time retention → its physical candidates, one per materialization assignment | `LogicalASAPMemoCandidates`; entries, `DataWorkload`, query-time and raw-data retention → `PhysicalASAPMemoCandidates`: every operator node of every candidate with each allowed timing and materialization (one physical candidate each, no copies), the producers of each input, reader constraints (each applied to every selected run of its candidate), and the pruned candidates (`PrunedCandidate`) | `generate_physical_asap_memo_candidates` (`asap_physical_asap_optimization`) | +| 2 | One whole-workload logical candidate; entries (recurrence, predictability), `DataWorkload`, query-time retention → its physical candidates, one per physical properties assignment | `LogicalASAPMemoCandidates`; entries, `DataWorkload`, query-time and raw-data retention → `PhysicalASAPMemoCandidates`: every operator node of every candidate with each allowed timing and lifetime (one physical candidate each, no copies), the producers of each input, reader constraints (each applied to every selected run of its candidate), and the pruned candidates (`PrunedCandidate`) | `generate_physical_asap_memo_candidates` (`asap_physical_asap_optimization`) | | 3 | Every physical candidate; accuracy targets, accuracy model and evidence, cost model → one plan | `PhysicalASAPMemoCandidates`; entries, `DataWorkload`, `AccuracyCheckInputs`, `CostInputs`, Stage 2's realizer → `Selection`: the selected plan (`selected_dag`, a `DagOfSelectedCandidates`: one node per run of a candidate, so a candidate recomputed by two readers or shared as two instances over different inputs is two nodes, with an edge per input), its cost, the accepted accuracy per query and the search report | `select_plan` (`asap_plan_selection`); the facade (`run_planning`, output `PlanningRun`) runs it once and builds the selected `PhysicalWorkloadPlan` with `AssignmentRealizer::realize`, one plan node per selected node | | 4. Execution | The selected `PhysicalASAPDAG` → query results | Same: an `InstallablePlan` (`{dag, query_texts}`) → query results | The deployment runtime installs the `InstallablePlan` | @@ -611,7 +611,7 @@ What changed with the memo: jointly. - Stage 2 is no longer a pass over whole candidate DAGs. It implements every operator node of every candidate for a memo group once, enumerates each - node's timings and materializations, and prunes what cannot be built, + node's timings and lifetimes, and prunes what cannot be built, used or kept; Stage 3 selects among what remains. - The concrete summary-state groups are expanded by Stage 3's search today, through the Stage 1 functions above. A planned change moves this expansion @@ -844,7 +844,7 @@ tumbling windows keeps the guarantee when the family's merged error is `Exact` o `AsIfBuiltOverUnion`), and a build shared by several readers takes the strictest reader's size, one state group per distinct size. The failure probability is part of Pass 1's size too -(`generate_per_computation_logical_memo_candidates`): each approximate input +(`pass1_memo`): each approximate input of a candidate with `n` budget parts is sized for `δ / (n · V)`, the smallest over the queries reaching its group, where `V` is the number of values the query's accuracy scope covers (1 per value, at most `⌊1 / share⌋` for a record @@ -1198,21 +1198,26 @@ the `DataWorkload`. Typical outcomes: A shared summary is materialized once for all its consumers. See Example 4. -The planner implements a subset of these options today. Stage 2 enumerates +The planner implements a subset of these options today. In code each option +is a timing and a lifetime (`PhysicalProperties`, #1200): materialized at +ingestion time is ingestion time with lifetime `AcrossEvaluations`; +materialized at query time is query time with lifetime `OneEvaluation` +(reused within one evaluation) or `AcrossEvaluations` (materialized for later +evaluations); not materialized is lifetime `None`. Stage 2 enumerates materialization for summary builds (`SummaryAgg` and `SummaryTumblingWindowChain`); each -gets `NotMaterialized`, `MatAtIngestion(Memory)` when it is eligible to run -at ingestion time, and `MatAtQuery(Memory)` when two or more queries read -it. Of every other sub-DAG that two or more queries read (for example a -`Scan` that a 1 m and a 5 m window both read), only the highest node is a -materialization candidate: one that a query's output, or a node only one -query reads, consumes. It gets `NotMaterialized` and `MatAtQuery(Memory)` -only, since raw rows are not maintained at ingestion time. Below an -ingestion-time build it stays `NotMaterialized`: the ingestion side reads -its own copy. A `MatAtQuery` state is reused within one evaluation only. When the -deployment's capabilities declare `query_time_results_retained`, a -tumbling-window chain that builds exactly one new tumbling window per evaluation also gets -`MatAtQueryKeptAcrossEvaluations(Memory)`: the tumbling windows built at query time are -kept for the next evaluations. Nothing is placed on disk yet. +is offered not materialized, materialized at ingestion time `AcrossEvaluations(Memory)` when +it is eligible to run at ingestion time, and materialized at query time +`OneEvaluation(Memory)` when two or more queries read it. Of every other +sub-DAG that two or more queries read (for example a `Scan` that a 1 m and a +5 m window both read), only the highest node is a materialized-output candidate: +one that a query's output, or a node only one query reads, consumes. It is +offered not materialized and `OneEvaluation(Memory)` only, since raw rows are not +maintained at ingestion time. Below an ingestion-time build it stays not +materialized: the ingestion side reads its own copy. When the deployment's +capabilities declare `query_time_results_retained`, a tumbling-window chain +that builds exactly one new tumbling window per evaluation is also offered +query time `AcrossEvaluations(Memory)`: the tumbling windows built at query +time are materialized for the next evaluations. Nothing is placed on disk yet. A runtime builds every ingestion-time state on the grid of its width anchored at the epoch, so an ingestion-time build must end on that grid at every @@ -1243,20 +1248,20 @@ Output: C3 — CandidatePhysicalASAPDAGs 2: for each LogicalASAPDAG d in C2 do 3: // Materialization: options per sub-DAG 4: for each sub-DAG s in d do -5: Opt(s) ← { NotMaterialized } +5: Opt(s) ← { (QueryTime, None) } 6: for each medium in {memory, disk} do -7: Opt(s) ← Opt(s) ∪ { MatAtIngestion(medium), MatAtQuery(medium) } +7: Opt(s) ← Opt(s) ∪ { (IngestionTime, AcrossEvaluations(medium)), (QueryTime, OneEvaluation(medium)) } 8: end for 9: end for 10: 11: for each assignment m in Opt(s_1) × … × Opt(s_n) do 12: // constraint: everything upstream of an ingestion-time node also runs at ingestion time -13: if some s with m(s) = MatAtIngestion has an upstream node u with m(u) ≠ MatAtIngestion then +13: if some s with timing(m(s)) = IngestionTime has an upstream node u with timing(m(u)) ≠ IngestionTime then 14: PRUNE(m, "upstream of ingestion-time node not at ingestion time") 15: continue 16: end if 17: // constraint: keep a materialized output as long as any consumer needs it -18: for each s with m(s) ≠ NotMaterialized do +18: for each s with lifetime(m(s)) ≠ None do 19: retention(s) ← latest time any consumer of s reads it // from recurrence, windows 20: end for 21: // a shared summary is one node, so it is materialized once for all consumers @@ -1299,7 +1304,7 @@ into a node. *Example for Algorithm 3* ([Example 4, Pattern B](#example-4-materialization-of-window-summaries-in-physical-planning)). The logical candidate is the `Tumbling(1m)` KLL option from Algorithm 2. The -diagram shows three of its materialization assignments, each drawn as a copy +diagram shows three of its physical properties assignments, each drawn as a copy of the DAG with every node colored by its materialization. Line 13 enforces one rule: a node can run at ingestion time only if every node it reads from also runs at ingestion time, because otherwise its input does not exist yet when @@ -1328,7 +1333,7 @@ flowchart TB end subgraph LEG["Legend: when a node runs and whether its output is stored"] direction LR - l1["MatAtIngestion"]:::ingest ~~~ l2["MatAtQuery"]:::qtime ~~~ l3["NotMaterialized"]:::notmat + l1["(IngestionTime, AcrossEvaluations)"]:::ingest ~~~ l2["(QueryTime, OneEvaluation)"]:::qtime ~~~ l3["(QueryTime, None)"]:::notmat end M1 --> P1 LEG ~~~ M1 diff --git a/docs/design_docs/proposals/window-materialization-choices.md b/docs/design_docs/proposals/window-materialization-choices.md index 7646956cd..026edabfb 100644 --- a/docs/design_docs/proposals/window-materialization-choices.md +++ b/docs/design_docs/proposals/window-materialization-choices.md @@ -22,5 +22,5 @@ A tumbling window is an aligned, non-overlapping fixed-width interval and its ag This table describes possible strategies. Current tumbling window support does not imply general direct sliding-state or EH support. Offer only executable candidates. Multiple tumbling-window widths can coexist; alignment, coverage, mergeability, retention, and error contracts determine legality. A 15m window evaluated every 1m can use 1m tumbling windows; complete 5m tumbling windows alone generally cannot cover every endpoint. -Use existing `Materialization` options: `NotMaterialized`, `MatAtQuery(Memory)`, `MatAtIngestion(Memory)`, and `MatAtQueryKeptAcrossEvaluations(Memory)`. Assign them to eligible internal nodes, not necessarily the entire query. For example, maintain tumbling windows at ingestion and merge at query time. Generate only supported assignments: retained query-time state currently requires deployment capability and an eligible tumbling-window chain. Ingestion timing and upstream dependencies must satisfy Stage 2 rules. +Use existing physical properties: `Lifetime::None`, `(QueryTime, OneEvaluation(Memory))`, `(IngestionTime, AcrossEvaluations(Memory))`, and `(QueryTime, AcrossEvaluations(Memory))`. Assign them to eligible internal nodes, not necessarily the entire query. For example, maintain tumbling windows at ingestion and merge at query time. Generate only supported assignments: retained query-time state currently requires deployment capability and an eligible tumbling-window chain. Ingestion timing and upstream dependencies must satisfy Stage 2 rules. diff --git a/docs/develop_docs/asap-aware-mapping-architecture.md b/docs/develop_docs/asap-aware-mapping-architecture.md index d638c7a36..a1b24044c 100644 --- a/docs/develop_docs/asap-aware-mapping-architecture.md +++ b/docs/develop_docs/asap-aware-mapping-architecture.md @@ -17,7 +17,7 @@ in `asap-integration-tests` (`crate_graph`) checks these edges with | Crate | Role | | --- | --- | -| `asap-types` (`crates/types`) | Shared vocabulary, by IR section: `ir::operator` (`OperatorNode`, `NonASAPOp`, `ASAPOp`, operator parameters, `AggIntent`), `ir::scalar` (`ScalarExpr`, `ColumnRef`), `ir::schema` (`Schema`, `FieldDataType`, summary-state types such as `SketchAlgorithm` and `FamilyCapability`), `ir::properties` (`AccuracyTarget`, `AccuracyMetric`, `AccuracyGuarantee`, execution timing, summary coverage), plus `ir::flat` and `ir::physical_export`; the planner inputs (`workload`); Stage 2 decision data (`physical`: `Materialization`, tumbling windows) | +| `asap-types` (`crates/types`) | Shared vocabulary, by IR section: `ir::operator` (`OperatorNode`, `NonASAPOp`, `ASAPOp`, operator parameters, `AggIntent`), `ir::scalar` (`ScalarExpr`, `ColumnRef`), `ir::schema` (`Schema`, `FieldDataType`, summary-state types such as `SketchAlgorithm` and `FamilyCapability`), `ir::properties` (`AccuracyTarget`, `AccuracyMetric`, `AccuracyGuarantee`, execution timing, summary coverage), plus `ir::flat` and `ir::physical_export`; the planner inputs (`workload`); Stage 2 decision data (`physical`: `PhysicalProperties` (timing and `Lifetime`), tumbling windows) | | `asap-frontend-sql`, `asap-frontend-promql`, `asap-frontend-metricsql` (with `asap-frontend-common`) | Stage 0, the query-language frontends: lower each query into the logical DAG (`QueryRoot`s of `NonASAPOp` nodes, no summaries, no timing) | | `asap-logical-asap-optimization` (`crates/logical-asap-optimization`) | Stage 1: Pass 1 and Pass 2 (the memo rules) | | `asap-accuracy` (`crates/accuracy`) | The accuracy model the stages share: the analytical rules (per-family estimators, composition, budget split, target check) and the deployment's `AccuracyModel` and `AccuracyEvidenceProvider`; depends only on `asap-types` | @@ -42,8 +42,8 @@ asap-types ◄── asap-executor (Stage 4; no stage crate depends on it) ```text QueryRoots (Stage 0) - │ generate_dag_of_logical_memo_groups → DagOfLogicalMemoGroups, then - │ generate_per_computation_logical_memo_candidates + │ memo_of_queries → DagOfLogicalMemoGroups, then + │ pass1_memo ▼ Stage 1, Pass 1 PerComputationLogicalMemoCandidates (Memo + pruned_candidates) │ generate_logical_asap_memo_candidates @@ -71,8 +71,8 @@ same pipeline behind the `OptimizationPass` trait. | Pass 2 memo rules: shared producers (identical expressions by memo group identity, one build at different sizes), scan-label widening, window composition (tumbling windows), rollup, exact top-k reuse and finer-aggregate reuse | `asap-logical-asap-optimization`, `pass2/memo/rules/` and `pass2/memo/window_candidates.rs`, over the helpers in `pass2/asap_aware_cse.rs` | | Enumeration of the plans a memo stands for (reference pricer, DAG viewer) | `asap-logical-asap-optimization`, `pass2/memo/enumeration.rs` | | What each summary family can merge and answer (`FamilyCapability::of`: `merge_inputs`, `readouts`) | `asap-types`, `ir/schema/family_capability.rs` | -| Stage 2: materialization options per summary build (`NotMaterialized`, `MatAtIngestion`, `MatAtQuery`, `MatAtQueryKeptAcrossEvaluations`), ingestion-time eligibility, physical operator implementation | `asap-physical-asap-optimization` | -| Timing and data-state edge rules applied from a `MaterializationAssignment` | `asap-types`, `ir/properties/timing.rs` and `ir/properties/execution_data_state.rs`; `Materialization` in `physical/materialization.rs` | +| Stage 2: physical properties options per summary build (not materialized; materialized at ingestion time `AcrossEvaluations`; materialized at query time `OneEvaluation` or `AcrossEvaluations`), ingestion-time eligibility, physical operator implementation | `asap-physical-asap-optimization` | +| Timing and data-state edge rules applied from a `PhysicalPropertiesAssignment` | `asap-types`, `ir/properties/timing.rs` and `ir/properties/execution_data_state.rs`; `PhysicalProperties` and `Lifetime` in `physical/properties.rs` | | Stage 3: accuracy check, latency check, resource usage (`ResourceUsage`), cost models 1–3 (`CostModel`), measured operation costs | `asap-plan-selection`, `lib.rs`; `asap-cost`, `lib.rs` and `measured.rs` | | The analytical per-operator resource model Stage 3 measures with | `asap-cost`, `analytical.rs` and `operator_statistics.rs` | | The accuracy algebra: per-family guarantees, composition, budget split; guarantee propagation through a DAG; sizing a summary for its target, (ε, δ) budget and default sizes | `asap-accuracy` (`AccuracyModel`, `BuiltinAccuracyModel`, `estimators/`, `composition.rs`, `budget.rs`, `propagation.rs`, `sizing.rs`) | @@ -98,8 +98,8 @@ The unwired maintained-population rule has been removed as well; no Stage 1 pass For each composed DAG, the Pass 2 rules find what can be shared, split into tumbling windows or reused, and each subset of their candidates becomes another Stage 1 candidate. -3. **Stage 2** gives each summary build its materialization options and emits - one physical candidate per assignment (the all-`NotMaterialized` one +3. **Stage 2** gives each summary build its physical properties options and emits + one physical candidate per assignment (the all-not-materialized one first), skipping assignments that put a build at ingestion time over one that is not. 4. **Stage 3** checks each physical candidate: every query's root guarantee @@ -121,7 +121,7 @@ No stage caps the candidates it plans, and neither does `asap_planner`. Pass 1. - `crates/integration-tests/tests/planner_layering_example{1,2,3,4}.rs` pin the design document's examples through every stage; - `reuse_rules.rs`, `deployment_capabilities.rs`, `materialization_e2e.rs` + `reuse_rules.rs`, `deployment_capabilities.rs`, `physical_properties_e2e.rs` and `stage1_logical_asap_optimization.rs` cover single rules end to end. - `crates/devtools/tests/asap_planner.rs` checks the stage document and the committed viewer examples; `asap_planner_acceptance.rs` checks diff --git a/docs/develop_docs/asap-aware-mapping-contracts.md b/docs/develop_docs/asap-aware-mapping-contracts.md index 6086507c2..98c9219ce 100644 --- a/docs/develop_docs/asap-aware-mapping-contracts.md +++ b/docs/develop_docs/asap-aware-mapping-contracts.md @@ -15,12 +15,12 @@ asap-physical-asap-optimization -p asap-plan-selection -p asap-types | --- | --- | --- | | Stage 0 (frontends) | `NonASAPOp` nodes only; aggregates carry `AggIntent` measures with each query's `AccuracyTarget` as written | none | | Stage 1 (`compose_logical_candidate`, Pass 2 rules) | `ASAPOp` nodes where a candidate chose an exact accumulator or a summary (`SummaryAgg`, `SummaryTumblingWindowChain`, `SummaryMerge`, `SummaryEstimate`, `FinalizeExactAccumulator`); everything else stays `NonASAPOp` | none: Stage 1 never decides when anything runs | -| Stage 2 (`generate_physical_asap_memo_candidates`; a selected plan built by `Stage2AssignmentRealizer`) | the same operators, with a `Materialization` on each summary build | every node timed from the candidate's `MaterializationAssignment` | +| Stage 2 (`generate_physical_asap_memo_candidates`; a selected plan built by `Stage2AssignmentRealizer`) | the same operators, with a timing and a `Lifetime` on each summary build | every node timed from the candidate's `PhysicalPropertiesAssignment` | - Structurally equal sub-DAGs are interned before Stage 1 (`stage0/cse.rs` in `asap_logical_asap_optimization`), so Stage 1 compares nodes by structure and Pass 2 matches readers structurally, never by pointer. -- Timing is applied with `apply_materialization_timings`, which validates +- Timing is applied with `apply_physical_properties`, which validates every data-state edge. A violation is an `ExecutionDataStateError`: for example `QueryTimeValueUnderIngestionSummary` (a query-time value beneath an ingestion-time build), `ConflictingTiming` (one shared node @@ -78,13 +78,14 @@ asap-physical-asap-optimization -p asap-plan-selection -p asap-types ## 4. Stage 2 - Only summary builds (`SummaryAgg`, `SummaryTumblingWindowChain`) get materialization - options: `NotMaterialized` always; `MatAtIngestion(Memory)` when the build - is eligible to run as data arrives; `MatAtQuery(Memory)` when two or more - queries read it (reused within one evaluation); - `MatAtQueryKeptAcrossEvaluations(Memory)` for a tumbling-window chain that + options, each a timing and a lifetime (`PhysicalProperties`): not materialized always; + materialized at ingestion time, `AcrossEvaluations(Memory)`, when the build is + eligible to run as data arrives; materialized at query time for `OneEvaluation(Memory)` + when two or more queries read it (reused within one evaluation); materialized at + query time `AcrossEvaluations(Memory)` for a tumbling-window chain that builds one new tumbling window per evaluation, when the deployment declares `query_time_results_retained`. -- One `PhysicalCandidate` per assignment, the all-`NotMaterialized` one first +- One `PhysicalCandidate` per assignment, the all-not-materialized one first (`P`, then `P-m`). An assignment that puts a build at ingestion time over one that is not is not generated. A logical candidate Stage 2 cannot implement becomes a `PrunedCandidate` with its reason. @@ -98,7 +99,7 @@ asap-physical-asap-optimization -p asap-plan-selection -p asap-types candidate no buildable input, query output or reader count can use is removed (`InputHasNoBuildableProducer`, `ReachesNoQueryOutput`, `TooFewQueryReaders`). Stage 2 emits one candidate per operator node, - timing and materialization; it makes no execution-scope copies. + timing and lifetime; it makes no execution-scope copies. ## 5. Stage 3 diff --git a/docs/develop_docs/end-to-end-accuracy-guarantees.md b/docs/develop_docs/end-to-end-accuracy-guarantees.md index 4374708cf..1e4a98a05 100644 --- a/docs/develop_docs/end-to-end-accuracy-guarantees.md +++ b/docs/develop_docs/end-to-end-accuracy-guarantees.md @@ -174,9 +174,7 @@ against the DataSketches maximum-rank-error characterization `2.446 / k^0.9433` at a per-trial failure probability of 0.01, not against this single-sided fit or its δ scaling. -A KLL shared by readers with different failure probabilities is sized at the -smallest one. Each reader's `k` is first resized to it at no larger rank -error (`kll::shared_k`), and the largest is kept. See +See [Apache DataSketches KLL accuracy](https://datasketches.apache.org/docs/KLL/KLLAccuracyAndSize.html) and its [C++ contract pinned at `a9b42755072b`](https://github.com/apache/datasketches-cpp/blob/a9b42755072b079fd90b29b9851adc121015c58e/kll/include/kll_sketch.hpp). diff --git a/docs/develop_docs/extend-asap-aware-mapping.md b/docs/develop_docs/extend-asap-aware-mapping.md index 9c03a415e..f59566dcd 100644 --- a/docs/develop_docs/extend-asap-aware-mapping.md +++ b/docs/develop_docs/extend-asap-aware-mapping.md @@ -74,20 +74,21 @@ and when it must not; an end-to-end test like `crates/integration-tests/tests/reuse_rules.rs` that the planned workload contains the candidate and that Stage 3 prices it. -## 3. Add a Stage 2 materialization option +## 3. Add a Stage 2 physical properties option -1. Add the `Materialization` variant - (`crates/types/src/physical/materialization.rs`) and its timing in +1. Add the `Lifetime` variant or the valid (timing, lifetime) pair + (`crates/types/src/physical/properties.rs`, `PhysicalProperties::new` and + `readable_at`) and its placement in `crates/types/src/ir/properties/timing.rs`, with its data-state edge rules. 2. Offer it in `crates/physical-asap-optimization/src/lib.rs` (the - `MaterializationCandidate` options), after the existing options so default + `MaterializedOutputCandidate` options), after the existing options so default candidate ids stay the same, and gate it on the deployment input that allows it. 3. Price it in `crates/cost/src/usage.rs` (`asap_cost::node_usage`): what it keeps in memory or on disk, and what each evaluation rebuilds. Tests: Stage 2 unit tests for when the option is offered; -`crates/integration-tests/tests/materialization_e2e.rs` and the Example 4 +`crates/integration-tests/tests/physical_properties_e2e.rs` and the Example 4 tests in `planner_layering_example4.rs`. ## 4. Add a Stage 3 check or a deployment input diff --git a/docs/develop_docs/library-api.md b/docs/develop_docs/library-api.md index 03f3a695c..122eae3b8 100644 --- a/docs/develop_docs/library-api.md +++ b/docs/develop_docs/library-api.md @@ -197,7 +197,7 @@ asap_plan_selection::select_plan( | --- | --- | | `PlanningModels::new(accuracy, evidence)` | Your `&dyn AccuracyModel` and `&dyn AccuracyEvidenceProvider`, with default cost inputs and deployment capabilities; `with_accuracy` and `with_evidence` replace one of them | | `PlanningModels::builtin()` | Built-in accuracy model (`BuiltinAccuracyModel`) and no extra evidence (`NoAccuracyEvidence`). `with_cost_inputs(&settings)` sets the deployment's `CostInputs`; `with_deployment_capabilities(&capabilities)` sets its `DeploymentCapabilities` (field `deployment_capabilities`) | -| `DeploymentCapabilities` (`asap_logical_asap_optimization`, re-exported as `asap_planner::DeploymentCapabilities`) | What the executor supports: `supported_summaries` (`None`, the default, supports every summary the planner builds; otherwise `SupportedSummary { family: SketchAlgorithm, readouts: Vec }` entries; Pass 1 prunes others with `PruneReason::UnsupportedByDeployment`), `raw_data_retained` (default `true`; when `false`, a candidate that scans raw data at query time is pruned by Stage 2, and stored data is refused with `asap_planner::SelectionError::InvalidDeploymentCapabilities`) and `query_time_results_retained` (default `false`; when `true`, Stage 2 offers `MatAtQueryKeptAcrossEvaluations` for tumbling windows) | +| `DeploymentCapabilities` (`asap_logical_asap_optimization`, re-exported as `asap_planner::DeploymentCapabilities`) | What the executor supports: `supported_summaries` (`None`, the default, supports every summary the planner builds; otherwise `SupportedSummary { family: SketchAlgorithm, readouts: Vec }` entries; Pass 1 prunes others with `PruneReason::UnsupportedByDeployment`), `raw_data_retained` (default `true`; when `false`, a candidate that scans raw data at query time is pruned by Stage 2, and stored data is refused with `asap_planner::SelectionError::InvalidDeploymentCapabilities`) and `query_time_results_retained` (default `false`; when `true`, Stage 2 offers query-time `AcrossEvaluations` for tumbling-window chains) | | `CostInputs` (`asap_cost`) | `cost_model`, `resource_speed` (`ms_per_cpu_op`, `ms_per_scanned_byte`), `non_repeating_query_horizon_seconds` (default 3,600) and `measured_operation_costs` (`MeasuredOperationCosts`, default empty: `operators`, one `OperatorCost { operator, phase, sketch / exact / measure, ns_per_unit, ns_per_invocation, memory_bytes_per_state }` per measured operator kind, phase and family; `ns_per_evaluation` per query request; `provenance` (executor version, machine). A node with an entry is priced from it, a sketch between the two nearest measured sizes; every other node from the formulas times one calibration factor fitted from the measured nodes; each node's `NodeCost::price_source` says which. The earlier JSON array of `MeasuredOperationCost { sketch, operation, cpu_ns_per_operation }` still loads) | | `CostModel::UsageOverTime(ResourceUnitCost)` | Cost model 1 (default, `ResourceUnitCost::illustrative()`): unit cost × usage per second; the cheapest valid candidate wins | | `CostModel::ProvisionedCapacity(ResourceUnitCost)` | Cost model 2: unit cost × peak memory, peak CPU cores and disk; the cheapest valid candidate wins | @@ -206,7 +206,7 @@ asap_plan_selection::select_plan( | `PlanOutput.selection` | `Some` with Stage 3's `Selection` under the default pass: `selected_dag`, `nodes_with_proven_non_negative_weights`, `cost` (the selected plan's `PlanCost` with its `ResourceUsage` and `cost_per_second`, `None` under cost model 3), `accepted_accuracy_per_query`, `pruned_candidates` and `search_report` | | `PlanOutput.tumbling_window_retention_ms` | For each tumbling-window chain the selected plan keeps beyond one evaluation, by node id, how long its tumbling windows are kept (`RealizedPlan::tumbling_window_retention_ms`) | | `PlanningRun` | `per_computation_candidates`, `logical_asap_candidates`, `physical_asap_candidates`, `selection`, `selected_physical_plan` (`RealizedPlan`), `selected_plan` (the roots to run) and `report` (Pass 2's and Stage 2's counts and each stage's time); `per_computation_candidates` is kept only with `PlanSearchSettings::inspect`; `asap_planner::selected_chosen_candidates` reads the selected plan as one candidate per group of `logical_asap_candidates.memo` | -| `PlanOutput::physical_asap_dag()` | The planned queries as one `PhysicalASAPDAG`, timed with the default `MaterializationAssignment` (see [Export](#export)) | +| `PlanOutput::physical_asap_dag()` | The planned queries as one `PhysicalASAPDAG`, timed with the default `PhysicalPropertiesAssignment` (see [Export](#export)) | Each candidate's `ResourceUsage` holds its usage without prices: `cpu_ops_per_second`, `scanned_bytes_per_second`, `average_memory_bytes`, @@ -360,8 +360,8 @@ optional also does not guarantee every planning operation can succeed without it | Function/type | Purpose | | --- | --- | -| `asap_types::ir::apply_materialization_timings(&root, &assignment, &mut NodeToTimedNodeCache::new())` | Write execution timing into every node from a `MaterializationAssignment` (default: all query time) and validate the data-state edges; `PlanOutput::physical_asap_dag()` applies the default to a planned workload | -| `asap_types::ir::physical_export::compile_physical_asap_dag(&timed_root)` | Export a timed DAG as a `PhysicalASAPDAG`; rejects an untimed node; `compile_physical_asap_workload(&roots)` exports a batch with one root per query; `.validate()` checks it explicitly | +| `asap_types::ir::apply_physical_properties(&root, &assignment)` | Complete a `PhysicalPropertiesAssignment` (default: all query time) into the physical properties of every node and validate the data-state edges; `apply_physical_properties_to_roots` does it for a workload; `PlanOutput::physical_asap_dag()` applies the pass's `physical_properties` to a planned workload | +| `asap_types::ir::physical_export::compile_physical_asap_dag(&root, &timed)` | Export a DAG as a `PhysicalASAPDAG`, each node with its properties from the completed assignment `timed`; rejects a node it does not list; `compile_physical_asap_workload(&roots, &timed)` exports a batch with one root per query; `.validate()` checks it explicitly | | `PhysicalASAPDAG::materialized_views()` and `.raw_data_retention()` (`asap_types::ir::materialized_view`) | What a runtime keeps for an installed DAG, derived from the DAG and the `retention_ms` installation sets: each materialized view (refresh mode, coverage, inputs, the node's `retention_ms`) and how far back each raw-data source is kept for query-time scans (each query-time `Scan`'s `retention_ms`) | Choose the export matching your intended handoff: an untimed semantic DAG is not diff --git a/docs/develop_docs/physical-compile-coverage.md b/docs/develop_docs/physical-compile-coverage.md index 9d68caec1..a0b221ddf 100644 --- a/docs/develop_docs/physical-compile-coverage.md +++ b/docs/develop_docs/physical-compile-coverage.md @@ -5,7 +5,7 @@ Audience: developers moving computation from ASAPQuery-backend into ## Contract -Logical selection decides what to compute. A `MaterializationAssignment` sets +Logical selection decides what to compute. A `PhysicalPropertiesAssignment` sets node timing (Stage 2 materialization chooses it per physical candidate). `physical_planner::compile` turns a timed `PhysicalASAPDAG` into physical operator DAGs. The backend owns ingestion, tumbling windows, storage, stored-state evaluation, external exact engines, pricing/selection, and execution scheduling. diff --git a/docs/develop_docs/pre-asap-ir.md b/docs/develop_docs/pre-asap-ir.md index 5fcd74c4c..9a2d23dc5 100644 --- a/docs/develop_docs/pre-asap-ir.md +++ b/docs/develop_docs/pre-asap-ir.md @@ -33,8 +33,6 @@ pub struct OperatorNode { pub result_kind: OperatorResultKind, // Relation | InstantVector | RangeVector | State | Scalar pub schema: Schema, // output schema, derived at construction pub guarantee: Option, // None until accuracy assessment establishes one - pub timing: Option, // None until a materialization assignment is applied - pub materialization: Materialization, // NotMaterialized until Stage 2 assigns one } ``` @@ -47,12 +45,11 @@ pub struct OperatorNode { derived (a column reference out of range, a reserved ASAP operator). ASAP planning may retain a more specific schema through `OperatorNode::with_schema`. - `guarantee` is `None` until accuracy assessment establishes one; `None` never means exact. -- `timing` is `None` in every front-end DAG and every candidate. It is written by - `ir::timing::apply_materialization_timings` (see the Post-ASAP IR document); export rejects an - untimed node. -- `materialization` is `NotMaterialized` in every front-end DAG. Stage 2 writes it, with - `timing`, through `ir::timing::apply_materialization_timings` from its materialization - assignment. +- A node has no physical properties (timing, lifetime): a logical candidate has only its + logical properties (#1200). Physical stages keep each node's `PhysicalProperties` in a + `PhysicalPropertiesAssignment` keyed by node; `ir::timing::apply_physical_properties` + completes Stage 2's assignment over every node (see the Post-ASAP IR document), and + export reads it. `OperatorNode::children()` returns the operator's inputs in field order followed by the operator nodes its scalar expressions read (see "Scalar expressions"). Every DAG traversal — diff --git a/docs/develop_docs/target-candidate-api-migration.md b/docs/develop_docs/target-candidate-api-migration.md index 77cc5379d..7d89d4389 100644 --- a/docs/develop_docs/target-candidate-api-migration.md +++ b/docs/develop_docs/target-candidate-api-migration.md @@ -47,7 +47,7 @@ or `Operator::ASAP(ASAPOp)`. Old public names are not kept as aliases. | `SummaryExpr::ValueOperation { .. }` over an evaluation | An ordinary `NonASAPOp` (`Project`, `Filter`, `Sort`, `Limit`, `Aggregate`) reading an ASAP node; `FinalizeExactAccumulator`, `MaintainPopulation`, `EvaluatePopulation` are `ASAPOp` variants | | `Replacement::Summary(..)` / `Replacement::Rewrite(..)` | `Replacement::SubDAG(Rc)`; `is_logical_rewrite` tells them apart | | `SummaryFamilyType` | `FieldDataType` (its non-`Plain` variants) | -| Timing stored on post-ASAP nodes | `OperatorNode::timing`, `None` until `ir::timing::apply_materialization_timings` writes it from a `MaterializationAssignment` (default: all query time) | +| Timing stored on post-ASAP nodes | `OperatorNode::timing`, `None` until `ir::timing::apply_physical_properties` writes it from a `PhysicalPropertiesAssignment` (default: all query time) | | `UnresolvedQueryExpr` + `asap_types::pre_asap::resolve_root` | `UnresolvedOp` / `UnresolvedScalar` + `asap_frontend_common::resolve_root` | | `pre_asap::canonicalize`, `pre_asap::cse::share_common_sub_dags` | `asap_frontend_common` `canonicalize` (crate-private), `share_common_sub_dags` (crate-private in `asap_logical_asap_optimization`, `stage0/cse.rs`) | | `asap_types::post_asap::compile_post_asap_dag` (wire version 5, `Fallback`/`Binary`/`Value` payloads) | `asap_types::ir::physical_export::compile_physical_asap_dag` (`PhysicalASAPDAG`, carried in the plan file versioned by `asap_planner::INSTALLABLE_PLAN_WIRE_VERSION`: one node per operator, `Operator` payloads, `ScalarRef` edges); input must be timed | diff --git a/docs/develop_docs/tumbling-window-search-compression-evaluation.md b/docs/develop_docs/tumbling-window-search-compression-evaluation.md index 2419cf97e..5ddcdfc26 100644 --- a/docs/develop_docs/tumbling-window-search-compression-evaluation.md +++ b/docs/develop_docs/tumbling-window-search-compression-evaluation.md @@ -110,7 +110,7 @@ The nine viewer workloads also passed **9/9**, with **40/40** query requirements ### Final regression follow-up -The regression suite found a genuine optimization regression outside window planning. Two identical SQL percentiles over an at-rest table were sent through progressive tumbling window rounds solely because they repeat every ten minutes. No computation had a tumbling-window-eligible window, so the rounds rebuilt the same 40-configuration, 150-connection model and restarted refinement. The 128-evaluation budget expired with two identical unmaterialized KLL builds, costing 0.007678692288679677 per second instead of the baseline's one shared query-materialized build at 0.006000017830006526 (+27.98%). The original pointer-sharing assertion failed both in the suite and in isolation; it was retained unchanged. +The regression suite found a genuine optimization regression outside window planning. Two identical SQL percentiles over an at-rest table were sent through progressive tumbling window rounds solely because they repeat every ten minutes. No computation had a tumbling-window-eligible window, so the rounds rebuilt the same 40-configuration, 150-connection model and restarted refinement. The 128-evaluation budget expired with two identical not_materialized KLL builds, costing 0.007678692288679677 per second instead of the baseline's one shared query-materialized build at 0.006000017830006526 (+27.98%). The original pointer-sharing assertion failed both in the suite and in isolation; it was retained unchanged. Revision `fe171f64` bypasses tumbling-window-domain expansion when the initial computations contain neither a positive range window nor an existing tumbling-window chain. It searches the complete candidate domain directly; at-rest status alone does not disable tumbling window planning. The same SQL fixture now passes the original sharing test and selects one shared KLL with exactly the baseline cost. Its report records 58 evaluations, `optimal`, one `Complete` round, and no pending widths. This fix avoids repeated identical-domain searches rather than increasing the budget or relaxing correctness checks. diff --git a/tools/dag-viewer/README.md b/tools/dag-viewer/README.md index c6800ee1c..f713f4a0f 100644 --- a/tools/dag-viewer/README.md +++ b/tools/dag-viewer/README.md @@ -9,7 +9,7 @@ document shows: - **Stage 1**, the logical candidates of the memo and the selected plan's logical DAG (`L1`); - **Stage 2**, the selected plan as an executable physical DAG (`P1`), with - each node's timing and materialization, and the physical candidates Stage 2 + each node's timing and lifetime, and the physical candidates Stage 2 pruned; - **Stage 3**, the selected plan's cost, the accuracy it accepted per query, its materialized views and the search report. @@ -78,7 +78,7 @@ To run it, see [RUNNING.md](RUNNING.md). candidates are filled green with `✓ selected`. Clicking a candidate or a group shows its nodes, children, parents and queries in the details. - **Details**: click a node for its operator, the queries reaching it, - timing, materialization, output schema, guarantee and Stage 3 cost; click + timing, lifetime, output schema, guarantee and Stage 3 cost; click an edge for its schema and data state. Other documents open with **Open stage document or workload…**, by dropping diff --git a/tools/dag-viewer/app.js b/tools/dag-viewer/app.js index 3c1b4e877..34aed44c5 100644 --- a/tools/dag-viewer/app.js +++ b/tools/dag-viewer/app.js @@ -124,7 +124,7 @@ if (d.tumblingWindowOf !== undefined) items.push(['tumbling window', `${d.tumblingWindowIndex + 1} of ${body.tumbling_window_count} of SummaryTumblingWindowChain ${d.tumblingWindowOf}, window ${tumblingWindowTimeText(d.tumbling_window, body.tumbling_window_width_ms)}`]); if (d.expandable) items.push(['tumbling windows', `${body.tumbling_window_count} × ${durationText(body.tumbling_window_width_ms)} (click the chain to show them, a tumbling window to hide them)`]); if (d.timing !== undefined) items.push(['runs at', formatTiming(d.timing)]); - if (n.materialization) items.push(['materialization', compactWire(n.materialization)]); + if (n.lifetime) items.push(['lifetime', compactWire(n.lifetime)]); if (n.output_state && n.output_state.primitive) items.push(['output primitive', compactWire(n.output_state.primitive)]); if (d.cost) { const value = typeof d.cost.cost_per_second === 'number' @@ -352,9 +352,9 @@ $('views-meta').innerHTML = selection.materialized_views_error ? `no installable plan: ${esc(selection.materialized_views_error)}` : `${count} materialized view${count === 1 ? '' : 's'} in ${groups.length} group${groups.length === 1 ? '' : 's'} of ${esc(selection.selected_candidate)}` + - (count ? '' : ': nothing is kept across evaluations; every query computes at query time'); + (count ? '' : ': nothing is materialized across evaluations; every query computes at query time'); const tumblingWindowsCell = (g) => (g.tumblingWindows.length > 1 - ? `
${esc(g.tumblingWindows.length)} installed nodes
    ${g.tumblingWindows.map((p) => `
  • node ${esc(p.nodeId)} · ${esc(p.tumblingWindow)} · window ${esc(p.window)} · kept ${esc(p.retention)}
    • `).join('')}
` + ? `
${esc(g.tumblingWindows.length)} installed nodes
    ${g.tumblingWindows.map((p) => `
  • node ${esc(p.nodeId)} · ${esc(p.tumblingWindow)} · window ${esc(p.window)} · materialized ${esc(p.retention)}
    • `).join('')}
` : `node ${esc(g.tumblingWindows[0].nodeId)} · window ${esc(g.tumblingWindows[0].window)}`); $('materializedViews').innerHTML = groups.length ? table( ['Stage 2 node', 'definition', 'kind', 'tumbling windows', 'refresh', 'retention', 'installed', 'Stage 3 cost', 'read by'], diff --git a/tools/dag-viewer/memo.js b/tools/dag-viewer/memo.js index 5216dbab3..d13aa801b 100644 --- a/tools/dag-viewer/memo.js +++ b/tools/dag-viewer/memo.js @@ -50,11 +50,11 @@ const memoIsNode = (nodes, id) => Number.isInteger(id) && id >= 0 && id < nodes. // One line per entry of a stage's `pruned_candidates` on group `group`: // Pass 1's `candidate`, or Stage 2's operator with its timing, -// materialization and scope copy; then the reason. 'none' without any. +// lifetime and scope copy; then the reason. 'none' without any. function memoPrunedText(entries, group) { const lines = (Array.isArray(entries) ? entries : []).filter((p) => p && p.group === group).map((p) => { const what = 'candidate' in p ? p.candidate - : `node ${p.operator_node} ${p.operator} ${p.timing} ${JSON.stringify(p.materialization)}`; + : `node ${p.operator_node} ${p.operator} ${p.timing} ${JSON.stringify(p.lifetime)}`; return `${what}: ${p.reason}`; }); return lines.length ? lines.join('; ') : 'none'; diff --git a/tools/dag-viewer/stages.js b/tools/dag-viewer/stages.js index aff3da8fa..d30b2c307 100644 --- a/tools/dag-viewer/stages.js +++ b/tools/dag-viewer/stages.js @@ -224,7 +224,7 @@ function stageDag(dag, physical) { const [kind, body] = stageOperatorParts(node.payload); return { id: node.id, kind, body, schema: node.output_schema, guarantee: node.guarantee, - materialization: node.materialization, timing: node.output_state ? node.output_state.timing : undefined, raw: node, + lifetime: node.lifetime, timing: node.output_state ? node.output_state.timing : undefined, raw: node, }; }); const edges = dag.edges.map((edge) => ({ producer: edge.producer, consumer: edge.consumer, raw: edge })); @@ -241,8 +241,7 @@ function stageDag(dag, physical) { const [kind, body] = stageOperatorParts(node.operator); stageLogicalChildren(body).forEach((producer) => edges.push({ producer, consumer: id, raw: null })); return { - id, kind, body, schema: node.schema, guarantee: node.guarantee, - materialization: node.materialization, timing: node.timing === null ? undefined : node.timing, raw: node, + id, kind, body, schema: node.schema, guarantee: node.guarantee, raw: node, }; }); const queries = dag.roots.map((root) => ('Scalar' in root ? { scalar: root.Scalar } : { node: root.Operator })); @@ -1079,7 +1078,7 @@ function materializedViewRows(doc) { definition: output.coverage.definition_id, kind: summary ? `${kind} · ${summary}` : kind, tumblingWindow, - refresh: output.refresh === 'at_ingestion' ? 'at ingestion' : 'on first read (kept across evaluations)', + refresh: output.refresh === 'at_ingestion' ? 'at ingestion' : 'on first read (materialized across evaluations)', inputs: `${output.inputs.raw_data_windows.length} raw-data window(s)` + (output.inputs.materialized_views.length ? `, views ${output.inputs.materialized_views.join(', ')}` : ''), retention: durationText(output.retention_ms), diff --git a/tools/dag-viewer/test-fixtures/memo-enumeration-small.json b/tools/dag-viewer/test-fixtures/memo-enumeration-small.json index 633a16fa6..a5bcc3476 100644 --- a/tools/dag-viewer/test-fixtures/memo-enumeration-small.json +++ b/tools/dag-viewer/test-fixtures/memo-enumeration-small.json @@ -1 +1 @@ -{"memo":{"nodes":[{"guarantee":null,"materialization":"NotMaterialized","operator":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"source":{"TimeSeries":{"metric":"http_requests_total"}}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}},"result_kind":"range_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"NonASAP":{"Aggregate":{"child":1,"filters":[],"having":null,"measures":[{"kind":"rate"}],"output_names":[""],"reduction":"PerEntity"}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"NonASAP":{"Aggregate":{"child":2,"filters":[],"having":null,"measures":[{"col":null,"kind":"sum"}],"output_names":[""],"reduction":{"Reduce":[2]}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"NonASAP":{"Aggregate":{"child":1,"filters":[],"having":null,"measures":[{"col":null,"kind":"sum"}],"output_names":[""],"reduction":"PerEntity"}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"NonASAP":{"Aggregate":{"child":4,"filters":[],"having":null,"measures":[{"accuracy":{"accuracy_scope":"whole_result","failure_probability":0.001,"max_error":0.01},"k":10,"kind":"top_k"}],"output_names":[],"reduction":{"Reduce":[2]}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"topk_10","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"timing":null},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"materialization":"NotMaterialized","operator":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"source":{"TimeSeries":{"metric":"http_requests_total"}}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"timing":null},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"materialization":"NotMaterialized","operator":{"NonASAP":{"TimeRange":{"child":6,"kind":"range","range":{"nanos":0,"secs":60}}}},"result_kind":"range_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"ASAP":{"SummaryAgg":{"child":7,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"timing":null},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"materialization":"NotMaterialized","operator":{"ASAP":{"FinalizeExactAccumulator":{"child":8}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"ASAP":{"SummaryAgg":{"child":9,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[2]}}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"timing":null},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"materialization":"NotMaterialized","operator":{"ASAP":{"FinalizeExactAccumulator":{"child":10}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"ASAP":{"SummaryAgg":{"child":7,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"timing":null},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"materialization":"NotMaterialized","operator":{"ASAP":{"FinalizeExactAccumulator":{"child":12}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"timing":null},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"materialization":"NotMaterialized","operator":{"NonASAP":{"Aggregate":{"child":13,"filters":[],"having":null,"measures":[{"accuracy":{"accuracy_scope":"whole_result","failure_probability":0.001,"max_error":0.01},"k":10,"kind":"top_k"}],"output_names":[],"reduction":{"Reduce":[2]}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"topk_10","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"ASAP":{"SummaryAgg":{"child":7,"family":{"Sketch":[{"algorithm":"CmsWithHeap","category":"TopK","params":{"CmsWithHeap":{"depth":7,"heap_size":100,"width":272}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":{"Column":{"Named":"$promql_series_identity"}},"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"samples_of_metric","metric":"http_requests_total"}},"reduction":{"Reduce":[2]}}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Sketch":[{"algorithm":"CmsWithHeap","category":"TopK","params":{"CmsWithHeap":{"depth":7,"heap_size":100,"width":272}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"ASAP":{"SummaryEstimate":{"query":{"TopK":{"k":10}},"summary_input":15}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"topk","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"NonASAP":{"Project":{"child":16,"cols":[{"alias":"job","expr":{"Column":0}},{"alias":"topk_10","expr":{"Column":2}},{"alias":"$promql_series_identity","expr":{"Column":1}}],"qualifier":null}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"topk_10","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"ASAP":{"SummaryAgg":{"child":7,"family":{"Sketch":[{"algorithm":"CountSketchWithHeap","category":"TopK","params":{"CountSketchWithHeap":{"depth":81,"heap_size":100,"width":30000}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":{"Column":{"Named":"$promql_series_identity"}},"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"samples_of_metric","metric":"http_requests_total"}},"reduction":{"Reduce":[2]}}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Sketch":[{"algorithm":"CountSketchWithHeap","category":"TopK","params":{"CountSketchWithHeap":{"depth":81,"heap_size":100,"width":30000}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"ASAP":{"SummaryEstimate":{"query":{"TopK":{"k":10}},"summary_input":18}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"topk","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"NonASAP":{"Project":{"child":19,"cols":[{"alias":"job","expr":{"Column":0}},{"alias":"topk_10","expr":{"Column":2}},{"alias":"$promql_series_identity","expr":{"Column":1}}],"qualifier":null}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"topk_10","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"timing":null}],"steps":[{"step":"stage0","title":"Stage 0","groups":[{"id":0,"label":"Scan http_requests_total","children":[],"parents":[1],"queries":[0],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":0},{"id":1,"label":"TimeRange 1m","children":[0],"parents":[2],"queries":[0],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":1},{"id":2,"label":"rate per series","children":[1],"parents":[3],"queries":[0],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":2},{"id":3,"label":"sum by (job)","children":[2],"parents":[],"queries":[0],"roots_of":[0],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":3},{"id":4,"label":"Scan http_requests_total","children":[],"parents":[5],"queries":[1],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":0},{"id":5,"label":"TimeRange 1m","children":[4],"parents":[6],"queries":[1],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":1},{"id":6,"label":"sum_over_time per series","children":[5],"parents":[7],"queries":[1],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":4},{"id":7,"label":"topk by (job) 10","children":[6],"parents":[],"queries":[1],"roots_of":[1],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":5}]},{"step":"pass1","title":"Stage 1, Pass 1","groups":[{"id":0,"label":"Scan http_requests_total","children":[],"parents":[1],"queries":[0],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[0],"children":[],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"Scan http_requests_total","group_of_node":{},"node":0}],"node":0},{"id":1,"label":"TimeRange 1m","children":[0],"parents":[2],"queries":[0],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[1],"children":[0],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"TimeRange 1m","group_of_node":{"0":0},"node":1}],"node":1},{"id":2,"label":"rate per series","children":[1],"parents":[3],"queries":[0],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[2],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"rate per series","group_of_node":{"1":1},"node":2}],"node":2},{"id":3,"label":"sum by (job)","children":[2],"parents":[],"queries":[0],"roots_of":[0],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[3],"children":[2],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"sum by (job)","group_of_node":{"2":2},"node":3}],"node":3},{"id":4,"label":"Scan http_requests_total","children":[],"parents":[5],"queries":[1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[4],"children":[],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"Scan http_requests_total","group_of_node":{},"node":0}],"node":0},{"id":5,"label":"TimeRange 1m","children":[4],"parents":[6],"queries":[1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[5],"children":[4],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"TimeRange 1m","group_of_node":{"0":4},"node":1}],"node":1},{"id":6,"label":"sum_over_time per series","children":[5],"parents":[7],"queries":[1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[6],"children":[5],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"sum_over_time per series","group_of_node":{"1":5},"node":4}],"node":4},{"id":7,"label":"topk by (job) 10","children":[6],"parents":[],"queries":[1],"roots_of":[1],"candidate_count":4,"candidate_count_by_origin":{"pass1":3,"shared_build":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[7],"children":[6],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"exact accumulator","group_of_node":{"4":6},"node":5},{"index":1,"implements":[7],"children":[6],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"CMS+heap","group_of_node":{"4":6},"node":5},{"index":2,"implements":[7],"children":[6],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"CountSketch+heap","group_of_node":{"4":6},"node":5},{"index":3,"implements":[7],"children":[6],"origin":"shared_build","accuracy_budget_parts":0.5,"added":true,"label":"CMS+heap shared build","group_of_node":{"4":6},"node":5}],"node":5}]}],"selected_choice":{"0":0,"1":0,"2":0,"3":0,"4":0,"5":0,"6":0,"7":1},"enumeration":{"step":"pass1","per_step":{"stage0":{"count":1,"complete":true},"pass1":{"count":4,"complete":true}},"limit":1000,"choices":[{"id":"E1","choice":{"0":0,"1":0,"2":0,"3":0,"4":0,"5":0,"6":0,"7":0},"accuracy_budget_parts_1":true,"selected":false,"valid":true,"rejection":null,"cost_per_second":3.0,"cheaper_than_selected":false,"roots":[11,14]},{"id":"E2","choice":{"0":0,"1":0,"2":0,"3":0,"4":0,"5":0,"6":0,"7":1},"accuracy_budget_parts_1":true,"selected":true,"valid":true,"rejection":null,"cost_per_second":2.0,"cheaper_than_selected":false,"roots":[11,17]},{"id":"E3","choice":{"0":0,"1":0,"2":0,"3":0,"4":0,"5":0,"6":0,"7":2},"accuracy_budget_parts_1":true,"selected":false,"valid":true,"rejection":null,"cost_per_second":1.5,"cheaper_than_selected":true,"roots":[11,20]},{"id":"E4","choice":{"0":0,"1":0,"2":0,"3":0,"4":0,"5":0,"6":0,"7":3},"accuracy_budget_parts_1":false,"selected":false,"valid":false,"rejection":"accuracy_target_missed","cost_per_second":null,"cheaper_than_selected":false,"roots":[11,17]}],"priced":{"count":4,"cheaper_than_selected":1,"selected_cost_per_second":2.0}}}} +{"memo":{"nodes":[{"guarantee":null,"operator":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"source":{"TimeSeries":{"metric":"http_requests_total"}}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}},"result_kind":"range_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"NonASAP":{"Aggregate":{"child":1,"filters":[],"having":null,"measures":[{"kind":"rate"}],"output_names":[""],"reduction":"PerEntity"}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"NonASAP":{"Aggregate":{"child":2,"filters":[],"having":null,"measures":[{"col":null,"kind":"sum"}],"output_names":[""],"reduction":{"Reduce":[2]}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":null,"operator":{"NonASAP":{"Aggregate":{"child":1,"filters":[],"having":null,"measures":[{"col":null,"kind":"sum"}],"output_names":[""],"reduction":"PerEntity"}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"NonASAP":{"Aggregate":{"child":4,"filters":[],"having":null,"measures":[{"accuracy":{"accuracy_scope":"whole_result","failure_probability":0.001,"max_error":0.01},"k":10,"kind":"top_k"}],"output_names":[],"reduction":{"Reduce":[2]}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"topk_10","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"operator":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"source":{"TimeSeries":{"metric":"http_requests_total"}}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"operator":{"NonASAP":{"TimeRange":{"child":6,"kind":"range","range":{"nanos":0,"secs":60}}}},"result_kind":"range_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryAgg":{"child":7,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"operator":{"ASAP":{"FinalizeExactAccumulator":{"child":8}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryAgg":{"child":9,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[2]}}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"operator":{"ASAP":{"FinalizeExactAccumulator":{"child":10}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryAgg":{"child":7,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"operator":{"ASAP":{"FinalizeExactAccumulator":{"child":12}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"operator":{"NonASAP":{"Aggregate":{"child":13,"filters":[],"having":null,"measures":[{"accuracy":{"accuracy_scope":"whole_result","failure_probability":0.001,"max_error":0.01},"k":10,"kind":"top_k"}],"output_names":[],"reduction":{"Reduce":[2]}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"topk_10","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryAgg":{"child":7,"family":{"Sketch":[{"algorithm":"CmsWithHeap","category":"TopK","params":{"CmsWithHeap":{"depth":7,"heap_size":100,"width":272}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":{"Column":{"Named":"$promql_series_identity"}},"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"samples_of_metric","metric":"http_requests_total"}},"reduction":{"Reduce":[2]}}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Sketch":[{"algorithm":"CmsWithHeap","category":"TopK","params":{"CmsWithHeap":{"depth":7,"heap_size":100,"width":272}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryEstimate":{"query":{"TopK":{"k":10}},"summary_input":15}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"topk","nullable":false,"table":null}],"time_index":null,"unique_keys":[]}},{"guarantee":null,"operator":{"NonASAP":{"Project":{"child":16,"cols":[{"alias":"job","expr":{"Column":0}},{"alias":"topk_10","expr":{"Column":2}},{"alias":"$promql_series_identity","expr":{"Column":1}}],"qualifier":null}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"topk_10","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":null,"unique_keys":[]}},{"guarantee":null,"operator":{"ASAP":{"SummaryAgg":{"child":7,"family":{"Sketch":[{"algorithm":"CountSketchWithHeap","category":"TopK","params":{"CountSketchWithHeap":{"depth":81,"heap_size":100,"width":30000}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":{"Column":{"Named":"$promql_series_identity"}},"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"samples_of_metric","metric":"http_requests_total"}},"reduction":{"Reduce":[2]}}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Sketch":[{"algorithm":"CountSketchWithHeap","category":"TopK","params":{"CountSketchWithHeap":{"depth":81,"heap_size":100,"width":30000}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryEstimate":{"query":{"TopK":{"k":10}},"summary_input":18}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"topk","nullable":false,"table":null}],"time_index":null,"unique_keys":[]}},{"guarantee":null,"operator":{"NonASAP":{"Project":{"child":19,"cols":[{"alias":"job","expr":{"Column":0}},{"alias":"topk_10","expr":{"Column":2}},{"alias":"$promql_series_identity","expr":{"Column":1}}],"qualifier":null}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"topk_10","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":null,"unique_keys":[]}}],"steps":[{"step":"stage0","title":"Stage 0","groups":[{"id":0,"label":"Scan http_requests_total","children":[],"parents":[1],"queries":[0],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":0},{"id":1,"label":"TimeRange 1m","children":[0],"parents":[2],"queries":[0],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":1},{"id":2,"label":"rate per series","children":[1],"parents":[3],"queries":[0],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":2},{"id":3,"label":"sum by (job)","children":[2],"parents":[],"queries":[0],"roots_of":[0],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":3},{"id":4,"label":"Scan http_requests_total","children":[],"parents":[5],"queries":[1],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":0},{"id":5,"label":"TimeRange 1m","children":[4],"parents":[6],"queries":[1],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":1},{"id":6,"label":"sum_over_time per series","children":[5],"parents":[7],"queries":[1],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":4},{"id":7,"label":"topk by (job) 10","children":[6],"parents":[],"queries":[1],"roots_of":[1],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":5}]},{"step":"pass1","title":"Stage 1, Pass 1","groups":[{"id":0,"label":"Scan http_requests_total","children":[],"parents":[1],"queries":[0],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[0],"children":[],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"Scan http_requests_total","group_of_node":{},"node":0}],"node":0},{"id":1,"label":"TimeRange 1m","children":[0],"parents":[2],"queries":[0],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[1],"children":[0],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"TimeRange 1m","group_of_node":{"0":0},"node":1}],"node":1},{"id":2,"label":"rate per series","children":[1],"parents":[3],"queries":[0],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[2],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"rate per series","group_of_node":{"1":1},"node":2}],"node":2},{"id":3,"label":"sum by (job)","children":[2],"parents":[],"queries":[0],"roots_of":[0],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[3],"children":[2],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"sum by (job)","group_of_node":{"2":2},"node":3}],"node":3},{"id":4,"label":"Scan http_requests_total","children":[],"parents":[5],"queries":[1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[4],"children":[],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"Scan http_requests_total","group_of_node":{},"node":0}],"node":0},{"id":5,"label":"TimeRange 1m","children":[4],"parents":[6],"queries":[1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[5],"children":[4],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"TimeRange 1m","group_of_node":{"0":4},"node":1}],"node":1},{"id":6,"label":"sum_over_time per series","children":[5],"parents":[7],"queries":[1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[6],"children":[5],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"sum_over_time per series","group_of_node":{"1":5},"node":4}],"node":4},{"id":7,"label":"topk by (job) 10","children":[6],"parents":[],"queries":[1],"roots_of":[1],"candidate_count":4,"candidate_count_by_origin":{"pass1":3,"shared_build":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[7],"children":[6],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"exact accumulator","group_of_node":{"4":6},"node":5},{"index":1,"implements":[7],"children":[6],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"CMS+heap","group_of_node":{"4":6},"node":5},{"index":2,"implements":[7],"children":[6],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"CountSketch+heap","group_of_node":{"4":6},"node":5},{"index":3,"implements":[7],"children":[6],"origin":"shared_build","accuracy_budget_parts":0.5,"added":true,"label":"CMS+heap shared build","group_of_node":{"4":6},"node":5}],"node":5}]}],"selected_choice":{"0":0,"1":0,"2":0,"3":0,"4":0,"5":0,"6":0,"7":1},"enumeration":{"step":"pass1","per_step":{"stage0":{"count":1,"complete":true},"pass1":{"count":4,"complete":true}},"limit":1000,"choices":[{"id":"E1","choice":{"0":0,"1":0,"2":0,"3":0,"4":0,"5":0,"6":0,"7":0},"accuracy_budget_parts_1":true,"selected":false,"valid":true,"rejection":null,"cost_per_second":3.0,"cheaper_than_selected":false,"roots":[11,14]},{"id":"E2","choice":{"0":0,"1":0,"2":0,"3":0,"4":0,"5":0,"6":0,"7":1},"accuracy_budget_parts_1":true,"selected":true,"valid":true,"rejection":null,"cost_per_second":2.0,"cheaper_than_selected":false,"roots":[11,17]},{"id":"E3","choice":{"0":0,"1":0,"2":0,"3":0,"4":0,"5":0,"6":0,"7":2},"accuracy_budget_parts_1":true,"selected":false,"valid":true,"rejection":null,"cost_per_second":1.5,"cheaper_than_selected":true,"roots":[11,20]},{"id":"E4","choice":{"0":0,"1":0,"2":0,"3":0,"4":0,"5":0,"6":0,"7":3},"accuracy_budget_parts_1":false,"selected":false,"valid":false,"rejection":"accuracy_target_missed","cost_per_second":null,"cheaper_than_selected":false,"roots":[11,17]}],"priced":{"count":4,"cheaper_than_selected":1,"selected_cost_per_second":2.0}}}} diff --git a/tools/dag-viewer/test-fixtures/memo-small.json b/tools/dag-viewer/test-fixtures/memo-small.json index cf7f96aea..2b8af0eb1 100644 --- a/tools/dag-viewer/test-fixtures/memo-small.json +++ b/tools/dag-viewer/test-fixtures/memo-small.json @@ -1 +1 @@ -{"memo":{"nodes":[{"guarantee":null,"materialization":"NotMaterialized","operator":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"source":{"TimeSeries":{"metric":"http_requests_total"}}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}},"result_kind":"range_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"NonASAP":{"Aggregate":{"child":1,"filters":[],"having":null,"measures":[{"kind":"rate"}],"output_names":[""],"reduction":"PerEntity"}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"NonASAP":{"Aggregate":{"child":2,"filters":[],"having":null,"measures":[{"col":null,"kind":"sum"}],"output_names":[""],"reduction":{"Reduce":[2]}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"NonASAP":{"Aggregate":{"child":1,"filters":[],"having":null,"measures":[{"col":null,"kind":"sum"}],"output_names":[""],"reduction":"PerEntity"}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"NonASAP":{"Aggregate":{"child":4,"filters":[],"having":null,"measures":[{"accuracy":{"accuracy_scope":"whole_result","failure_probability":0.001,"max_error":0.01},"k":10,"kind":"top_k"}],"output_names":[],"reduction":{"Reduce":[2]}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"topk_10","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"ASAP":{"SummaryAgg":{"child":1,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"timing":null},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"materialization":"NotMaterialized","operator":{"ASAP":{"FinalizeExactAccumulator":{"child":6}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"ASAP":{"SummaryAgg":{"child":2,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[2]}}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"timing":null},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"materialization":"NotMaterialized","operator":{"ASAP":{"FinalizeExactAccumulator":{"child":8}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"ASAP":{"SummaryAgg":{"child":1,"family":{"Sketch":[{"algorithm":"CmsWithHeap","category":"TopK","params":{"CmsWithHeap":{"depth":7,"heap_size":100,"width":272}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":{"Column":{"Named":"$promql_series_identity"}},"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"samples_of_metric","metric":"http_requests_total"}},"reduction":{"Reduce":[2]}}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Sketch":[{"algorithm":"CmsWithHeap","category":"TopK","params":{"CmsWithHeap":{"depth":7,"heap_size":100,"width":272}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"ASAP":{"SummaryEstimate":{"query":{"TopK":{"k":10}},"summary_input":10}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"topk","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"NonASAP":{"Project":{"child":11,"cols":[{"alias":"job","expr":{"Column":0}},{"alias":"topk_10","expr":{"Column":2}},{"alias":"$promql_series_identity","expr":{"Column":1}}],"qualifier":null}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"topk_10","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"ASAP":{"SummaryAgg":{"child":1,"family":{"Sketch":[{"algorithm":"CountSketchWithHeap","category":"TopK","params":{"CountSketchWithHeap":{"depth":81,"heap_size":100,"width":30000}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":{"Column":{"Named":"$promql_series_identity"}},"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"samples_of_metric","metric":"http_requests_total"}},"reduction":{"Reduce":[2]}}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Sketch":[{"algorithm":"CountSketchWithHeap","category":"TopK","params":{"CountSketchWithHeap":{"depth":81,"heap_size":100,"width":30000}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"ASAP":{"SummaryEstimate":{"query":{"TopK":{"k":10}},"summary_input":13}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"topk","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"NonASAP":{"Project":{"child":14,"cols":[{"alias":"job","expr":{"Column":0}},{"alias":"topk_10","expr":{"Column":2}},{"alias":"$promql_series_identity","expr":{"Column":1}}],"qualifier":null}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"topk_10","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"timing":null},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"materialization":"NotMaterialized","operator":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"source":{"TimeSeries":{"metric":"http_requests_total"}}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"ASAP":{"SummaryTumblingWindowChain":{"child":16,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"tumbling_window_count":6,"tumbling_window_width_ms":10000,"reduction":"PerEntity"}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"ASAP":{"SummaryMerge":{"children":[17]}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"timing":null},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"materialization":"NotMaterialized","operator":{"ASAP":{"FinalizeExactAccumulator":{"child":18}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"ASAP":{"SummaryTumblingWindowChain":{"child":0,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"tumbling_window_count":6,"tumbling_window_width_ms":10000,"reduction":"PerEntity"}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"timing":null}],"steps":[{"step":"stage0","title":"Stage 0","groups":[{"id":0,"label":"Scan http_requests_total","children":[],"parents":[1],"queries":[0,1],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":0},{"id":1,"label":"TimeRange 1m","children":[0],"parents":[2,4],"queries":[0,1],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":1},{"id":2,"label":"rate per series","children":[1],"parents":[3],"queries":[0],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":2},{"id":3,"label":"sum by (job)","children":[2],"parents":[],"queries":[0],"roots_of":[0],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":3},{"id":4,"label":"sum_over_time per series","children":[1],"parents":[5],"queries":[1],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":4},{"id":5,"label":"topk 10 by (job)","children":[4],"parents":[],"queries":[1],"roots_of":[1],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":5}]},{"step":"pass1","title":"Stage 1, Pass 1","groups":[{"id":0,"label":"Scan http_requests_total","children":[],"parents":[1],"queries":[0,1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"Scan","group_of_node":{},"node":0}],"node":0},{"id":1,"label":"TimeRange 1m","children":[0],"parents":[2,4],"queries":[0,1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[0],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"TimeRange 1m","group_of_node":{"0":0},"node":1}],"node":1},{"id":2,"label":"rate per series","children":[1],"parents":[3],"queries":[0],"roots_of":[],"candidate_count":2,"candidate_count_by_origin":{"pass1":2},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"exact rate accumulator","group_of_node":{"1":1},"node":7},{"index":1,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"exact Aggregate rate","group_of_node":{"1":1},"node":2}],"node":2},{"id":3,"label":"sum by (job)","children":[2],"parents":[],"queries":[0],"roots_of":[0],"candidate_count":2,"candidate_count_by_origin":{"pass1":2},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[2],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"exact Aggregate sum","group_of_node":{"2":2},"node":3},{"index":1,"implements":[],"children":[2],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"exact sum accumulator","group_of_node":{"2":2},"node":9}],"node":3},{"id":4,"label":"sum_over_time per series","children":[1],"parents":[5],"queries":[1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"exact Aggregate sum_over_time","group_of_node":{"1":1},"node":4}],"node":4},{"id":5,"label":"topk 10 by (job)","children":[4],"parents":[],"queries":[1],"roots_of":[1],"candidate_count":3,"candidate_count_by_origin":{"pass1":3},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[4],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"exact Aggregate topk","group_of_node":{"4":4},"node":5},{"index":1,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"CMS+heap depth 7 width 272","group_of_node":{"1":1},"node":12},{"index":2,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"CountSketch+heap","group_of_node":{"1":1},"node":15}],"node":5}]},{"step":"pass2","title":"Stage 1, Pass 2","groups":[{"id":0,"label":"Scan http_requests_total","children":[],"parents":[1,6],"queries":[0,1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"Scan","group_of_node":{},"node":0}],"node":0},{"id":1,"label":"TimeRange 1m","children":[0],"parents":[2,4],"queries":[0,1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[0],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"TimeRange 1m","group_of_node":{"0":0},"node":1}],"node":1},{"id":2,"label":"rate per series","children":[1],"parents":[3],"queries":[0],"roots_of":[],"candidate_count":20,"candidate_count_by_origin":{"pass1":2,"tumbling_windows":9,"shared_build":9},"candidates_capped":true,"candidates":[{"index":0,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"exact rate accumulator","group_of_node":{"1":1},"node":7},{"index":1,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"exact Aggregate rate","group_of_node":{"1":1},"node":2},{"index":2,"implements":[],"children":[6],"origin":"tumbling_windows","accuracy_budget_parts":1,"added":true,"label":"rate accumulator over G6's panes, variant 1","group_of_node":{"17":6},"node":19},{"index":3,"implements":[],"children":[6],"origin":"tumbling_windows","accuracy_budget_parts":1,"added":true,"label":"rate accumulator over G6's panes, variant 2","group_of_node":{"17":6},"node":19},{"index":4,"implements":[],"children":[6],"origin":"tumbling_windows","accuracy_budget_parts":1,"added":true,"label":"rate accumulator over G6's panes, variant 3","group_of_node":{"17":6},"node":19},{"index":5,"implements":[],"children":[6],"origin":"tumbling_windows","accuracy_budget_parts":1,"added":true,"label":"rate accumulator over G6's panes, variant 4","group_of_node":{"17":6},"node":19},{"index":6,"implements":[],"children":[6],"origin":"tumbling_windows","accuracy_budget_parts":1,"added":true,"label":"rate accumulator over G6's panes, variant 5","group_of_node":{"17":6},"node":19},{"index":7,"implements":[],"children":[6],"origin":"tumbling_windows","accuracy_budget_parts":1,"added":true,"label":"rate accumulator over G6's panes, variant 6","group_of_node":{"17":6},"node":19},{"index":8,"implements":[],"children":[6],"origin":"shared_build","accuracy_budget_parts":1,"added":true,"label":"rate accumulator read from shared build, variant 1","group_of_node":{"17":6},"node":19},{"index":9,"implements":[],"children":[6],"origin":"shared_build","accuracy_budget_parts":1,"added":true,"label":"rate accumulator read from shared build, variant 2","group_of_node":{"17":6},"node":19},{"index":10,"implements":[],"children":[6],"origin":"shared_build","accuracy_budget_parts":1,"added":true,"label":"rate accumulator read from shared build, variant 3","group_of_node":{"17":6},"node":19},{"index":11,"implements":[],"children":[6],"origin":"shared_build","accuracy_budget_parts":1,"added":true,"label":"rate accumulator read from shared build, variant 4","group_of_node":{"17":6},"node":19},{"index":12,"implements":[],"children":[6],"origin":"shared_build","accuracy_budget_parts":1,"added":true,"label":"rate accumulator read from shared build, variant 5","group_of_node":{"17":6},"node":19}],"node":2},{"id":3,"label":"sum by (job)","children":[2],"parents":[],"queries":[0],"roots_of":[0],"candidate_count":3,"candidate_count_by_origin":{"pass1":2,"rollup":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[2],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"exact Aggregate sum","group_of_node":{"2":2},"node":3},{"index":1,"implements":[],"children":[2],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"exact sum accumulator","group_of_node":{"2":2},"node":9},{"index":2,"implements":[],"children":[2],"origin":"rollup","accuracy_budget_parts":1,"added":true,"label":"sum by (job) rolled up from G2's accumulator","group_of_node":{"2":2},"node":9}],"node":3},{"id":4,"label":"sum_over_time per series","children":[1],"parents":[5],"queries":[1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"exact Aggregate sum_over_time","group_of_node":{"1":1},"node":4}],"node":4},{"id":5,"label":"topk 10 by (job)","children":[4],"parents":[],"queries":[1],"roots_of":[1],"candidate_count":3,"candidate_count_by_origin":{"pass1":3},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[4],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"exact Aggregate topk","group_of_node":{"4":4},"node":5},{"index":1,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"CMS+heap depth 7 width 272","group_of_node":{"1":1},"node":12},{"index":2,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"CountSketch+heap","group_of_node":{"1":1},"node":15}],"node":5},{"id":6,"label":"SummaryTumblingWindowChain 6 × 10 s","children":[0],"parents":[],"queries":[0],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"shared_build":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[0],"origin":"shared_build","accuracy_budget_parts":1,"added":true,"label":"SummaryTumblingWindowChain 6 × 10 s, rate accumulator","group_of_node":{"0":0},"node":20}],"node":17}]}],"selected_choice":{"5":1,"3":0,"2":2,"6":0,"1":0,"0":0,"4":0}}} +{"memo":{"nodes":[{"guarantee":null,"operator":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"source":{"TimeSeries":{"metric":"http_requests_total"}}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}},"result_kind":"range_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"NonASAP":{"Aggregate":{"child":1,"filters":[],"having":null,"measures":[{"kind":"rate"}],"output_names":[""],"reduction":"PerEntity"}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"NonASAP":{"Aggregate":{"child":2,"filters":[],"having":null,"measures":[{"col":null,"kind":"sum"}],"output_names":[""],"reduction":{"Reduce":[2]}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":null,"operator":{"NonASAP":{"Aggregate":{"child":1,"filters":[],"having":null,"measures":[{"col":null,"kind":"sum"}],"output_names":[""],"reduction":"PerEntity"}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"NonASAP":{"Aggregate":{"child":4,"filters":[],"having":null,"measures":[{"accuracy":{"accuracy_scope":"whole_result","failure_probability":0.001,"max_error":0.01},"k":10,"kind":"top_k"}],"output_names":[],"reduction":{"Reduce":[2]}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"topk_10","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryAgg":{"child":1,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"operator":{"ASAP":{"FinalizeExactAccumulator":{"child":6}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryAgg":{"child":2,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[2]}}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"operator":{"ASAP":{"FinalizeExactAccumulator":{"child":8}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryAgg":{"child":1,"family":{"Sketch":[{"algorithm":"CmsWithHeap","category":"TopK","params":{"CmsWithHeap":{"depth":7,"heap_size":100,"width":272}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":{"Column":{"Named":"$promql_series_identity"}},"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"samples_of_metric","metric":"http_requests_total"}},"reduction":{"Reduce":[2]}}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Sketch":[{"algorithm":"CmsWithHeap","category":"TopK","params":{"CmsWithHeap":{"depth":7,"heap_size":100,"width":272}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryEstimate":{"query":{"TopK":{"k":10}},"summary_input":10}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"topk","nullable":false,"table":null}],"time_index":null,"unique_keys":[]}},{"guarantee":null,"operator":{"NonASAP":{"Project":{"child":11,"cols":[{"alias":"job","expr":{"Column":0}},{"alias":"topk_10","expr":{"Column":2}},{"alias":"$promql_series_identity","expr":{"Column":1}}],"qualifier":null}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"topk_10","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":null,"unique_keys":[]}},{"guarantee":null,"operator":{"ASAP":{"SummaryAgg":{"child":1,"family":{"Sketch":[{"algorithm":"CountSketchWithHeap","category":"TopK","params":{"CountSketchWithHeap":{"depth":81,"heap_size":100,"width":30000}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":{"Column":{"Named":"$promql_series_identity"}},"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"samples_of_metric","metric":"http_requests_total"}},"reduction":{"Reduce":[2]}}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Sketch":[{"algorithm":"CountSketchWithHeap","category":"TopK","params":{"CountSketchWithHeap":{"depth":81,"heap_size":100,"width":30000}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryEstimate":{"query":{"TopK":{"k":10}},"summary_input":13}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"topk","nullable":false,"table":null}],"time_index":null,"unique_keys":[]}},{"guarantee":null,"operator":{"NonASAP":{"Project":{"child":14,"cols":[{"alias":"job","expr":{"Column":0}},{"alias":"topk_10","expr":{"Column":2}},{"alias":"$promql_series_identity","expr":{"Column":1}}],"qualifier":null}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"topk_10","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":null,"unique_keys":[]}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"operator":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"source":{"TimeSeries":{"metric":"http_requests_total"}}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryTumblingWindowChain":{"child":16,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"tumbling_window_count":6,"tumbling_window_width_ms":10000,"reduction":"PerEntity"}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryMerge":{"children":[17]}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"operator":{"ASAP":{"FinalizeExactAccumulator":{"child":18}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryTumblingWindowChain":{"child":0,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"tumbling_window_count":6,"tumbling_window_width_ms":10000,"reduction":"PerEntity"}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}}],"steps":[{"step":"stage0","title":"Stage 0","groups":[{"id":0,"label":"Scan http_requests_total","children":[],"parents":[1],"queries":[0,1],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":0},{"id":1,"label":"TimeRange 1m","children":[0],"parents":[2,4],"queries":[0,1],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":1},{"id":2,"label":"rate per series","children":[1],"parents":[3],"queries":[0],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":2},{"id":3,"label":"sum by (job)","children":[2],"parents":[],"queries":[0],"roots_of":[0],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":3},{"id":4,"label":"sum_over_time per series","children":[1],"parents":[5],"queries":[1],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":4},{"id":5,"label":"topk 10 by (job)","children":[4],"parents":[],"queries":[1],"roots_of":[1],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":5}]},{"step":"pass1","title":"Stage 1, Pass 1","groups":[{"id":0,"label":"Scan http_requests_total","children":[],"parents":[1],"queries":[0,1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"Scan","group_of_node":{},"node":0}],"node":0},{"id":1,"label":"TimeRange 1m","children":[0],"parents":[2,4],"queries":[0,1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[0],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"TimeRange 1m","group_of_node":{"0":0},"node":1}],"node":1},{"id":2,"label":"rate per series","children":[1],"parents":[3],"queries":[0],"roots_of":[],"candidate_count":2,"candidate_count_by_origin":{"pass1":2},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"exact rate accumulator","group_of_node":{"1":1},"node":7},{"index":1,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"exact Aggregate rate","group_of_node":{"1":1},"node":2}],"node":2},{"id":3,"label":"sum by (job)","children":[2],"parents":[],"queries":[0],"roots_of":[0],"candidate_count":2,"candidate_count_by_origin":{"pass1":2},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[2],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"exact Aggregate sum","group_of_node":{"2":2},"node":3},{"index":1,"implements":[],"children":[2],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"exact sum accumulator","group_of_node":{"2":2},"node":9}],"node":3},{"id":4,"label":"sum_over_time per series","children":[1],"parents":[5],"queries":[1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"exact Aggregate sum_over_time","group_of_node":{"1":1},"node":4}],"node":4},{"id":5,"label":"topk 10 by (job)","children":[4],"parents":[],"queries":[1],"roots_of":[1],"candidate_count":3,"candidate_count_by_origin":{"pass1":3},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[4],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"exact Aggregate topk","group_of_node":{"4":4},"node":5},{"index":1,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"CMS+heap depth 7 width 272","group_of_node":{"1":1},"node":12},{"index":2,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"CountSketch+heap","group_of_node":{"1":1},"node":15}],"node":5}]},{"step":"pass2","title":"Stage 1, Pass 2","groups":[{"id":0,"label":"Scan http_requests_total","children":[],"parents":[1,6],"queries":[0,1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"Scan","group_of_node":{},"node":0}],"node":0},{"id":1,"label":"TimeRange 1m","children":[0],"parents":[2,4],"queries":[0,1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[0],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"TimeRange 1m","group_of_node":{"0":0},"node":1}],"node":1},{"id":2,"label":"rate per series","children":[1],"parents":[3],"queries":[0],"roots_of":[],"candidate_count":20,"candidate_count_by_origin":{"pass1":2,"tumbling_windows":9,"shared_build":9},"candidates_capped":true,"candidates":[{"index":0,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"exact rate accumulator","group_of_node":{"1":1},"node":7},{"index":1,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"exact Aggregate rate","group_of_node":{"1":1},"node":2},{"index":2,"implements":[],"children":[6],"origin":"tumbling_windows","accuracy_budget_parts":1,"added":true,"label":"rate accumulator over G6's panes, variant 1","group_of_node":{"17":6},"node":19},{"index":3,"implements":[],"children":[6],"origin":"tumbling_windows","accuracy_budget_parts":1,"added":true,"label":"rate accumulator over G6's panes, variant 2","group_of_node":{"17":6},"node":19},{"index":4,"implements":[],"children":[6],"origin":"tumbling_windows","accuracy_budget_parts":1,"added":true,"label":"rate accumulator over G6's panes, variant 3","group_of_node":{"17":6},"node":19},{"index":5,"implements":[],"children":[6],"origin":"tumbling_windows","accuracy_budget_parts":1,"added":true,"label":"rate accumulator over G6's panes, variant 4","group_of_node":{"17":6},"node":19},{"index":6,"implements":[],"children":[6],"origin":"tumbling_windows","accuracy_budget_parts":1,"added":true,"label":"rate accumulator over G6's panes, variant 5","group_of_node":{"17":6},"node":19},{"index":7,"implements":[],"children":[6],"origin":"tumbling_windows","accuracy_budget_parts":1,"added":true,"label":"rate accumulator over G6's panes, variant 6","group_of_node":{"17":6},"node":19},{"index":8,"implements":[],"children":[6],"origin":"shared_build","accuracy_budget_parts":1,"added":true,"label":"rate accumulator read from shared build, variant 1","group_of_node":{"17":6},"node":19},{"index":9,"implements":[],"children":[6],"origin":"shared_build","accuracy_budget_parts":1,"added":true,"label":"rate accumulator read from shared build, variant 2","group_of_node":{"17":6},"node":19},{"index":10,"implements":[],"children":[6],"origin":"shared_build","accuracy_budget_parts":1,"added":true,"label":"rate accumulator read from shared build, variant 3","group_of_node":{"17":6},"node":19},{"index":11,"implements":[],"children":[6],"origin":"shared_build","accuracy_budget_parts":1,"added":true,"label":"rate accumulator read from shared build, variant 4","group_of_node":{"17":6},"node":19},{"index":12,"implements":[],"children":[6],"origin":"shared_build","accuracy_budget_parts":1,"added":true,"label":"rate accumulator read from shared build, variant 5","group_of_node":{"17":6},"node":19}],"node":2},{"id":3,"label":"sum by (job)","children":[2],"parents":[],"queries":[0],"roots_of":[0],"candidate_count":3,"candidate_count_by_origin":{"pass1":2,"rollup":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[2],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"exact Aggregate sum","group_of_node":{"2":2},"node":3},{"index":1,"implements":[],"children":[2],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"exact sum accumulator","group_of_node":{"2":2},"node":9},{"index":2,"implements":[],"children":[2],"origin":"rollup","accuracy_budget_parts":1,"added":true,"label":"sum by (job) rolled up from G2's accumulator","group_of_node":{"2":2},"node":9}],"node":3},{"id":4,"label":"sum_over_time per series","children":[1],"parents":[5],"queries":[1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"exact Aggregate sum_over_time","group_of_node":{"1":1},"node":4}],"node":4},{"id":5,"label":"topk 10 by (job)","children":[4],"parents":[],"queries":[1],"roots_of":[1],"candidate_count":3,"candidate_count_by_origin":{"pass1":3},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[4],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"exact Aggregate topk","group_of_node":{"4":4},"node":5},{"index":1,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"CMS+heap depth 7 width 272","group_of_node":{"1":1},"node":12},{"index":2,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"CountSketch+heap","group_of_node":{"1":1},"node":15}],"node":5},{"id":6,"label":"SummaryTumblingWindowChain 6 × 10 s","children":[0],"parents":[],"queries":[0],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"shared_build":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[0],"origin":"shared_build","accuracy_budget_parts":1,"added":true,"label":"SummaryTumblingWindowChain 6 × 10 s, rate accumulator","group_of_node":{"0":0},"node":20}],"node":17}]}],"selected_choice":{"5":1,"3":0,"2":2,"6":0,"1":0,"0":0,"4":0}}}