From f57fcc06fe176228b96e1d656e2830a1cbed1ed5 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Sat, 7 Nov 2026 01:37:24 +0000 Subject: [PATCH 1/8] =?UTF-8?q?feat(types):=20export=20with=20each=20node'?= =?UTF-8?q?s=20timing,=20lifetime,=20=CE=B8=20and=20summary=20coverage;=20?= =?UTF-8?q?workload=20chain=20expansion;=20view=20coverage=20from=20the=20?= =?UTF-8?q?node=20(#1207=20PR=206)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit physical_asap_dag_with_assignments exports roots with an ExportedNodeAssignment per operator node (physical properties, θ, summary coverage); the query-time test exporter calls it. PhysicalASAPDAGNode gains summary_coverage: Option, the memo group's coverage the realizer sets, which materialized_views reads instead of deriving coverage from node trees (operator_nodes and the coverage todo deleted). ViewCoverage::of and of_tumbling_window give a plan node's coverage and each expanded tumbling window's. expand_tumbling_window_chains_in_workload returns again for installation. ScanWindow::union is public for Stage 3's memo-level scan windows. test_support::set_summary_coverage_from_memo sets coverage on hand-built plans through a Memo. --- crates/types/src/ir/materialized_view.rs | 81 ++++++++------ .../src/ir/operator/tumbling_window_chain.rs | 95 +++++++++++++++-- crates/types/src/ir/physical_export.rs | 100 +++++++++--------- crates/types/src/test_support.rs | 43 ++++++++ crates/types/tests/physical_export.rs | 2 + 5 files changed, 237 insertions(+), 84 deletions(-) diff --git a/crates/types/src/ir/materialized_view.rs b/crates/types/src/ir/materialized_view.rs index d9971be0c..80068336b 100644 --- a/crates/types/src/ir/materialized_view.rs +++ b/crates/types/src/ir/materialized_view.rs @@ -64,7 +64,7 @@ //! it once. use std::collections::{HashMap, HashSet}; -use std::rc::Rc; +use std::ops::Bound; use serde::{Deserialize, Serialize}; use thiserror::Error; @@ -74,11 +74,11 @@ use super::physical_export::{ PhysicalASAPNodeId, ScanWindow, }; use crate::ir::operator::asap::ASAPOp; -use crate::ir::operator::node::{Operator, OperatorNode}; +use crate::ir::operator::node::Operator; use crate::ir::operator::non_asap::NonASAPOp; use crate::ir::operator::operator_properties::Source; use crate::ir::properties::execution_data_state::ExecutionTiming; -use crate::ir::properties::summary_coverage::{CoverageError, SelectionBox, SummaryCoverage}; +use crate::ir::properties::summary_coverage::{SelectionBox, SummaryCoverage}; use crate::physical::Lifetime; /// One materialized view of a DAG: what its stored state is, what it is @@ -137,6 +137,49 @@ pub struct ViewCoverage { pub(crate) selection: Vec, } +impl ViewCoverage { + /// The coverage a plan node carries for the summary coverage of its + /// memo group: the definition's id and the selection. + pub fn of(coverage: &SummaryCoverage) -> Self { + Self { + definition_id: coverage.definition_id(), + selection: coverage.selection.clone(), + } + } + + /// The coverage of tumbling window `index` (0 = newest) of a chain of + /// tumbling windows `width_ms` wide whose coverage is `self`: the same + /// definition, each box's time window narrowed to the tumbling window's, + /// `(u - (index + 1)·w, u - index·w]` for the box's upper time bound `u` + /// (0 when the box has none: the chain ends at the evaluation time). + pub fn of_tumbling_window(&self, index: u32, width_ms: u64) -> Self { + let width_ms = i64::try_from(width_ms).unwrap_or(i64::MAX); + let back_ms = width_ms.saturating_mul(i64::from(index)); + let selection = self + .selection + .iter() + .map(|selection_box| { + let upper_ms = match selection_box.relative_time { + Some((_, Bound::Included(upper) | Bound::Excluded(upper))) => upper, + _ => 0, + }; + let end_ms = upper_ms.saturating_sub(back_ms); + SelectionBox { + columns: selection_box.columns.clone(), + relative_time: Some(( + Bound::Excluded(end_ms.saturating_sub(width_ms)), + Bound::Included(end_ms), + )), + } + }) + .collect(); + Self { + definition_id: self.definition_id.clone(), + selection, + } + } +} + #[cfg(any(test, feature = "test-support"))] impl ViewCoverage { /// `SummaryCoverage::definition_id`. @@ -735,14 +778,7 @@ impl PhysicalASAPDAG { .filter(|node| matches!(node.lifetime, Lifetime::AcrossEvaluations(_))) .map(|node| node.id) .collect(); - // The IR of each node, which coverage is derived from. - let ir = if views.is_empty() { - HashMap::new() - } else { - self.operator_nodes()? - }; let mut derived = Vec::new(); - let mut coverage_by_node: HashMap = HashMap::new(); // The window each view summarizes ([`PhysicalASAPDAG::view_window`]): // its raw-data windows and those of the views it reads, unioned. let mut window_by_view: HashMap = HashMap::new(); @@ -790,13 +826,14 @@ impl PhysicalASAPDAG { ); } } - let coverage = coverage_of(id, &inputs, &ir, &coverage_by_node).map_err(|error| { + // The coverage its memo group stores, which the realizer put + // on the node: coverage is derived only in the memo. + let coverage = node.summary_coverage.clone().ok_or_else(|| { InstallableDagError::ViewWithoutCoverage { node_id: id, - reason: error.to_string(), + reason: "its plan node carries no summary coverage".into(), } })?; - coverage_by_node.insert(id, coverage.clone()); let view = MaterializedView { node_id: id, refresh: if refreshed_at_ingestion { @@ -804,10 +841,7 @@ impl PhysicalASAPDAG { } else { MaterializedViewRefresh::OnFirstRead }, - coverage: ViewCoverage { - definition_id: coverage.definition_id(), - selection: coverage.selection, - }, + coverage, inputs, retention_ms: node.retention_ms.unwrap_or(0), }; @@ -1086,16 +1120,3 @@ impl<'a> ViewIndex<'a> { Ok((inputs, scans)) } } - -/// The coverage of view `id` (IR `ir`), refreshed from `inputs`: the -/// coverage its memo group stores, which the realized plan will carry. -/// Coverage is derived only in the memo, never on node trees. -fn coverage_of( - id: PhysicalASAPNodeId, - inputs: &MaterializedViewInputs, - ir: &HashMap>, - coverage_by_node: &HashMap, -) -> Result { - let _ = (id, inputs, ir, coverage_by_node); - todo!("#1207 PR 6: the view's coverage from its memo group's GroupProperties::summary_coverage") -} diff --git a/crates/types/src/ir/operator/tumbling_window_chain.rs b/crates/types/src/ir/operator/tumbling_window_chain.rs index 8d1ea5811..cc5451b93 100644 --- a/crates/types/src/ir/operator/tumbling_window_chain.rs +++ b/crates/types/src/ir/operator/tumbling_window_chain.rs @@ -11,7 +11,6 @@ //! build over `TimeRange(m·w)` (`merged_window_build`): one chain materialized for //! the longest window serves every query whose window, on the same tumbling windows, //! ends at the same time and is shorter. -#[cfg(any(test, feature = "test-support"))] use std::collections::HashMap; use std::rc::Rc; use std::time::Duration; @@ -21,8 +20,8 @@ use super::input::node_of_build; use super::input::{InputOrTemporaryBuild, OperatorInput}; use super::node::{Operator, OperatorNode}; use super::non_asap::{NonASAPOp, TimeRangeKind}; -#[cfg(any(test, feature = "test-support"))] use super::operator_properties::TimeShift; +use super::query::QueryRoot; use crate::ir::schema::Schema; use crate::ir::SchemaDerivationError; @@ -149,7 +148,6 @@ pub(crate) fn window_build( ) } -#[cfg(any(test, feature = "test-support"))] /// Tumbling window `index` (0 = newest) of `chain`: its `SummaryAgg` over the range /// `TimeRange(tumbling_window_width_ms)` ending `index · tumbling_window_width_ms` before the /// chain's input. @@ -179,7 +177,6 @@ pub(crate) fn tumbling_window( ) } -#[cfg(any(test, feature = "test-support"))] /// `child` evaluated `back_ms` earlier: one `TimeShift`, folded into a /// `TimeShift` directly below. fn shifted( @@ -288,7 +285,94 @@ pub fn expand_tumbling_window_chains( expand(root, &mut NodeToExpandedNodeCache::default()) } -#[cfg(any(test, feature = "test-support"))] +/// A workload with its tumbling-window chains expanded +/// ([`expand_tumbling_window_chains_in_workload`]). +#[derive(Debug, Clone)] +pub struct TumblingWindowChainExpansion { + /// The workload's roots, in order, with every chain expanded. + pub roots: Vec, + /// Each expanded chain with its tumbling windows some merge reads, newest first, + /// in the order the chains were first reached. + pub chains: Vec, + /// Every other node of the input workload the expansion reached, with + /// the node that replaces it (itself when nothing below it changed), + /// so a caller can carry what it knows about a node to the expanded + /// workload. + pub expanded_nodes: Vec<(Rc, Rc)>, +} + +/// One expanded chain: the `SummaryTumblingWindowChain` node of the input workload and the +/// `SummaryAgg` tumbling windows that replace it. +#[derive(Debug, Clone)] +pub struct ExpandedTumblingWindowChain { + /// The chain node, as the input workload holds it. + pub chain: Rc, + /// Its tumbling windows in the expanded workload that some merge reads, newest + /// first. + pub tumbling_windows: Vec>, +} + +/// `expand_tumbling_window_chains` over every root of one workload, with one cache, so +/// a node several roots share stays one node and a chain several merges +/// read expands into one set of tumbling windows. A standalone scalar root has the +/// operators it reads expanded. Also returns each chain's tumbling windows, so a +/// caller can carry what it knows about a chain (its physical properties, +/// θ, coverage and retention) to them. +pub fn expand_tumbling_window_chains_in_workload( + roots: &[QueryRoot], +) -> Result { + let mut cache = NodeToExpandedNodeCache::default(); + let roots = roots + .iter() + .map(|root| match root { + QueryRoot::Operator(node) => expand(node, &mut cache).map(QueryRoot::Operator), + QueryRoot::Scalar(expr) => { + let mut error = None; + let expr = expr.map_operator_refs(&mut |node| { + expand(node, &mut cache).unwrap_or_else(|e| { + error.get_or_insert(e); + Rc::clone(node) + }) + }); + match error { + Some(e) => Err(e), + None => Ok(QueryRoot::Scalar(expr)), + } + } + }) + .collect::, _>>()?; + let chains = cache + .chains + .into_iter() + .map(|chain| { + let tumbling_window_count = match chain.asap() { + Some(ASAPOp::SummaryTumblingWindowChain { + tumbling_window_count, + .. + }) => *tumbling_window_count, + _ => unreachable!("only chains are recorded"), + }; + let tumbling_windows = (0..tumbling_window_count) + .map_while(|index| { + cache + .tumbling_window_of + .get(&(Rc::as_ptr(&chain), index)) + .cloned() + }) + .collect(); + ExpandedTumblingWindowChain { + chain, + tumbling_windows, + } + }) + .collect(); + Ok(TumblingWindowChainExpansion { + roots, + chains, + expanded_nodes: cache.expanded, + }) +} + /// What one expansion has built: each expanded node by its input's /// address, and each tumbling window by its chain's address and index, so a chain two /// merges read expands into one set of tumbling windows. @@ -302,7 +386,6 @@ struct NodeToExpandedNodeCache { expanded: Vec<(Rc, Rc)>, } -#[cfg(any(test, feature = "test-support"))] fn expand( node: &Rc, cache: &mut NodeToExpandedNodeCache, diff --git a/crates/types/src/ir/physical_export.rs b/crates/types/src/ir/physical_export.rs index f6d383b28..b6d9ca9b9 100644 --- a/crates/types/src/ir/physical_export.rs +++ b/crates/types/src/ir/physical_export.rs @@ -19,10 +19,11 @@ use crate::ir::operator::asap::ASAPOp; use crate::ir::operator::node::{Operator, OperatorNode}; use crate::ir::operator::non_asap::NonASAPOp; +use super::materialized_view::ViewCoverage; use crate::ir::properties::execution_data_state::{ExecutionDataState, ExecutionTiming}; use crate::ir::properties::guarantee::AccuracyGuarantee; use crate::ir::schema::{FieldDataType, Schema, SketchParams}; -use crate::physical::{Lifetime, PhysicalPropertiesError}; +use crate::physical::{Lifetime, PhysicalProperties, PhysicalPropertiesError}; /// A node's operator, with each child replaced by its node id. pub type PhysicalASAPOperatorPayload = Operator; @@ -87,6 +88,13 @@ pub struct PhysicalASAPDAGNode { /// enumerates lifetimes before Stage 3 knows retention. #[serde(default, skip_serializing_if = "Option::is_none")] pub retention_ms: Option, + /// For a summary state, the summary coverage of its memo group + /// (`GroupProperties::summary_coverage`) as the plan carries it: what + /// a materialized view of it stores ([`PhysicalASAPDAG::materialized_views`]). + /// Coverage is derived only in the memo, so the realizer sets it; `None` + /// on every other node. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub summary_coverage: Option, } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] @@ -238,7 +246,7 @@ pub enum ScanWindow { impl ScanWindow { /// The smallest window holding both. - pub(crate) fn union(self, other: Self) -> Self { + pub fn union(self, other: Self) -> Self { let ( Self::Within { end_offset_ms: a_end, @@ -341,42 +349,6 @@ pub(crate) fn offset_below<'a>( } impl PhysicalASAPDAG { - /// Every node of this DAG as IR, by node id, with its exported - /// guarantee (an operator node has no physical properties): what a caller derives - /// IR properties of a plan file's node from, such as its - /// [`SummaryCoverage`](crate::ir::properties::summary_coverage::SummaryCoverage). - /// Shared inputs stay shared. Validates the DAG first. - pub(crate) fn operator_nodes( - &self, - ) -> Result>, PhysicalASAPDAGValidationError> { - self.validate()?; - let by_id: HashMap = - self.nodes.iter().map(|node| (node.id, node)).collect(); - let mut rebuilt: HashMap> = HashMap::new(); - // Children first, without recursion: `validate` rejected cycles. - for node in &self.nodes { - let mut pending = vec![(node.id, false)]; - while let Some((id, children_done)) = pending.pop() { - if rebuilt.contains_key(&id) { - continue; - } - let node = by_id[&id]; - if !children_done { - pending.push((id, true)); - pending.extend(node.payload.children().into_iter().map(|c| (*c, false))); - continue; - } - let operator = node - .payload - .map_children(|child| Rc::clone(&rebuilt[child])); - let restored = OperatorNode::with_schema(operator, node.output_schema.clone()) - .with_guarantee(node.guarantee.clone()); - rebuilt.insert(id, Rc::new(restored)); - } - } - Ok(rebuilt) - } - /// The rows of the `Scan` `scan` its consumers read, back from the /// evaluation time `T`: the union of each consumer's window. A consumer /// reaches the `Scan` through `TimeShift`s without an `@` anchor @@ -802,14 +774,28 @@ pub(crate) fn is_materializable_state_output(operator: &Operator) -> bool ) } -/// Test-only: `roots` exported as one DAG with every node at query time and -/// not materialized, and no θ (every `parameters` is `None`), with each -/// exported node's operator node by node id. Executor tests hand-build DAGs -/// this way; planning exports only through the realizer -/// (`AssignmentRealizer::realize`). -#[cfg(any(test, feature = "test-support"))] -pub(crate) fn physical_asap_dag_at_query_time_with_node_ids( +/// What the realizer assigns one node of a plan it exports +/// ([`physical_asap_dag_with_assignments`]). +#[derive(Debug, Clone, PartialEq)] +pub struct ExportedNodeAssignment { + /// The node's timing and lifetime. + pub physical_properties: PhysicalProperties, + /// θ of a summary build; `None` otherwise. + pub parameters: Option, + /// Its memo group's summary coverage, for a summary state. + pub summary_coverage: Option, +} + +/// `roots` exported as one DAG, each node with what `assignment_of` gives +/// its operator node (timing, lifetime, θ and coverage), with each exported +/// node's operator node by node id. Nodes shared by pointer are one node; +/// operator queries become `roots` and standalone scalar queries +/// `scalar_roots`, keyed by their position. The realizer +/// (`AssignmentRealizer::realize`) exports the selected plan this way. +/// Validated before it is returned. +pub fn physical_asap_dag_with_assignments( roots: &[crate::ir::QueryRoot], + assignment_of: &dyn Fn(&Rc) -> ExportedNodeAssignment, ) -> Result<(PhysicalASAPDAG, Vec>), PhysicalASAPDAGValidationError> { use crate::ir::properties::execution_data_state::DataPrimitive; use crate::ir::QueryRoot; @@ -839,8 +825,9 @@ pub(crate) fn physical_asap_dag_at_query_time_with_node_ids( } _ => DataPrimitive::Raw, }; + let assignment = assignment_of(node); let output_state = ExecutionDataState { - timing: ExecutionTiming::QueryTime, + timing: assignment.physical_properties.timing(), primitive, }; // Children come before their parents, so every producer is already in `nodes`. @@ -859,11 +846,12 @@ pub(crate) fn physical_asap_dag_at_query_time_with_node_ids( id, payload: flat_node.operator, output_state, - lifetime: Lifetime::None, - parameters: None, + lifetime: assignment.physical_properties.lifetime(), + parameters: assignment.parameters, output_schema: flat_node.schema, guarantee: flat_node.guarantee, retention_ms: None, + summary_coverage: assignment.summary_coverage, }); } let dag = PhysicalASAPDAG { @@ -876,7 +864,23 @@ pub(crate) fn physical_asap_dag_at_query_time_with_node_ids( Ok((dag, nodes_by_id)) } +/// Test-only: `roots` exported as one DAG with every node at query time and +/// not materialized, with no θ and no coverage +/// ([`physical_asap_dag_with_assignments`]). Executor tests hand-build DAGs +/// this way; planning exports only through the realizer +/// (`AssignmentRealizer::realize`). #[cfg(any(test, feature = "test-support"))] +pub(crate) fn physical_asap_dag_at_query_time_with_node_ids( + roots: &[crate::ir::QueryRoot], +) -> Result<(PhysicalASAPDAG, Vec>), PhysicalASAPDAGValidationError> { + physical_asap_dag_with_assignments(roots, &|_| ExportedNodeAssignment { + physical_properties: PhysicalProperties::new(ExecutionTiming::QueryTime, Lifetime::None) + .expect("query time, not materialized, is valid"), + parameters: None, + summary_coverage: None, + }) +} + fn grouping_compatibility( producer: &Operator, consumer: &Operator, diff --git a/crates/types/src/test_support.rs b/crates/types/src/test_support.rs index 684fa2433..40ba9a74a 100644 --- a/crates/types/src/test_support.rs +++ b/crates/types/src/test_support.rs @@ -154,3 +154,46 @@ pub fn population_over_selector( }); Rc::new(OperatorNode::with_schema(evaluation, root.schema.clone())) } + +/// Each summary-state node of `dag` with the summary coverage of its memo +/// group: the DAG's operators hash-consed bottom up into a `Memo` +/// (`Memo::find_or_add_group`), each node's coverage read from its group's +/// `GroupProperties::summary_coverage`, as the realizer does for a selected +/// plan. For tests that hand-build plans with materialized views; coverage +/// is derived only in the memo, never on node trees. +pub fn set_summary_coverage_from_memo(dag: &mut crate::ir::physical_export::PhysicalASAPDAG) { + use crate::ir::materialized_view::ViewCoverage; + use crate::stages::{Memo, MemoGroupId}; + use std::collections::HashMap; + let mut memo = Memo::default(); + let mut group_of: HashMap = HashMap::new(); + // Children before parents: repeat passes until every node's children + // have groups (hand-built DAGs need not number children first). + while group_of.len() < dag.nodes.len() { + let before = group_of.len(); + for node in &dag.nodes { + if group_of.contains_key(&node.id) + || node + .payload + .children() + .iter() + .any(|c| !group_of.contains_key(*c)) + { + continue; + } + let expression = node.payload.map_children(|child| group_of[child]); + let group = memo + .find_or_add_group(expression) + .expect("a valid plan's operators form memo groups"); + group_of.insert(node.id, group); + } + assert!(group_of.len() > before, "the DAG has a cycle"); + } + for node in &mut dag.nodes { + node.summary_coverage = memo.groups()[group_of[&node.id].0 as usize] + .group_properties + .summary_coverage + .as_ref() + .map(ViewCoverage::of); + } +} diff --git a/crates/types/tests/physical_export.rs b/crates/types/tests/physical_export.rs index 7324878b5..6a142a7f1 100644 --- a/crates/types/tests/physical_export.rs +++ b/crates/types/tests/physical_export.rs @@ -437,6 +437,7 @@ fn high_fan_in_edges_preserve_multiplicity_without_order_dependence() { output_schema: schema.clone(), guarantee: None, retention_ms: None, + summary_coverage: None, }) .collect(); let inputs: Vec<_> = (0..count).chain([0, 0]).collect(); @@ -451,6 +452,7 @@ fn high_fan_in_edges_preserve_multiplicity_without_order_dependence() { output_schema: schema.clone(), guarantee: None, retention_ms: None, + summary_coverage: None, }); let mut dag = PhysicalASAPDAG { nodes, From 941173958917580355ed28f495513d370254d865 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Sat, 7 Nov 2026 01:37:24 +0000 Subject: [PATCH 2/8] =?UTF-8?q?feat(cost):=20a=20summary's=20state=20shape?= =?UTF-8?q?=20and=20family=20from=20the=20node's=20=CE=B8=20(#1207=20PR=20?= =?UTF-8?q?6)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit summary_shape takes the build's parameters (PhysicalASAPDAGNode::parameters) and fails with UnsupportedCandidate for a sketch without θ; state_family reads the node's θ. Restores the sizing tests deleted in PR 0. --- crates/cost/src/state_bytes.rs | 167 +++++++++++++++++++++++++++++++-- crates/cost/src/usage.rs | 20 ++-- 2 files changed, 169 insertions(+), 18 deletions(-) diff --git a/crates/cost/src/state_bytes.rs b/crates/cost/src/state_bytes.rs index 4b7d297e6..a4f4f8742 100644 --- a/crates/cost/src/state_bytes.rs +++ b/crates/cost/src/state_bytes.rs @@ -1,6 +1,7 @@ //! The bytes one summary or exact state holds and its update operations //! per input row, from its parameters and the rows it folds. +use crate::analytical::AnalyticalCostError; use asap_types::ir::operator::AggIntent; use asap_types::ir::schema::sketch::KLL_MAX_K; use asap_types::ir::schema::{ @@ -58,15 +59,21 @@ pub fn keeps_every_value(measure: &AggIntent) -> bool { /// Count-Min and Count-Sketch, plus one heap update for their top-k variants; /// for UnivMon, the same per layer it updates (`univmon_expected_layers`); /// 1 for every other sketch. -pub fn summary_shape(family: &FieldDataType, input_rows: StateInputRows) -> (f64, u64) { - match family { - FieldDataType::Sketch(..) => { - let _ = input_rows; - todo!("#1207 PR 6: the state shape of θ from the candidate's LogicalProperties::parameters") - } - FieldDataType::ExactAggregate(..) => (1.0, EXACT_ACCUMULATOR_STATE_BYTES), +/// +/// `parameters` is the build's θ (`PhysicalASAPDAGNode::parameters`). Err +/// [`AnalyticalCostError::UnsupportedCandidate`] for a sketch without θ: a +/// planner bug, since every sketch build carries its θ. +pub fn summary_shape( + family: &FieldDataType, + parameters: Option<&SketchParams>, + input_rows: StateInputRows, +) -> Result<(f64, u64), AnalyticalCostError> { + Ok(match (family, parameters) { + (FieldDataType::Sketch(..), Some(parameters)) => sketch_shape(parameters, input_rows), + (FieldDataType::Sketch(..), None) => return Err(AnalyticalCostError::UnsupportedCandidate), + (FieldDataType::ExactAggregate(..), _) => (1.0, EXACT_ACCUMULATOR_STATE_BYTES), _ => (1.0, 8), - } + }) } /// What one summary state's bytes depend on besides its parameters (only @@ -290,9 +297,10 @@ mod tests { assert_eq!( summary_shape( &FieldDataType::ExactAggregate(kind, params), + None, StateInputRows::UNKNOWN ), - (1.0, 56) + Ok((1.0, 56)) ); } assert_eq!(asap_types::ir::schema::EXACT_ACCUMULATOR_STATE_BYTES, 56); @@ -357,6 +365,147 @@ mod tests { assert_eq!(priced(Some(10_000), false), priced(None, false)); } + /// The state shape of a sketch of `algorithm` with θ `params`, its + /// samples unknown. + fn shape(algorithm: SketchAlgorithm, params: SketchParams) -> (f64, u64) { + summary_shape( + &FieldDataType::Sketch(algorithm, GroupingStrategy::default()), + Some(¶ms), + StateInputRows::UNKNOWN, + ) + .unwrap() + } + + /// A sketch build without θ is a planner bug: it cannot be sized. + #[test] + fn a_sketch_without_parameters_is_not_sized() { + assert_eq!( + summary_shape( + &FieldDataType::Sketch(SketchAlgorithm::Kll, GroupingStrategy::default()), + None, + StateInputRows::UNKNOWN, + ), + Err(AnalyticalCostError::UnsupportedCandidate) + ); + } + + /// DDSketch state grows as its relative accuracy α tightens. + #[test] + fn ddsketch_state_grows_as_alpha_shrinks() { + let bytes = |alpha| shape(SketchAlgorithm::DDSketch, SketchParams::DDSketch { alpha }).1; + assert!( + bytes(0.1) < bytes(0.01), + "{} vs {}", + bytes(0.1), + bytes(0.01) + ); + assert!( + bytes(0.01) < bytes(0.001), + "{} vs {}", + bytes(0.01), + bytes(0.001) + ); + } + + /// Every sketch family's state is sized from its parameters, as + /// `asap_sketchlib` allocates it: an HLL's registers, a KMV's `k` + /// hashes once it has seen `k` distinct items, and a UnivMon's layers + /// once keys of the assumed size have churned through every heap. + #[test] + fn summary_state_is_sized_from_sketch_params() { + use asap_sketchlib::DataInput; + assert_eq!( + shape( + SketchAlgorithm::Hll, + SketchParams::Hll { + precision: 14, + relative_error: None + } + ) + .1, + asap_sketchlib::HyperLogLog::::new().memory_bytes() as u64 + ); + let mut kmv: asap_sketchlib::Kmv = asap_sketchlib::Kmv::new(1_000); + for i in 0..2_000u64 { + kmv.update(&DataInput::U64(i)); + } + assert_eq!( + shape( + SketchAlgorithm::Kmv, + SketchParams::Kmv { + k: 1_000, + relative_error: None + } + ) + .1, + kmv.memory_bytes() as u64 + ); + let mut univmon = asap_sketchlib::UnivMon::new(8, 3, 64, 2); + for i in 0..10_000u64 { + let mut key = [0; ASSUMED_VALUE_KEY_BYTES]; + key[..8].copy_from_slice(&i.to_le_bytes()); + univmon.update_weighted(&DataInput::Bytes(&key), 1); + } + assert_eq!( + shape( + SketchAlgorithm::UnivMon, + SketchParams::UnivMon { + heap_size: 8, + sketch_rows: 3, + sketch_cols: 64, + layers: 2, + distinct_failure_probability: None, + } + ) + .1, + univmon.memory_bytes() as u64 + ); + } + + /// KLL state whose samples are unknown is charged its fixed 744 bytes + /// and the item array at the capacity `asap_sketchlib` grows it to: + /// `compute_max_capacity(k, 8)` 8-byte items, with `k` capped at 26 602. + #[test] + fn kll_state_is_sized_as_asap_sketchlib_allocates_it() { + let bytes = |k| { + shape( + SketchAlgorithm::Kll, + SketchParams::Kll { + k, + failure_probability: None, + }, + ) + .1 + }; + assert_eq!(bytes(269), 744 + 8 * 1_206); + assert_eq!(bytes(26_602), 744 + 8 * 80_118); + assert_eq!(bytes(30_000), 744 + 8 * 80_118); + } + + /// A UnivMon row updates layers `0..=b`, where its bottom layer `b` is + /// geometric (`asap_sketchlib`'s `bottom_layer_for_hash`), so + /// `2 - 2^(1 - layers)` layers in expectation; each updates one counter + /// per sketch row and its heap. + #[test] + fn univmon_update_cost_is_expected_layers_times_rows_and_heap() { + let ops = |layers| { + shape( + SketchAlgorithm::UnivMon, + SketchParams::UnivMon { + heap_size: 256, + sketch_rows: 5, + sketch_cols: 1_024, + layers, + distinct_failure_probability: None, + }, + ) + .0 + }; + assert_eq!(ops(1), 6.0); + assert_eq!(ops(2), 1.5 * 6.0); + assert_eq!(ops(16), (2.0 - 2f64.powi(-15)) * 6.0); + } + /// A KLL state is charged what `asap_sketchlib` allocates for the /// samples it sees, not its full capacity: its fixed struct and level /// array, plus the item array the library starts empty and doubles from diff --git a/crates/cost/src/usage.rs b/crates/cost/src/usage.rs index 6047d5fb2..97efdc1bc 100644 --- a/crates/cost/src/usage.rs +++ b/crates/cost/src/usage.rs @@ -458,15 +458,15 @@ fn each_node_usage<'a>( }) } -/// The state family of a node with `payload` whose first producer has -/// `producer_family`: a build's own (one sketch instance per group; a -/// shared Hydra structure is not that sketch), inherited by the merges, +/// The state family of `node` whose first producer has `producer_family`: +/// a build's own, its θ (`node.parameters`; one sketch instance per group; +/// a shared Hydra structure is not that sketch), inherited by the merges, /// estimates and finalizations over it, none otherwise. fn state_family<'a>( - payload: &'a Payload, + node: &'a asap_types::ir::physical_export::PhysicalASAPDAGNode, producer_family: Option>, ) -> StateFamily<'a> { - match payload { + match &node.payload { Payload::ASAP( ASAPOp::SummaryAgg { family, grouping, .. @@ -478,7 +478,9 @@ fn state_family<'a>( FieldDataType::Sketch(..) if *grouping == GroupingStrategy::PerSubpopulationInstance => { - todo!("#1207 PR 6: the state family of θ from the candidate's LogicalProperties::parameters") + node.parameters + .as_ref() + .map_or(StateFamily::None, StateFamily::Sketch) } FieldDataType::ExactAggregate(kind, _) => StateFamily::Exact(kind), _ => StateFamily::None, @@ -594,7 +596,7 @@ pub fn node_usage<'a>( missing: "an arrival rate (the arrival's rows_per_second, or the \ source's series_count with sample_interval)", }; - let family = state_family(&node.payload, producers.first().map(|p| p.family)); + let family = state_family(node, producers.first().map(|p| p.family)); let unchanged_columns = unchanged_source_columns( node, &producers @@ -916,7 +918,7 @@ pub fn node_usage<'a>( rows_per_state: (!ingestion).then(|| input.rows.div_ceil(group_count.max(1))), non_negative: raw_data_samples_non_negative(), }; - let (depth, state_bytes) = summary_shape(family, input_rows); + let (depth, state_bytes) = summary_shape(family, node.parameters.as_ref(), input_rows)?; let state_bytes = measured_state_bytes.unwrap_or(state_bytes); let out = EdgeStatistics { rows: group_count, @@ -972,7 +974,7 @@ pub fn node_usage<'a>( rows_per_state: Some(rows_per_tumbling_window.div_ceil(group_count.max(1))), non_negative: raw_data_samples_non_negative(), }; - let (depth, state_bytes) = summary_shape(family, input_rows); + let (depth, state_bytes) = summary_shape(family, node.parameters.as_ref(), input_rows)?; let state_bytes = measured_state_bytes.unwrap_or(state_bytes); let out = EdgeStatistics { rows: states, From a38583ffec11e33b4db5a7bc3f751473473efdef Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Sat, 7 Nov 2026 01:37:24 +0000 Subject: [PATCH 3/8] =?UTF-8?q?feat(executor):=20build,=20check=20and=20ru?= =?UTF-8?q?n=20summaries=20with=20=CE=B8=20from=20the=20exported=20node=20?= =?UTF-8?q?(#1207=20PR=206)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The physical planner passes PhysicalASAPDAGNode::parameters to the summary build operators, which carry it to StateColumnUpdater, create_planner_accumulator and create_hydra_cms. validate_summary_kernel checks θ against the family; a state's consumers check the state's own shape at run time (validate_state, keyed evaluation capacity). A SummaryMerge carries its build's θ for the empty global state. Tests set θ on hand-built plans (test_support::set_test_parameters) and restore the parameter tests deleted in PR 0. --- crates/executor/src/capability.rs | 273 ++++++++++---- crates/executor/src/operators/mod.rs | 20 +- crates/executor/src/operators/summary/mod.rs | 181 +++++++-- crates/executor/src/operators/unchecked.rs | 29 +- crates/executor/src/physical_planner/mod.rs | 138 ++++++- .../executor/src/summary_kernels/factory.rs | 169 ++++++++- .../summary_kernels/state_column_updater.rs | 88 +++-- .../src/summary_kernels/weighted_frequency.rs | 7 +- crates/executor/src/supported_summaries.rs | 67 +++- crates/executor/src/values/batch.rs | 31 +- crates/executor/src/values/mod.rs | 55 ++- crates/executor/tests/blocking_resources.rs | 7 +- crates/executor/tests/current_series_heap.rs | 3 +- crates/executor/tests/current_timestamp.rs | 1 + crates/executor/tests/deployment.rs | 91 +++-- crates/executor/tests/exact_state_columns.rs | 3 +- crates/executor/tests/hydra_cms_execution.rs | 27 +- .../tests/keyed_summary_row_weight.rs | 19 +- crates/executor/tests/physical_dag.rs | 37 +- crates/executor/tests/physical_semantics.rs | 50 ++- crates/executor/tests/plan_properties.rs | 58 ++- .../executor/tests/precompute_population.rs | 5 + crates/executor/tests/promql_binary.rs | 2 + crates/executor/tests/raw_scan.rs | 3 + crates/executor/tests/state_column_updater.rs | 350 ++++++++++++++++-- .../stored_tumbling_window_memory_limit.rs | 14 +- .../tests/summary_build_memory_limit.rs | 17 +- crates/executor/tests/summary_projection.rs | 4 + crates/executor/tests/univmon_execution.rs | 18 +- .../executor/tests/weighted_topk_binding.rs | 7 +- 30 files changed, 1508 insertions(+), 266 deletions(-) diff --git a/crates/executor/src/capability.rs b/crates/executor/src/capability.rs index 00aa39b12..ec8cdcf8d 100644 --- a/crates/executor/src/capability.rs +++ b/crates/executor/src/capability.rs @@ -8,46 +8,47 @@ //! //! Stored-state encodings belong to deployments. Full plan acceptance is //! owned by `binding`, which also validates schemas, expressions and inputs. - +use crate::summary_kernels::hll_sketch::HLL_PRECISIONS; +use crate::summary_kernels::weighted_frequency::{self, FrequencyAlgorithm}; use crate::Error; use planner_types::ir::schema::{ ExactKind, ExactParams, FamilyCapability, FieldDataType as SummaryFamilyType, GroupingStrategy, - SketchAlgorithm, SketchStatistic, StatisticKind, SummaryUpdate, + SketchAlgorithm, SketchParams, SketchStatistic, StatisticKind, SummaryUpdate, }; /// Check the same contract used by `create_planner_accumulator` before a plan /// is accepted. Execution timing is deliberately not a kernel property. +/// +/// `parameters` is θ, which the exported build node carries +/// (`PhysicalASAPDAGNode::parameters`): required for a sketch family, +/// `None` for an exact one. pub fn validate_summary_kernel( family: &SummaryFamilyType, + parameters: Option<&SketchParams>, input: &SummaryUpdate, grouping: &GroupingStrategy, ) -> Result<(), String> { if grouping != &GroupingStrategy::PerSubpopulationInstance { - return validate_hydra_cms(family, input, grouping); + return validate_hydra_cms(family, parameters, input, grouping); } let keyed = match family { - SummaryFamilyType::ExactAggregate(kind, params) => { - use ExactKind as K; - use ExactParams as P; - if !matches!( - (kind, params), - (K::Sum, P::Sum) - | (K::Count, P::Count) - | (K::Min, P::Min) - | (K::Max, P::Max) - | (K::Rate, P::Rate) - | (K::Increase, P::Increase) - ) { + SummaryFamilyType::ExactAggregate(..) => { + if !exact_kernel(family) { return Err(format!("unsupported exact kernel {family:?}")); } + if parameters.is_some() { + return Err("an exact summary has no parameters θ".into()); + } input.item.is_some() } - SummaryFamilyType::Sketch(kind, layout) => { + SummaryFamilyType::Sketch(algorithm, layout) => { if layout != grouping { return Err("Planner family and operator grouping disagree".into()); } - let _ = kind; - todo!("#1207 PR 6: check θ, which the exported node carries") + let Some(parameters) = parameters else { + return Err(format!("a {algorithm:?} summary needs its parameters θ")); + }; + validate_sketch_parameters(algorithm, parameters)? } _ => return Err(format!("unsupported summary kernel {family:?}")), }; @@ -57,6 +58,90 @@ pub fn validate_summary_kernel( Ok(()) } +/// Whether an exact family has a kernel. +fn exact_kernel(family: &SummaryFamilyType) -> bool { + use ExactKind as K; + use ExactParams as P; + matches!( + family, + SummaryFamilyType::ExactAggregate(K::Sum, P::Sum) + | SummaryFamilyType::ExactAggregate(K::Count, P::Count) + | SummaryFamilyType::ExactAggregate(K::Min, P::Min) + | SummaryFamilyType::ExactAggregate(K::Max, P::Max) + | SummaryFamilyType::ExactAggregate(K::Rate, P::Rate) + | SummaryFamilyType::ExactAggregate(K::Increase, P::Increase) + ) +} + +/// Check θ of a per-instance sketch of `algorithm` against what its kernel +/// builds; whether the kernel is keyed (its updates name an item). +fn validate_sketch_parameters( + algorithm: &SketchAlgorithm, + parameters: &SketchParams, +) -> Result { + use SketchAlgorithm as A; + use SketchParams as P; + let keyed = match (algorithm, parameters) { + // `asap_sketchlib` silently caps a larger `k`. + (A::Kll, P::Kll { k, .. }) + if (8..=planner_types::ir::schema::sketch::KLL_MAX_K).contains(k) => + { + false + } + (A::DDSketch, P::DDSketch { alpha }) + if alpha.is_finite() && *alpha > 0.0 && *alpha < 1.0 => + { + false + } + (A::Hll, P::Hll { precision, .. }) if HLL_PRECISIONS.contains(precision) => false, + (A::Kmv, P::Kmv { k, .. }) if *k >= crate::summary_kernels::kmv::KMV_MIN_K => false, + (A::Cms, P::Cms { width, depth }) if valid_matrix(*width, *depth) => true, + // The kernel's shape rules (an odd depth, for the median). + (A::CountSketch, P::CountSketch { width, depth }) + if valid_matrix(*width, *depth) + && weighted_frequency::check_shape( + FrequencyAlgorithm::CountSketch, + *width as usize, + *depth as usize, + ) + .is_ok() => + { + true + } + (A::CmsWithHeap | A::CountSketchWithHeap, _) + if weighted_frequency::WeightedFrequency::configuration(algorithm, parameters) + .is_ok_and(|(_, width, depth, _)| valid_matrix(width as u32, depth as u32)) => + { + true + } + ( + A::UnivMon, + P::UnivMon { + heap_size, + sketch_rows, + sketch_cols, + layers, + .. + }, + ) if crate::summary_kernels::univmon::check_shape( + *heap_size as usize, + *sketch_rows as usize, + *sketch_cols as usize, + usize::from(*layers), + ) + .is_ok() => + { + false + } + _ => { + return Err(format!( + "unsupported kernel or invalid parameters: {algorithm:?} {parameters:?}" + )) + } + }; + Ok(keyed) +} + /// The only shared grouping with a kernel: Hydra over Count-Min. Each update /// adds one item column's value to its group, with weight 1 (a row count, /// `UnitCount`), a positive integer constant (`PositiveConstant`) or a weight @@ -64,6 +149,7 @@ pub fn validate_summary_kernel( /// when no weight is negative). fn validate_hydra_cms( family: &SummaryFamilyType, + parameters: Option<&SketchParams>, input: &SummaryUpdate, grouping: &GroupingStrategy, ) -> Result<(), String> { @@ -71,7 +157,7 @@ fn validate_hydra_cms( if !matches!(family, SummaryFamilyType::Sketch(_, layout) if layout == grouping) { return Err("Planner family and operator grouping disagree".into()); } - hydra_cms_shape(family)?; + hydra_cms_shape(family, parameters)?; if !matches!(input.item, Some(SummaryInputExpr::Column(_))) { return Err("HydraCms requires one item column".into()); } @@ -113,26 +199,57 @@ pub(crate) fn constant_row_weight(weight: f64) -> Result Result<(), String> { + use planner_types::ir::schema::HydraKind; + match family { + SummaryFamilyType::Sketch( + SketchAlgorithm::Cms, + GroupingStrategy::SharedMultiSubpopulation { + kind: HydraKind::HydraCms, + }, + ) => Ok(()), + SummaryFamilyType::Sketch( + _, + GroupingStrategy::SharedMultiSubpopulation { + kind: HydraKind::HydraCms, + }, + ) => Err("HydraCms family must be Count-Min".into()), + _ => Err("a shared-grouping (Hydra) family has no registered kernel".into()), + } +} + +/// `(shared_rows, shared_columns, width, depth)` of a HydraCms family whose +/// θ is `parameters` (`SketchParams::Hydra(HydraParams::HydraCms { .. })`): +/// the grid's own rows and columns, and the width and depth of the per-group +/// Count-Min it emulates. pub(crate) fn hydra_cms_shape( family: &SummaryFamilyType, + parameters: Option<&SketchParams>, ) -> Result<(usize, usize, usize, usize), String> { - use planner_types::ir::schema::HydraKind; - let SummaryFamilyType::Sketch( - kind, - GroupingStrategy::SharedMultiSubpopulation { - kind: HydraKind::HydraCms, - }, - ) = family + use planner_types::ir::schema::HydraParams; + validate_hydra_cms_family(family)?; + let Some(SketchParams::Hydra(HydraParams::HydraCms { + width, + depth, + shared_rows, + shared_columns, + })) = parameters else { - return Err("a shared-grouping (Hydra) family has no registered kernel".into()); + return Err(format!( + "a HydraCms summary needs HydraCms parameters θ, got {parameters:?}" + )); }; - if kind != &SketchAlgorithm::Cms { - return Err("HydraCms family must be Count-Min with the Hydra width and depth".into()); + if !valid_matrix(*width, *depth) || !valid_matrix(*shared_columns, *shared_rows) { + return Err("invalid HydraCms dimensions".into()); } - todo!("#1207 PR 6: the Hydra grid's sizes are θ, which the exported node carries") + Ok(( + *shared_rows as usize, + *shared_columns as usize, + *width as usize, + *depth as usize, + )) } /// A filtered `SummaryAgg` never runs at ingestion time. An ingestion-time @@ -153,7 +270,6 @@ pub fn validate_ingestion_time_summary( } } -#[allow(dead_code)] // Used again in #1207 PR 6, with θ from the exported node. fn valid_matrix(width: u32, depth: u32) -> bool { // Construction uses the kernel's native row hashing, so no encoded-size // limit applies here. @@ -184,39 +300,39 @@ pub(crate) fn unit_frequency_column( } } +/// Whether the executor holds, merges and reads states of `family` natively. +/// θ is not part of a family: each kernel checks it when building a state, +/// and merges check that their states have one shape. pub fn validate_native_family(family: &SummaryFamilyType) -> Result<(), Error> { use planner_types::ir::schema::SketchAlgorithm as A; - if let SummaryFamilyType::Sketch(kind, grouping) = family { - // Plain Count-Min is native as stored state only: it merges and reads - // its bare count, but the DAG does not build it from rows. HydraCms - // is built by the all-groups summary build. - if let A::Cms = kind { - if grouping != &GroupingStrategy::PerSubpopulationInstance { - return hydra_cms_shape(family).map(|_| ()).map_err(Error::Invalid); - } - todo!("#1207 PR 6: check the Count-Min θ, which the exported node carries") - } - if matches!(kind, A::CmsWithHeap | A::CountSketchWithHeap) { - todo!("#1207 PR 6: check the weighted-frequency θ, which the exported node carries") - } - } - match family { - SummaryFamilyType::ExactAggregate(..) => {} - SummaryFamilyType::Sketch(A::Kll | A::DDSketch | A::Hll | A::Kmv | A::UnivMon, _) => {} - _ => { - return Err(Error::Invalid( - "summary family has no native DAG state implementation".into(), - )) + let native = match family { + SummaryFamilyType::ExactAggregate(..) => exact_kernel(family), + // HydraCms is built by the all-groups summary build. + SummaryFamilyType::Sketch(_, GroupingStrategy::SharedMultiSubpopulation { .. }) => { + return validate_hydra_cms_family(family).map_err(Error::Invalid) } + // Plain Count-Min is native as stored state only: it merges and reads + // its bare count, but the DAG does not build it from rows. + SummaryFamilyType::Sketch( + A::Cms + | A::CmsWithHeap + | A::CountSketchWithHeap + | A::Kll + | A::DDSketch + | A::Hll + | A::Kmv + | A::UnivMon, + GroupingStrategy::PerSubpopulationInstance, + ) => true, + _ => false, + }; + if native { + Ok(()) + } else { + Err(Error::Invalid(format!( + "summary family {family:?} has no native DAG state implementation" + ))) } - crate::capability::validate_summary_kernel( - family, - &planner_types::ir::schema::SummaryUpdate::column( - planner_types::ir::scalar::ColumnRef::SampleValue, - ), - &Default::default(), - ) - .map_err(Error::Invalid) } /// What the executor evaluates from a per-instance state of each family it @@ -350,4 +466,35 @@ mod tests { assert!(validate_ingestion_time_summary(Some(&filter)).is_err()); assert!(validate_ingestion_time_summary::(None).is_ok()); } + + // θ is checked with the kernel, not with the family: a Count Sketch with + // heap of even depth is refused (its estimate is a median of rows), + // while its family stays native. + #[test] + fn weighted_frequency_parameters_are_checked_with_the_kernel() { + let family = SummaryFamilyType::Sketch( + SketchAlgorithm::CountSketchWithHeap, + GroupingStrategy::PerSubpopulationInstance, + ); + let mut update = SummaryUpdate::column(planner_types::ir::scalar::ColumnRef::SampleValue); + update.item = Some(planner_types::ir::schema::SummaryInputExpr::Column( + planner_types::ir::scalar::ColumnRef::Named("host".into()), + )); + let theta = |depth| SketchParams::CountSketchWithHeap { + width: 32, + depth, + heap_size: 8, + }; + let grouping = GroupingStrategy::PerSubpopulationInstance; + assert!(validate_native_family(&family).is_ok()); + assert!(validate_summary_kernel(&family, Some(&theta(5)), &update, &grouping).is_ok()); + assert!(validate_summary_kernel(&family, Some(&theta(4)), &update, &grouping).is_err()); + assert!(validate_summary_kernel(&family, None, &update, &grouping).is_err()); + let heapless = SketchParams::CountSketchWithHeap { + width: 32, + depth: 5, + heap_size: 0, + }; + assert!(validate_summary_kernel(&family, Some(&heapless), &update, &grouping).is_err()); + } } diff --git a/crates/executor/src/operators/mod.rs b/crates/executor/src/operators/mod.rs index eb391edbd..79f7dfcfe 100644 --- a/crates/executor/src/operators/mod.rs +++ b/crates/executor/src/operators/mod.rs @@ -9,7 +9,8 @@ use crate::{ use futures::StreamExt; use planner_types::ir::scalar::ColumnRef; use planner_types::ir::schema::{ - DataType, Field as SummaryField, FieldDataType as SummaryFamilyType, Schema, SummaryUpdate, + DataType, Field as SummaryField, FieldDataType as SummaryFamilyType, Schema, SketchParams, + SummaryUpdate, }; use std::{collections::BTreeMap, sync::Arc}; pub(crate) mod common; @@ -156,6 +157,10 @@ enum Kind { /// (built by [`Operator::per_group_summary_build`]). PerGroupSummaryBuild { family: SummaryFamilyType, + /// θ of the states: the exported build node's `parameters`; `None` + /// for an exact family. + #[serde(default, skip_serializing_if = "Option::is_none")] + parameters: Option, value: usize, time: Option, groups: Vec, @@ -167,6 +172,10 @@ enum Kind { /// with heap (built by [`Operator::per_group_keyed_summary_build`]). PerGroupKeyedSummaryBuild { family: SummaryFamilyType, + /// θ of the states: the exported build node's `parameters`; `None` + /// for an exact family. + #[serde(default, skip_serializing_if = "Option::is_none")] + parameters: Option, items: Vec, /// What each row adds to its item's count. row_weight: RowWeight, @@ -179,6 +188,10 @@ enum Kind { /// HydraCms grid (built by [`Operator::all_groups_summary_build`]). AllGroupsSummaryBuild { family: SummaryFamilyType, + /// θ of the states: the exported build node's `parameters`; `None` + /// for an exact family. + #[serde(default, skip_serializing_if = "Option::is_none")] + parameters: Option, item: usize, /// What each row adds to its item's count. row_weight: RowWeight, @@ -199,6 +212,11 @@ enum Kind { /// with no groups; `false` (no output row) by default. #[serde(default, skip_serializing_if = "std::ops::Not::not")] emit_empty_global: bool, + /// θ of the merged states, which the empty global state takes: the + /// `parameters` of the build node that produced them. `None` for an + /// exact family, and without `emit_empty_global`. + #[serde(default, skip_serializing_if = "Option::is_none")] + parameters: Option, }, Evaluation { state: usize, diff --git a/crates/executor/src/operators/summary/mod.rs b/crates/executor/src/operators/summary/mod.rs index 2eee3373d..0fcad8de6 100644 --- a/crates/executor/src/operators/summary/mod.rs +++ b/crates/executor/src/operators/summary/mod.rs @@ -29,34 +29,97 @@ pub enum SummaryEvaluation { } impl Operator { - /// One keyed summary state per group (a CMS or Count Sketch with heap): - /// each row adds its `row_weight` to the tuple of its `items` values, in - /// its group's state. A weight column must be Float64 or Int64, the - /// types the kernel's `f64` weights hold; a NULL weight adds nothing. + /// One keyed summary state per group (a CMS or Count Sketch with heap) + /// of θ `parameters`: each row adds its `row_weight` to the tuple of its + /// `items` values, in its group's state. A weight column must be Float64 + /// or Int64, the types the kernel's `f64` weights hold; a NULL weight + /// adds nothing. pub fn per_group_keyed_summary_build( input: SchemaRef, family: SummaryFamilyType, + parameters: Option, items: Vec, row_weight: RowWeight, groups: Vec, ) -> Result { - let _ = (input, family, items, row_weight, groups); - todo!("#1207 PR 6: the weighted-frequency θ, which the exported node carries") + use crate::summary_kernels::weighted_frequency::WeightedFrequency; + use planner_types::ir::schema::{GroupingStrategy, SketchAlgorithm as A}; + crate::values::validate_family(&family)?; + let SummaryFamilyType::Sketch( + algorithm @ (A::CmsWithHeap | A::CountSketchWithHeap), + GroupingStrategy::PerSubpopulationInstance, + ) = &family + else { + return Err(invalid("keyed sketch required")); + }; + let Some(theta) = ¶meters else { + return Err(invalid("a keyed sketch needs its parameters θ")); + }; + WeightedFrequency::configuration(algorithm, theta)?; + validate_groups(&input, &groups)?; + if items.is_empty() { + return Err(invalid("keyed summary requires item columns")); + } + if let RowWeight::Column(weight) = row_weight { + if !matches!( + plain(&input, weight)?.0, + DataType::Float64 | DataType::Int64 + ) { + return Err(invalid("keyed summary weights must be Float64 or Int64")); + } + } + for &item in &items { + if !matches!( + plain(&input, item)?.0, + DataType::Utf8 + | DataType::Timestamp + | DataType::Int64 + | DataType::Float64 + | DataType::Bool + | DataType::Null + ) { + return Err(invalid("unsupported keyed summary identity type")); + } + } + let mut fields = groups + .iter() + .map(|&i| input.fields[i].clone()) + .collect::>(); + fields.push(SummaryField { + name: "state".into(), + dtype: family.clone(), + nullable: false, + }); + Ok(Self { + kind: Kind::PerGroupKeyedSummaryBuild { + family, + parameters, + items, + row_weight, + groups, + filter: None, + }, + inputs: vec![input], + output: schema(fields), + }) } - /// One summary state shared by all groups (a HydraCms grid), emitted as - /// one row per group that reads its own part of it. Each row adds its + /// One summary state shared by all groups (a HydraCms grid sized by θ + /// `parameters`, `SketchParams::Hydra(HydraParams::HydraCms { .. })`), + /// emitted as one row per group that reads its own part of it. Each row adds its /// `row_weight` to its `item` value in its group; a NULL weight adds /// nothing. A constant weight is at most `i32::MAX` (asap_sketchlib's /// Hydra adds `i32` weights). pub fn all_groups_summary_build( input: SchemaRef, family: SummaryFamilyType, + parameters: Option, item: usize, row_weight: RowWeight, groups: Vec, ) -> Result { crate::values::validate_family(&family)?; - crate::factory::create_hydra_cms(&family).map_err(Error::Invalid)?; + // The grid's shape only: the grid itself is allocated when it runs. + crate::capability::hydra_cms_shape(&family, parameters.as_ref()).map_err(Error::Invalid)?; validate_groups(&input, &groups)?; if !matches!( plain(&input, item)?, @@ -96,6 +159,7 @@ impl Operator { Ok(Self { kind: Kind::AllGroupsSummaryBuild { family, + parameters, item, row_weight, groups, @@ -120,22 +184,55 @@ impl Operator { } Ok(self) } + /// The `k` heaviest items of each input row's keyed state (a CMS or + /// Count Sketch with heap), one output row per item. The state's heap + /// capacity is its θ, which the state carries: a state whose heap holds + /// fewer than `k` items is refused when it is read. pub fn keyed_evaluation( input: SchemaRef, state: usize, k: usize, output: SchemaRef, ) -> Result { - let _ = (input, state, k, output); - todo!("#1207 PR 6: the weighted-frequency capacity of θ, which the exported node carries") + use planner_types::ir::schema::{GroupingStrategy, SketchAlgorithm as A}; + let family = &field(&input, state)?.dtype; + crate::values::validate_family(family)?; + if !matches!( + family, + SummaryFamilyType::Sketch( + A::CmsWithHeap | A::CountSketchWithHeap, + GroupingStrategy::PerSubpopulationInstance + ) + ) { + return Err(invalid("keyed evaluation requires a keyed summary state")); + } + if output.fields.len() <= input.fields.len() { + return Err(invalid("invalid keyed evaluation shape")); + } + if state + 1 != input.fields.len() + || output.fields[..state] != input.fields[..state] + || output.fields.last().unwrap().dtype != SummaryFamilyType::Plain(DataType::Float64) + { + return Err(invalid( + "keyed evaluation must preserve partitions and return a Float64 score", + )); + } + crate::values::validate_schema(&output)?; + Ok(Self { + kind: Kind::KeyedEvaluation { state, k }, + inputs: vec![input], + output, + }) } /// One summary state per group, updated from the value of one input /// column per row (`time` orders counter samples): an exact aggregate, /// a quantile/distinct sketch, or an unkeyed frequency sketch (UnivMon) - /// over that column's values. + /// over that column's values. `parameters` is the sketch's θ (`None` + /// for an exact aggregate). pub fn per_group_summary_build( input: SchemaRef, family: SummaryFamilyType, + parameters: Option, value: usize, time: Option, groups: Vec, @@ -164,6 +261,7 @@ impl Operator { } crate::capability::validate_summary_kernel( &family, + parameters.as_ref(), &SummaryUpdate::column(ColumnRef::SampleValue), &Default::default(), ) @@ -180,6 +278,7 @@ impl Operator { Ok(Self { kind: Kind::PerGroupSummaryBuild { family, + parameters, value, time, groups, @@ -209,6 +308,7 @@ impl Operator { state, groups, emit_empty_global: false, + parameters: None, }, inputs: vec![input], output: schema(fields), @@ -217,14 +317,25 @@ impl Operator { /// A merge into no groups that outputs one empty state when it has no /// input state: the global SQL aggregate a rollup of a relation's /// grouped states to no keys stands for, whose one row is NULL over no - /// input rows. - pub fn with_emit_empty_global(mut self) -> Result { + /// input rows. `parameters` is the merged states' θ, which the empty + /// state takes: the exported build node's `parameters` (`None` for an + /// exact family). + pub fn with_emit_empty_global( + mut self, + parameters: Option, + ) -> Result { match &mut self.kind { Kind::SummaryMerge { + state, groups, emit_empty_global, - .. - } if groups.is_empty() => *emit_empty_global = true, + parameters: slot, + } if groups.is_empty() => { + // An empty state of this family and θ can be built. + StateColumnUpdater::new(&self.inputs[0].fields[*state].dtype, parameters.as_ref())?; + *emit_empty_global = true; + *slot = parameters; + } _ => { return Err(invalid( "an empty global state needs a merge into no groups", @@ -349,6 +460,7 @@ pub(super) fn execute<'a>( match &operator.kind { Kind::PerGroupSummaryBuild { family, + parameters, value, time, groups, @@ -358,6 +470,7 @@ pub(super) fn execute<'a>( input, PerGroupBuild { family, + parameters: parameters.as_ref(), value: *value, time: *time, groups, @@ -373,6 +486,7 @@ pub(super) fn execute<'a>( .boxed_local()), Kind::PerGroupKeyedSummaryBuild { family, + parameters, items, row_weight, groups, @@ -382,6 +496,7 @@ pub(super) fn execute<'a>( input, KeyedBuild { family, + parameters: parameters.as_ref(), items, row_weight: *row_weight, groups, @@ -396,6 +511,7 @@ pub(super) fn execute<'a>( .boxed_local()), Kind::AllGroupsSummaryBuild { family, + parameters, item, row_weight, groups, @@ -405,6 +521,7 @@ pub(super) fn execute<'a>( input, AllGroupsBuild { family, + parameters: parameters.as_ref(), item: *item, row_weight: *row_weight, groups, @@ -439,6 +556,7 @@ pub(super) fn execute_merge<'a>( state, groups, emit_empty_global, + parameters, } = &operator.kind else { unreachable!() @@ -447,7 +565,7 @@ pub(super) fn execute_merge<'a>( input, *state, groups, - *emit_empty_global, + emit_empty_global.then_some(parameters.as_ref()), &input_schema, &output, &context, @@ -495,6 +613,8 @@ fn grouped_states( /// A `PerGroupSummaryBuild`'s parameters. struct PerGroupBuild<'a> { family: &'a SummaryFamilyType, + /// θ (`None` for an exact family). + parameters: Option<&'a SketchParams>, value: usize, time: Option, groups: &'a [usize], @@ -517,7 +637,7 @@ async fn build_per_group_summary( .filter .map(|filter| filter.at_evaluation_time(context)); let mut table = GroupTable::new(build.groups.to_vec()); - let mut updater = StateColumnUpdater::new(build.family)?; + let mut updater = StateColumnUpdater::new(build.family, build.parameters)?; let mut work = Cooperative::new(context); let mut workspace = Workspace::new(context)?; // PromQL aggregation of an empty vector produces no sample. @@ -592,6 +712,8 @@ async fn build_per_group_summary( /// A `PerGroupKeyedSummaryBuild`'s parameters. struct KeyedBuild<'a> { family: &'a SummaryFamilyType, + /// θ. + parameters: Option<&'a SketchParams>, items: &'a [usize], row_weight: RowWeight, groups: &'a [usize], @@ -655,7 +777,7 @@ async fn build_per_group_keyed_summary( .filter .map(|filter| filter.at_evaluation_time(context)); let mut table = GroupTable::new(build.groups.to_vec()); - let mut updater = StateColumnUpdater::new(build.family)?; + let mut updater = StateColumnUpdater::new(build.family, build.parameters)?; let mut work = Cooperative::new(context); let mut workspace = Workspace::new(context)?; let mut items = Vec::with_capacity(build.items.len()); @@ -697,6 +819,8 @@ async fn build_per_group_keyed_summary( /// An `AllGroupsSummaryBuild`'s parameters. struct AllGroupsBuild<'a> { family: &'a SummaryFamilyType, + /// θ: the Hydra grid's sizes. + parameters: Option<&'a SketchParams>, item: usize, row_weight: RowWeight, groups: &'a [usize], @@ -718,7 +842,8 @@ async fn build_all_groups_summary( .map(|filter| filter.at_evaluation_time(context)); use crate::summary_kernels::HydraCmsGroup; use crate::AggregateCore; - let mut grid = crate::factory::create_hydra_cms(build.family).map_err(Error::Invalid)?; + let mut grid = + crate::factory::create_hydra_cms(build.family, build.parameters).map_err(Error::Invalid)?; let grid_bytes = grid.approx_memory_bytes(); let mut workspace = Workspace::new(context)?; workspace.set_bytes(grid_bytes)?; @@ -795,12 +920,13 @@ fn state_indices(batch: &Batch, state: usize) -> Result<(&StateColumn, &UInt32Ar } /// Merge the states of each group across all input rows; with -/// `emit_empty_global`, one empty state when there is no input state. +/// `empty_global` (`Some` of the states' θ), one empty state when there is +/// no input state. async fn merge_summary( mut input: Input<'_, Batch>, state_column: usize, groups: &[usize], - emit_empty_global: bool, + empty_global: Option>, input_schema: &SchemaRef, output: &SchemaRef, context: &RunContext, @@ -828,9 +954,9 @@ async fn merge_summary( ); } // A global SQL aggregate's one row over no input rows: one empty state. - if entries.is_empty() && emit_empty_global { + if let (true, Some(parameters)) = (entries.is_empty(), empty_global) { table.add_empty_group()?; - let mut updater = StateColumnUpdater::new(family)?; + let mut updater = StateColumnUpdater::new(family, parameters)?; updater.add_group()?; return grouped_states(&table, updater.seal(), input_schema, output); } @@ -1198,6 +1324,13 @@ fn keyed_evaluation( .as_any() .downcast_ref::() .ok_or_else(|| invalid("weighted frequency typed state required"))?; + // The heap capacity is the state's θ. + let (_, _, capacity) = summary.shape(); + if k > capacity { + return Err(Error::Invalid(format!( + "top-{k} exceeds the state's heap capacity {capacity}" + ))); + } for items in summary.rows(k) { source_rows.push(row as u32); for ((column, value), field) in item_columns diff --git a/crates/executor/src/operators/unchecked.rs b/crates/executor/src/operators/unchecked.rs index 173dce2d1..82a7e0651 100644 --- a/crates/executor/src/operators/unchecked.rs +++ b/crates/executor/src/operators/unchecked.rs @@ -168,16 +168,25 @@ impl TryFrom for Operator { )?, Kind::PerGroupSummaryBuild { family, + parameters, value, time, groups, filter, } => with_filter( - Operator::per_group_summary_build(input(0)?, family, value, time, groups)?, + Operator::per_group_summary_build( + input(0)?, + family, + parameters, + value, + time, + groups, + )?, filter, )?, Kind::PerGroupKeyedSummaryBuild { family, + parameters, items, row_weight, groups, @@ -186,6 +195,7 @@ impl TryFrom for Operator { Operator::per_group_keyed_summary_build( input(0)?, family, + parameters, items, row_weight, groups, @@ -194,12 +204,20 @@ impl TryFrom for Operator { )?, Kind::AllGroupsSummaryBuild { family, + parameters, item, row_weight, groups, filter, } => with_filter( - Operator::all_groups_summary_build(input(0)?, family, item, row_weight, groups)?, + Operator::all_groups_summary_build( + input(0)?, + family, + parameters, + item, + row_weight, + groups, + )?, filter, )?, Kind::KeyedEvaluation { state, k } => { @@ -209,10 +227,15 @@ impl TryFrom for Operator { state, groups, emit_empty_global, + parameters, } => { let merge = Operator::summary_merge(input(0)?, state, groups)?; if emit_empty_global { - merge.with_emit_empty_global()? + merge.with_emit_empty_global(parameters)? + } else if parameters.is_some() { + return Err(invalid( + "only a merge that outputs an empty global state takes parameters θ", + )); } else { merge } diff --git a/crates/executor/src/physical_planner/mod.rs b/crates/executor/src/physical_planner/mod.rs index 8c9afa66a..04e2b0f4e 100644 --- a/crates/executor/src/physical_planner/mod.rs +++ b/crates/executor/src/physical_planner/mod.rs @@ -485,8 +485,13 @@ fn compile_internal( "per-entity summary requires complete source identity", )); } - crate::capability::validate_summary_kernel(family, update, grouping) - .map_err(Error::Invalid)?; + crate::capability::validate_summary_kernel( + family, + node.parameters.as_ref(), + update, + grouping, + ) + .map_err(Error::Invalid)?; let value = match (frequency_item, &update.weight) { (Some(item), _) => item, (None, SummaryInputExpr::Column(value)) => value, @@ -507,6 +512,7 @@ fn compile_internal( let build = Operator::per_group_summary_build( input, family.clone(), + node.parameters.clone(), value, Some(coordinate), groups, @@ -624,8 +630,9 @@ fn compile_internal( }) = &node.payload { if keys.is_empty() && built_from_relation(&restored[&id]) { + let parameters = build_parameters(id, &nodes, &dependencies); let operator = compile_node(node, &schemas) - .and_then(Operator::with_emit_empty_global) + .and_then(|merge| merge.with_emit_empty_global(parameters)) .map_err(|error| invalid(format!("node {id}: {error}")))?; physical_dag.add(id, inputs, operator)?; continue; @@ -658,6 +665,29 @@ fn compile_internal( Ok(physical_dag) } +/// θ of the summary states node `id` reads: the `parameters` of the build +/// node (`SummaryAgg` or `SummaryTumblingWindowChain`) at or below it, +/// along first inputs. The states a node merges share one θ. `None` for an +/// exact family's build, or when there is no build below. +fn build_parameters( + id: NodeId, + nodes: &BTreeMap, + dependencies: &BTreeMap>, +) -> Option { + let mut below = Some(id); + while let Some(id) = below { + let node = nodes.get(&id)?; + if matches!( + node.payload, + Payload::ASAP(ASAPOp::SummaryAgg { .. } | ASAPOp::SummaryTumblingWindowChain { .. }) + ) { + return node.parameters.clone(); + } + below = first_input(id, dependencies); + } + None +} + /// Whether every summary state `state` merges was built from a relation's /// rows (a SQL table's), where a global aggregate has a row over no input /// row; not from a PromQL vector's, where it has none. @@ -1219,7 +1249,14 @@ fn bind_operation(node: &PhysicalASAPDAGNode, inputs: &[SchemaRef]) -> Result build.with_row_filter(expression(&filter.0, input)?), None => Ok(build), @@ -1286,12 +1323,13 @@ fn bind_operation(node: &PhysicalASAPDAGNode, inputs: &[SchemaRef]) -> Result, update: &planner_types::ir::schema::SummaryUpdate, reduction: &PlannerReduction, grouping: &planner_types::ir::schema::GroupingStrategy, ) -> Result { if grouping != &planner_types::ir::schema::GroupingStrategy::PerSubpopulationInstance { - crate::capability::validate_summary_kernel(family, update, grouping) + crate::capability::validate_summary_kernel(family, parameters, update, grouping) .map_err(Error::Invalid)?; let PlannerReduction::Reduce(keys) = reduction else { return Err(invalid("an all-groups summary requires explicit groups")); @@ -1304,6 +1342,7 @@ fn summary_agg_build( return Operator::all_groups_summary_build( input.clone(), family.clone(), + parameters.cloned(), named_column(input, item)?, row_weight(input, &update.weight)?, groups(input, keys)?, @@ -1315,7 +1354,7 @@ fn summary_agg_build( crate::capability::unit_frequency_column(update), unkeyed_frequency_family(family), ) { - crate::capability::validate_summary_kernel(family, update, grouping) + crate::capability::validate_summary_kernel(family, parameters, update, grouping) .map_err(Error::Invalid)?; let PlannerReduction::Reduce(keys) = reduction else { return Err(invalid( @@ -1325,6 +1364,7 @@ fn summary_agg_build( return Operator::per_group_summary_build( input.clone(), family.clone(), + parameters.cloned(), named_column(input, item)?, input.time_index, groups(input, keys)?, @@ -1363,12 +1403,14 @@ fn summary_agg_build( return Operator::per_group_keyed_summary_build( input.clone(), family.clone(), + parameters.cloned(), items, row_weight(input, &update.weight)?, groups(input, keys)?, ); } - crate::capability::validate_summary_kernel(family, update, grouping).map_err(Error::Invalid)?; + crate::capability::validate_summary_kernel(family, parameters, update, grouping) + .map_err(Error::Invalid)?; let SummaryInputExpr::Column(column) = &update.weight else { return Err(invalid( "summary update expression must be projected to a column", @@ -1382,6 +1424,7 @@ fn summary_agg_build( Operator::per_group_summary_build( input.clone(), family.clone(), + parameters.cloned(), named_column(input, column)?, input.time_index, groups(input, keys)?, @@ -1604,6 +1647,86 @@ pub mod test_support { use planner_types::physical::{Lifetime, StorageMedium}; use std::collections::HashMap; + /// θ for executor tests of a per-instance sketch of `algorithm`: values + /// its kernel builds, small enough for fast tests. `None` for an + /// algorithm with no kernel. + pub fn test_parameters( + algorithm: &planner_types::ir::schema::SketchAlgorithm, + ) -> Option { + use planner_types::ir::schema::{SketchAlgorithm as A, SketchParams as P}; + Some(match algorithm { + A::UnivMon => P::UnivMon { + heap_size: 32, + sketch_rows: 3, + sketch_cols: 1024, + layers: 8, + distinct_failure_probability: None, + }, + A::Kll => P::Kll { + k: 200, + failure_probability: None, + }, + A::Cms => P::Cms { + width: 64, + depth: 3, + }, + A::Hll => P::Hll { + precision: 12, + relative_error: None, + }, + A::DDSketch => P::DDSketch { alpha: 0.01 }, + A::CmsWithHeap => P::CmsWithHeap { + width: 64, + depth: 3, + heap_size: 10, + }, + A::Kmv => P::Kmv { + k: 1024, + relative_error: None, + }, + A::CountSketch => P::CountSketch { + width: 64, + depth: 3, + }, + A::CountSketchWithHeap => P::CountSketchWithHeap { + width: 64, + depth: 3, + heap_size: 10, + }, + _ => return None, + }) + } + + /// Set θ on every summary build node of `dag` (`SummaryAgg`, + /// `SummaryTumblingWindowChain`) whose family is a sketch, to + /// `parameters` of its family; exact builds keep `None`. For tests that + /// export hand-built plans, whose nodes carry no θ. + pub fn set_build_parameters( + dag: &mut PhysicalASAPDAG, + parameters: impl Fn(&FieldDataType) -> Option, + ) { + for node in &mut dag.nodes { + if let Payload::ASAP( + ASAPOp::SummaryAgg { family, .. } + | ASAPOp::SummaryTumblingWindowChain { family, .. }, + ) = &node.payload + { + if let FieldDataType::Sketch(..) = family { + node.parameters = parameters(family); + } + } + } + } + + /// [`set_build_parameters`] with [`test_parameters`] of each sketch's + /// algorithm. + pub fn set_test_parameters(dag: &mut PhysicalASAPDAG) { + set_build_parameters(dag, |family| match family { + FieldDataType::Sketch(algorithm, _) => test_parameters(algorithm), + _ => None, + }); + } + /// The materialized views at `nodes`, in that order: each node that is /// not a view is marked materialized at ingestion time, and it and every node below it /// run at ingestion time (their outputs and outgoing edges); then every @@ -1655,6 +1778,7 @@ pub mod test_support { edge.data_state_mut().timing = ExecutionTiming::IngestionTime; } } + planner_types::test_support::set_summary_coverage_from_memo(dag); let views = dag .set_retention(&HashMap::new(), &HashMap::new()) .map_err(|error| invalid(error.to_string()))?; diff --git a/crates/executor/src/summary_kernels/factory.rs b/crates/executor/src/summary_kernels/factory.rs index 890a16fb8..1ab53b37e 100644 --- a/crates/executor/src/summary_kernels/factory.rs +++ b/crates/executor/src/summary_kernels/factory.rs @@ -7,7 +7,7 @@ use crate::summary_kernels::{ }; use crate::AggregateCore; use planner_types::ir::schema::{ - FieldDataType as SummaryFamilyType, SketchAlgorithm, SketchKind, SketchParams, + FieldDataType as SummaryFamilyType, SketchAlgorithm, SketchParams, }; /// Generate the clone-based `AccumulatorUpdater` methods for updaters whose @@ -483,7 +483,7 @@ pub fn create_planner_accumulator( return Err("window-weighted summaries require typed DAG binding; integer heap updaters cannot consume rates".into()); } - crate::capability::validate_summary_kernel(family, input, grouping)?; + crate::capability::validate_summary_kernel(family, parameters, input, grouping)?; use planner_types::ir::schema::GroupingStrategy; if grouping != &GroupingStrategy::PerSubpopulationInstance { return Err( @@ -532,8 +532,7 @@ pub fn create_planner_accumulator( } (SketchAlgorithm::CmsWithHeap | SketchAlgorithm::CountSketchWithHeap, _) => { let (algorithm, width, depth, capacity) = - WeightedFrequency::configuration(&SketchKind::new(*kind, params.clone())) - .map_err(|e| e.to_string())?; + WeightedFrequency::configuration(kind, params).map_err(|e| e.to_string())?; Box::new( WeightedFrequencyUpdater::new(algorithm, width, depth, capacity) .map_err(|e| e.to_string())?, @@ -578,10 +577,15 @@ pub fn create_planner_accumulator( } /// Construct the one state a `SharedMultiSubpopulation { kind: HydraCms }` -/// family shares across its groups. The update contract is checked with the -/// plan by `validate_summary_kernel`. -pub fn create_hydra_cms(family: &SummaryFamilyType) -> Result { - let (shared_rows, shared_columns, width, depth) = crate::capability::hydra_cms_shape(family)?; +/// family shares across its groups, sized by its θ `parameters` +/// (`SketchParams::Hydra(HydraParams::HydraCms { .. })`). The update +/// contract is checked with the plan by `validate_summary_kernel`. +pub fn create_hydra_cms( + family: &SummaryFamilyType, + parameters: Option<&SketchParams>, +) -> Result { + let (shared_rows, shared_columns, width, depth) = + crate::capability::hydra_cms_shape(family, parameters)?; HydraCms::new(shared_rows, shared_columns, width, depth).map_err(|e| e.to_string()) } @@ -680,6 +684,155 @@ impl AccumulatorUpdater for UnivMonUpdater { } } +#[cfg(test)] +mod planner_parameter_regression { + use super::*; + use planner_types::ir::schema::{SummaryInputExpr, SummaryUpdate}; + + // Planner width is the bucket count; depth is the independent hash-row count. + #[test] + fn planner_sketch_dimensions_are_not_transposed() { + for (algorithm, params) in [ + ( + SketchAlgorithm::Cms, + SketchParams::Cms { + width: 128, + depth: 3, + }, + ), + ( + SketchAlgorithm::CountSketch, + SketchParams::CountSketch { + width: 128, + depth: 3, + }, + ), + ( + SketchAlgorithm::CmsWithHeap, + SketchParams::CmsWithHeap { + width: 128, + depth: 3, + heap_size: 8, + }, + ), + ( + SketchAlgorithm::CountSketchWithHeap, + SketchParams::CountSketchWithHeap { + width: 128, + depth: 3, + heap_size: 8, + }, + ), + ] { + let family = SummaryFamilyType::Sketch(algorithm, Default::default()); + let update = SummaryUpdate { + item: Some(SummaryInputExpr::Column( + planner_types::ir::scalar::ColumnRef::Named("host".into()), + )), + weight: SummaryInputExpr::Constant(1.0), + weight_domain: Default::default(), + }; + let state = + create_planner_accumulator(&family, Some(¶ms), &update, &Default::default()) + .unwrap() + .snapshot_accumulator(); + let dims = match algorithm { + SketchAlgorithm::Cms => { + let s = state + .as_any() + .downcast_ref::() + .unwrap(); + (s.inner.rows(), s.inner.cols()) + } + SketchAlgorithm::CountSketch + | SketchAlgorithm::CmsWithHeap + | SketchAlgorithm::CountSketchWithHeap => { + let (width, depth, _) = state + .as_any() + .downcast_ref::() + .unwrap() + .shape(); + (depth, width) + } + _ => unreachable!(), + }; + assert_eq!(dims, (3, 128), "{algorithm:?}"); + } + } + + // A sketch kernel needs θ of its own algorithm; an exact one takes none. + #[test] + fn sketch_kernels_need_their_own_parameters() { + use planner_types::ir::schema::{ExactKind, ExactParams}; + let update = SummaryUpdate::column(planner_types::ir::scalar::ColumnRef::SampleValue); + let kll = SummaryFamilyType::Sketch(SketchAlgorithm::Kll, Default::default()); + let k200 = SketchParams::Kll { + k: 200, + failure_probability: None, + }; + assert!( + create_planner_accumulator(&kll, Some(&k200), &update, &Default::default()).is_ok() + ); + assert!(create_planner_accumulator(&kll, None, &update, &Default::default()).is_err()); + let dd = SketchParams::DDSketch { alpha: 0.01 }; + assert!(create_planner_accumulator(&kll, Some(&dd), &update, &Default::default()).is_err()); + let sum = SummaryFamilyType::ExactAggregate(ExactKind::Sum, ExactParams::Sum); + assert!(create_planner_accumulator(&sum, None, &update, &Default::default()).is_ok()); + assert!( + create_planner_accumulator(&sum, Some(&k200), &update, &Default::default()).is_err() + ); + } + + // HydraCms builds one shared grid shaped by its Hydra θ; it is never a + // per-group updater, and its family must be Count-Min. + #[test] + fn hydra_cms_builds_one_shared_grid() { + use planner_types::ir::schema::{ + GroupingStrategy, HydraKind, HydraParams, NonNegativeWeightProof, WeightDomain, + }; + let grouping = GroupingStrategy::SharedMultiSubpopulation { + kind: HydraKind::HydraCms, + }; + let family = SummaryFamilyType::Sketch(SketchAlgorithm::Cms, grouping.clone()); + let hydra = SketchParams::Hydra(HydraParams::HydraCms { + width: 128, + depth: 3, + shared_rows: 5, + shared_columns: 32, + }); + assert_eq!( + create_hydra_cms(&family, Some(&hydra)).unwrap().shape(), + (5, 32, 128, 3) + ); + // θ of the per-group Count-Min, not the Hydra grid's. + let cms = SketchParams::Cms { + width: 128, + depth: 3, + }; + assert!(create_hydra_cms(&family, Some(&cms)).is_err()); + assert!(create_hydra_cms(&family, None).is_err()); + let kll = SummaryFamilyType::Sketch(SketchAlgorithm::Kll, grouping.clone()); + assert!(create_hydra_cms(&kll, Some(&hydra)).is_err()); + let update = SummaryUpdate { + item: Some(SummaryInputExpr::Column( + planner_types::ir::scalar::ColumnRef::Named("host".into()), + )), + weight: SummaryInputExpr::Constant(1.0), + weight_domain: WeightDomain::NonNegative { + proof: NonNegativeWeightProof::UnitCount, + }, + }; + assert!(crate::capability::validate_summary_kernel( + &family, + Some(&hydra), + &update, + &grouping + ) + .is_ok()); + assert!(create_planner_accumulator(&family, Some(&hydra), &update, &grouping).is_err()); + } +} + #[cfg(test)] mod ddsketch_domain { use super::*; diff --git a/crates/executor/src/summary_kernels/state_column_updater.rs b/crates/executor/src/summary_kernels/state_column_updater.rs index 358abed27..de02f50a3 100644 --- a/crates/executor/src/summary_kernels/state_column_updater.rs +++ b/crates/executor/src/summary_kernels/state_column_updater.rs @@ -2,25 +2,30 @@ //! group: the code the `SummaryAgg` build runs row by row, exposed so a //! runtime that ingests sample by sample builds the same states. use super::exact::ScalarState; -use super::factory::{create_hydra_cms, AccumulatorUpdater}; +use super::factory::{create_hydra_cms, create_planner_accumulator, AccumulatorUpdater}; use super::hydra_cms::{hydra_group_name, hydra_item, hydra_weight, HydraCms, HydraCmsGroup}; use super::weighted_frequency::WeightedFrequency; use crate::values::{StateColumn, Value}; use crate::{AggregateCore, Error}; - -use planner_types::ir::schema::{ExactKind, FieldDataType, GroupingStrategy}; +use planner_types::ir::scalar::ColumnRef; +use planner_types::ir::schema::{ + ExactKind, FieldDataType, GroupingStrategy, SketchParams, SummaryUpdate, +}; use std::sync::Arc; /// The open summary states of one family, one per group, updated one sample /// at a time; sealed into an immutable [`StateColumn`]. /// -/// Built from the plan's family and parameters with [`Self::new`]. Groups are +/// Built from the plan's family and θ with [`Self::new`]. Groups are /// numbered `0..group_count()` in the order [`Self::add_group`] (or /// [`Self::add_group_with_key`]) adds them; the caller keeps its own map from /// a group's labels to its number. pub struct StateColumnUpdater { - /// The family and parameters of every state. + /// The family of every state. family: FieldDataType, + /// θ of every state: the `parameters` of the exported build node + /// (`PhysicalASAPDAGNode::parameters`); `None` for an exact family. + parameters: Option, /// The states, by group. states: GroupStates, /// For a counter (rate, increase), each group's latest sample time: @@ -47,7 +52,6 @@ enum GroupStates { Sketches(Vec>), /// A keyed frequency sketch (Count-Min or Count Sketch with a heap) per /// group. - #[allow(dead_code)] // Used again in #1207 PR 6, with θ from the exported node. Keyed { /// An empty state of the family's shape, cloned for each new group /// (boxed: much larger than the other kinds' fields). @@ -70,12 +74,13 @@ fn operator_error(message: impl ToString) -> Error { } impl StateColumnUpdater { - /// An updater with no group for `family`, the family and parameters of a - /// `SummaryAgg` the plan builds one state per group of: an exact - /// accumulator, a value sketch (KLL, DDSketch, HLL, KMV, UnivMon), a - /// keyed frequency sketch with a heap, or a HydraCms grid shared by all - /// groups. Bad input for any other family. - pub fn new(family: &FieldDataType) -> Result { + /// An updater with no group for `family` and θ `parameters`, the family + /// and the exported node's `parameters` of a `SummaryAgg` the plan + /// builds one state per group of: an exact accumulator (no θ), a value + /// sketch (KLL, DDSketch, HLL, KMV, UnivMon), a keyed frequency sketch + /// with a heap, or a HydraCms grid shared by all groups. Bad input for + /// any other family, or θ its kernel does not build. + pub fn new(family: &FieldDataType, parameters: Option<&SketchParams>) -> Result { crate::values::validate_family(family)?; let hydra = matches!( family, @@ -83,26 +88,38 @@ impl StateColumnUpdater { ); let states = match family { FieldDataType::Sketch(..) if hydra => GroupStates::Hydra { - grid: Box::new(create_hydra_cms(family).map_err(Error::Invalid)?), + grid: Box::new(create_hydra_cms(family, parameters).map_err(Error::Invalid)?), names: Vec::new(), }, + FieldDataType::ExactAggregate(..) if parameters.is_some() => { + return Err(Error::Invalid( + "an exact summary has no parameters θ".into(), + )) + } FieldDataType::ExactAggregate(..) => GroupStates::Exact { empty: ScalarState::empty(family).map_err(Error::Invalid)?, states: Vec::new(), }, - FieldDataType::Sketch(algorithm, grouping) - if grouping == &GroupingStrategy::PerSubpopulationInstance - && matches!( - algorithm, - planner_types::ir::schema::SketchAlgorithm::CmsWithHeap - | planner_types::ir::schema::SketchAlgorithm::CountSketchWithHeap - ) => - { - todo!("#1207 PR 6: the keyed frequency state of θ, which the exported node carries") + FieldDataType::Sketch( + algorithm @ (planner_types::ir::schema::SketchAlgorithm::CmsWithHeap + | planner_types::ir::schema::SketchAlgorithm::CountSketchWithHeap), + GroupingStrategy::PerSubpopulationInstance, + ) => { + let Some(parameters) = parameters else { + return Err(Error::Invalid(format!( + "a {algorithm:?} summary needs its parameters θ" + ))); + }; + let (algorithm, width, depth, capacity) = + WeightedFrequency::configuration(algorithm, parameters)?; + GroupStates::Keyed { + empty: Box::new(WeightedFrequency::new(algorithm, width, depth, capacity)?), + states: Vec::new(), + } } FieldDataType::Sketch(..) => { // Fails for a family with no per-group value kernel. - Self::sketch_updater(family)?; + Self::sketch_updater(family, parameters)?; GroupStates::Sketches(Vec::new()) } _ => return Err(Error::Invalid("unsupported summary family".into())), @@ -118,6 +135,7 @@ impl StateColumnUpdater { }; Ok(Self { family: family.clone(), + parameters: parameters.cloned(), states, latest_sample_ms: counter.then(Vec::new), memory_bytes, @@ -125,9 +143,17 @@ impl StateColumnUpdater { }) } - fn sketch_updater(family: &FieldDataType) -> Result, Error> { - let _ = family; - todo!("#1207 PR 6: build it with `create_planner_accumulator` and θ from the exported node's `parameters`") + fn sketch_updater( + family: &FieldDataType, + parameters: Option<&SketchParams>, + ) -> Result, Error> { + create_planner_accumulator( + family, + parameters, + &SummaryUpdate::column(ColumnRef::SampleValue), + &Default::default(), + ) + .map_err(Error::Invalid) } /// The number of groups. @@ -225,7 +251,7 @@ impl StateColumnUpdater { (states.capacity() - before) * std::mem::size_of::() } GroupStates::Sketches(states) => { - let state = Self::sketch_updater(&self.family)?; + let state = Self::sketch_updater(&self.family, self.parameters.as_ref())?; let bytes = state.memory_usage_bytes(); states.push(state); bytes @@ -354,12 +380,12 @@ impl StateColumnUpdater { /// shards must be disjoint in time: one shard's last sample before the /// other's first. Overlapping shards would count the increase of their /// overlap twice, so they are refused as bad input, as are different - /// families. An operator error when a kernel refuses the merge. On an + /// families or θ. An operator error when a kernel refuses the merge. On an /// error this updater may hold some groups already merged. pub fn merge(&mut self, other: Self) -> Result<(), Error> { - if self.family != other.family { + if self.family != other.family || self.parameters != other.parameters { return Err(Error::Invalid( - "cannot merge different summary families".into(), + "cannot merge different summary families or parameters".into(), )); } if let ( @@ -488,7 +514,7 @@ mod tests { fn exact_memory_is_the_buffer_capacity() { let family = FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum); for groups in [1, 5, 17] { - let mut updater = StateColumnUpdater::new(&family).unwrap(); + let mut updater = StateColumnUpdater::new(&family, None).unwrap(); updater.ensure_groups(groups).unwrap(); let charged = updater.memory_bytes(); let GroupStates::Exact { states, .. } = &updater.states else { diff --git a/crates/executor/src/summary_kernels/weighted_frequency.rs b/crates/executor/src/summary_kernels/weighted_frequency.rs index e0211c327..1ee76bcb3 100644 --- a/crates/executor/src/summary_kernels/weighted_frequency.rs +++ b/crates/executor/src/summary_kernels/weighted_frequency.rs @@ -95,11 +95,14 @@ impl std::fmt::Debug for WeightedFrequency { } impl WeightedFrequency { + /// The algorithm and `(width, depth, capacity)` of a keyed frequency + /// sketch with a heap of `algorithm` whose θ is `parameters`. pub(crate) fn configuration( - kind: &planner_types::ir::schema::SketchKind, + algorithm: &planner_types::ir::schema::SketchAlgorithm, + parameters: &planner_types::ir::schema::SketchParams, ) -> Result<(FrequencyAlgorithm, usize, usize, usize), Error> { use planner_types::ir::schema::{SketchAlgorithm as A, SketchParams as P}; - let (algorithm, width, depth, capacity) = match (kind.algorithm(), kind.params()) { + let (algorithm, width, depth, capacity) = match (algorithm, parameters) { ( A::CmsWithHeap, P::CmsWithHeap { diff --git a/crates/executor/src/supported_summaries.rs b/crates/executor/src/supported_summaries.rs index 973ce0cfb..1a42d563c 100644 --- a/crates/executor/src/supported_summaries.rs +++ b/crates/executor/src/supported_summaries.rs @@ -29,7 +29,8 @@ mod tests { use planner_types::ir::scalar::ColumnRef; use planner_types::ir::schema::{ DataType, FamilyCapability, Field, FieldDataType, GroupingStrategy, Schema, - SketchAlgorithm, SketchStatistic, StatisticKind, SummaryInputExpr, SummaryUpdate, + SketchAlgorithm, SketchParams, SketchStatistic, StatisticKind, SummaryInputExpr, + SummaryUpdate, }; use planner_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode}; @@ -55,6 +56,60 @@ mod tests { StatisticKind::TopK, ]; + /// Valid θ of `algorithm`. + fn params(algorithm: &SketchAlgorithm) -> SketchParams { + use SketchAlgorithm as A; + match algorithm { + A::UnivMon => SketchParams::UnivMon { + heap_size: 32, + sketch_rows: 3, + sketch_cols: 1024, + layers: 8, + distinct_failure_probability: None, + }, + A::Kll => SketchParams::Kll { + k: 200, + failure_probability: None, + }, + A::Cms => SketchParams::Cms { + width: 64, + depth: 3, + }, + A::Hll => SketchParams::Hll { + precision: 12, + relative_error: None, + }, + A::DDSketch => SketchParams::DDSketch { alpha: 0.01 }, + A::CmsWithHeap => SketchParams::CmsWithHeap { + width: 64, + depth: 3, + heap_size: 10, + }, + A::Kmv => SketchParams::Kmv { + k: 1024, + relative_error: None, + }, + A::CountSketch => SketchParams::CountSketch { + width: 64, + depth: 3, + }, + A::CountSketchWithHeap => SketchParams::CountSketchWithHeap { + width: 64, + depth: 3, + heap_size: 10, + }, + other => unreachable!("{other:?} is not probed"), + } + } + + /// θ of a per-instance `family`. + fn family_params(family: &FieldDataType) -> Option { + match family { + FieldDataType::Sketch(algorithm, _) => Some(params(algorithm)), + _ => None, + } + } + fn per_instance(algorithm: &SketchAlgorithm) -> FieldDataType { FieldDataType::Sketch(*algorithm, GroupingStrategy::PerSubpopulationInstance) } @@ -92,8 +147,9 @@ mod tests { item: Some(SummaryInputExpr::Column(ColumnRef::Named("item".into()))), ..value.clone() }; - create_planner_accumulator(family, None, &value, grouping).is_ok() - || create_planner_accumulator(family, None, &keyed, grouping).is_ok() + let theta = family_params(family); + create_planner_accumulator(family, theta.as_ref(), &value, grouping).is_ok() + || create_planner_accumulator(family, theta.as_ref(), &keyed, grouping).is_ok() } /// Whether the executor answers `readout` from `family`'s state: top-k @@ -127,8 +183,9 @@ mod tests { item: Some(SummaryInputExpr::Column(ColumnRef::Named("item".into()))), ..value.clone() }; - let Ok(mut updater) = create_planner_accumulator(family, None, &value, grouping) - .or_else(|_| create_planner_accumulator(family, None, &keyed, grouping)) + let theta = family_params(family); + let Ok(mut updater) = create_planner_accumulator(family, theta.as_ref(), &value, grouping) + .or_else(|_| create_planner_accumulator(family, theta.as_ref(), &keyed, grouping)) else { return false; }; diff --git a/crates/executor/src/values/batch.rs b/crates/executor/src/values/batch.rs index d9b9374c2..1f631ff1f 100644 --- a/crates/executor/src/values/batch.rs +++ b/crates/executor/src/values/batch.rs @@ -851,7 +851,7 @@ mod tests { use crate::StateColumnUpdater; use planner_types::ir::schema::{ExactKind, ExactParams}; let family = FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum); - let mut updater = StateColumnUpdater::new(&family).unwrap(); + let mut updater = StateColumnUpdater::new(&family, None).unwrap(); updater.ensure_groups(2).unwrap(); updater.update(0, 1.5, 0).unwrap(); updater.update(1, 4.0, 0).unwrap(); @@ -912,10 +912,10 @@ mod tests { states: states.clone(), }; let other_family = Arc::new( - StateColumnUpdater::new(&FieldDataType::ExactAggregate( - ExactKind::Count, - ExactParams::Count, - )) + StateColumnUpdater::new( + &FieldDataType::ExactAggregate(ExactKind::Count, ExactParams::Count), + None, + ) .unwrap() .seal(), ); @@ -993,7 +993,7 @@ mod tests { use crate::StateColumnUpdater; use planner_types::ir::schema::{ExactKind, ExactParams}; let family = FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum); - let mut updater = StateColumnUpdater::new(&family).unwrap(); + let mut updater = StateColumnUpdater::new(&family, None).unwrap(); updater.ensure_groups(10).unwrap(); for group in 0..10 { updater.update(group, f64::from(group), 0).unwrap(); @@ -1065,7 +1065,7 @@ mod tests { // States [10, 20, 30] referenced as rows [2, 0, 2]; states [1, 2] // referenced as rows [1]. let batch = |sums: &[f64], rows: Vec| { - let mut updater = StateColumnUpdater::new(&family).unwrap(); + let mut updater = StateColumnUpdater::new(&family, None).unwrap(); updater.ensure_groups(sums.len()).unwrap(); for (group, &sum) in sums.iter().enumerate() { updater.update(group as u32, sum, 0).unwrap(); @@ -1120,12 +1120,21 @@ mod tests { fn state_batch(count: u32, exact: bool) -> Batch { use crate::StateColumnUpdater; use planner_types::ir::schema::{ExactKind, ExactParams, SketchAlgorithm}; - let family = if exact { - FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum) + let (family, parameters) = if exact { + ( + FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), + None, + ) } else { - FieldDataType::Sketch(SketchAlgorithm::Kll, Default::default()) + ( + FieldDataType::Sketch(SketchAlgorithm::Kll, Default::default()), + Some(planner_types::ir::schema::SketchParams::Kll { + k: 200, + failure_probability: None, + }), + ) }; - let mut updater = StateColumnUpdater::new(&family).unwrap(); + let mut updater = StateColumnUpdater::new(&family, parameters.as_ref()).unwrap(); updater.ensure_groups(count as usize).unwrap(); for group in 0..count { updater.update(group, f64::from(group), 0).unwrap(); diff --git a/crates/executor/src/values/mod.rs b/crates/executor/src/values/mod.rs index 40c273375..efdacfdf6 100644 --- a/crates/executor/src/values/mod.rs +++ b/crates/executor/src/values/mod.rs @@ -206,38 +206,55 @@ impl Value { pub(crate) use crate::capability::validate_native_family as validate_family; +/// Whether `state` is a state of `family`: the kernel's state type of the +/// family's algorithm and population layout. θ is not part of a family, so +/// a state's shape is its own; a merge refuses states of different shapes. pub(super) fn validate_state( family: &SummaryFamilyType, state: &dyn AggregateCore, ) -> Result<(), Error> { - use crate::summary_kernels::exact::ExactAccumulator; - + use crate::summary_kernels::weighted_frequency::{FrequencyAlgorithm, WeightedFrequency}; + use crate::summary_kernels::{ + count_min_sketch::CountMinSketchAccumulator, datasketches_kll::DatasketchesKLLAccumulator, + dd_sketch::DDSketchAccumulator, exact::ExactAccumulator, hll_sketch::HllSketchAccumulator, + kmv::KmvAccumulator, univmon::UnivMonAccumulator, + }; + use planner_types::ir::schema::SketchAlgorithm as A; validate_family(family)?; + let state = state.as_any(); + let frequency = |algorithm| { + state + .downcast_ref::() + .is_some_and(|s| s.algorithm() == algorithm && s.shape().2 > 0) + }; let valid = match family { SummaryFamilyType::Sketch(_, grouping) if grouping != &planner_types::ir::schema::GroupingStrategy::PerSubpopulationInstance => { - let shape = crate::capability::hydra_cms_shape(family).map_err(Error::Invalid)?; - state - .as_any() - .downcast_ref::() - .is_some_and(|s| s.grid.shape() == shape) + state.is::() } SummaryFamilyType::ExactAggregate(..) => state - .as_any() .downcast_ref::() .is_some_and(|s| s.family() == family && !s.is_keyed()), - SummaryFamilyType::Sketch(..) => { - todo!("#1207 PR 6: compare the state with θ, which the exported node carries") - } + SummaryFamilyType::Sketch(algorithm, _) => match algorithm { + A::CmsWithHeap => frequency(FrequencyAlgorithm::Cms), + A::CountSketchWithHeap => frequency(FrequencyAlgorithm::CountSketch), + A::Kll => state.is::(), + A::DDSketch => state.is::(), + A::Hll => state.is::(), + A::Kmv => state.is::(), + A::UnivMon => state.is::(), + A::Cms => state.is::(), + _ => false, + }, _ => false, }; if valid { Ok(()) } else { Err(Error::Invalid( - "state payload differs from declared family, parameters or population layout".into(), + "state payload differs from declared family or population layout".into(), )) } } @@ -286,20 +303,22 @@ mod weighted_state_tests { use crate::summary_kernels::weighted_frequency::{FrequencyAlgorithm, WeightedFrequency}; use planner_types::ir::schema::SketchAlgorithm; - // A state cannot acquire a different family or shape merely by relabeling its batch. + // A state cannot acquire a different family merely by relabeling its + // batch; states of different shapes (θ) do not merge. #[test] - fn weighted_state_family_and_shape_must_match() { + fn weighted_state_family_must_match_and_shapes_must_merge() { let cms = SummaryFamilyType::Sketch(SketchAlgorithm::CmsWithHeap, Default::default()); let cs = SummaryFamilyType::Sketch(SketchAlgorithm::CountSketchWithHeap, Default::default()); let state = WeightedFrequency::new(FrequencyAlgorithm::CountSketch, 32, 5, 8).unwrap(); assert!(validate_state(&cs, &state).is_ok()); assert!(validate_state(&cms, &state).is_err()); + // A Count Sketch without a heap is not a heap family's state. + let no_heap = WeightedFrequency::new(FrequencyAlgorithm::CountSketch, 32, 5, 0).unwrap(); + assert!(validate_state(&cs, &no_heap).is_err()); let wrong_shape = WeightedFrequency::new(FrequencyAlgorithm::CountSketch, 64, 5, 8).unwrap(); - assert!(validate_state(&cs, &wrong_shape).is_err()); - let even_depth = - SummaryFamilyType::Sketch(SketchAlgorithm::CountSketchWithHeap, Default::default()); - assert!(validate_family(&even_depth).is_err()); + assert!(validate_state(&cs, &wrong_shape).is_ok()); + assert!(state.merge(&wrong_shape).is_err()); } } diff --git a/crates/executor/tests/blocking_resources.rs b/crates/executor/tests/blocking_resources.rs index d72749462..b1e6fa091 100644 --- a/crates/executor/tests/blocking_resources.rs +++ b/crates/executor/tests/blocking_resources.rs @@ -185,7 +185,7 @@ fn cooperative_sort_preserves_ties_across_chunks() { // The integrated weighted-summary path obeys the same cooperative cancellation contract. #[test] fn weighted_summary_build_yields_within_a_batch() { - use planner_types::ir::schema::SketchAlgorithm; + use planner_types::ir::schema::{SketchAlgorithm, SketchParams}; let input = Arc::new(planner_types::ir::schema::Schema { unique_keys: vec![], @@ -223,6 +223,11 @@ fn weighted_summary_build_yields_within_a_batch() { let operator = Operator::per_group_keyed_summary_build( input, family, + Some(SketchParams::CmsWithHeap { + width: 64, + depth: 3, + heap_size: 8, + }), vec![0], asap_executor::operators::RowWeight::Column(1), vec![], diff --git a/crates/executor/tests/current_series_heap.rs b/crates/executor/tests/current_series_heap.rs index 804a6d4f7..96e728cb1 100644 --- a/crates/executor/tests/current_series_heap.rs +++ b/crates/executor/tests/current_series_heap.rs @@ -159,7 +159,7 @@ fn spatial_heap_ranks_latest_values_in_independent_runs() { SketchAlgorithm::CmsWithHeap, SketchAlgorithm::CountSketchWithHeap, ] { - let _params = match algorithm { + let params = match algorithm { SketchAlgorithm::CmsWithHeap => SketchParams::CmsWithHeap { width: 2048, depth: 5, @@ -175,6 +175,7 @@ fn spatial_heap_ranks_latest_values_in_independent_runs() { let build = Operator::per_group_keyed_summary_build( schema(), family, + Some(params), vec![3], asap_executor::operators::RowWeight::Column(1), vec![2], diff --git a/crates/executor/tests/current_timestamp.rs b/crates/executor/tests/current_timestamp.rs index bbd1c7a40..f3a183f7b 100644 --- a/crates/executor/tests/current_timestamp.rs +++ b/crates/executor/tests/current_timestamp.rs @@ -67,6 +67,7 @@ fn scan_plan(predicates: Vec>) -> PhysicalASAPDAG { output_state: ExecutionDataState::QUERY_ROWS, lifetime: Default::default(), parameters: None, + summary_coverage: None, output_schema: schema(), guarantee: None, retention_ms: None, diff --git a/crates/executor/tests/deployment.rs b/crates/executor/tests/deployment.rs index 9e0cc7bee..171a9d692 100644 --- a/crates/executor/tests/deployment.rs +++ b/crates/executor/tests/deployment.rs @@ -5,20 +5,32 @@ use asap_executor::planner::{ }; use asap_executor::{factory::create_planner_accumulator, AggregateCore}; -fn family(_k: u32) -> FieldDataType { - FieldDataType::Sketch( - SketchAlgorithm::Kll, - GroupingStrategy::PerSubpopulationInstance, +/// A sketch family and its θ, as a build node carries them. +type Family = (FieldDataType, SketchParams); + +/// The kernel of `family` for `update`. +fn accumulator( + family: &Family, + update: &SummaryUpdate, +) -> Result, String> { + create_planner_accumulator(&family.0, Some(&family.1), update, &Default::default()) +} + +fn family(k: u32) -> Family { + ( + FieldDataType::Sketch( + SketchAlgorithm::Kll, + GroupingStrategy::PerSubpopulationInstance, + ), + SketchParams::Kll { + k, + failure_probability: None, + }, ) } fn build(values: &[f64]) -> Box { - let mut operator = create_planner_accumulator( - &family(512), - None, - &SummaryUpdate::column(ColumnRef::SampleValue), - &Default::default(), - ) - .unwrap(); + let mut operator = + accumulator(&family(512), &SummaryUpdate::column(ColumnRef::SampleValue)).unwrap(); for (at, value) in values.iter().enumerate() { operator.validate_single_input(*value).unwrap(); operator.update_single(*value, at as i64).unwrap(); @@ -49,12 +61,7 @@ fn raw_partial_and_fully_precomputed_use_the_same_kernels() { // A compiler must reject invalid physical parameters before starting execution. #[test] fn invalid_kll_parameters_are_rejected_at_binding() { - let result = create_planner_accumulator( - &family(0), - None, - &SummaryUpdate::column(ColumnRef::SampleValue), - &Default::default(), - ); + let result = accumulator(&family(0), &SummaryUpdate::column(ColumnRef::SampleValue)); assert!(result.is_err()); } @@ -63,11 +70,17 @@ fn invalid_kll_parameters_are_rejected_at_binding() { #[test] fn native_count_sketch_dimensions_are_not_packed_wire_dimensions() { use asap_executor::planner::ir::schema::SummaryInputExpr; - let family = FieldDataType::Sketch(SketchAlgorithm::CountSketchWithHeap, Default::default()); + let family = ( + FieldDataType::Sketch(SketchAlgorithm::CountSketchWithHeap, Default::default()), + SketchParams::CountSketchWithHeap { + width: 1200, + depth: 55, + heap_size: 3, + }, + ); let mut update = SummaryUpdate::column(ColumnRef::SampleValue); update.item = Some(SummaryInputExpr::Column(ColumnRef::Named("host".into()))); - let mut operator = - create_planner_accumulator(&family, None, &update, &Default::default()).unwrap(); + let mut operator = accumulator(&family, &update).unwrap(); operator .update_keyed(&["a".to_string()], 7.0, 1000) .unwrap(); @@ -84,8 +97,11 @@ fn native_count_sketch_dimensions_are_not_packed_wire_dimensions() { ); } -fn plain_count_sketch(_width: u32, _depth: u32) -> FieldDataType { - FieldDataType::Sketch(SketchAlgorithm::CountSketch, Default::default()) +fn plain_count_sketch(width: u32, depth: u32) -> Family { + ( + FieldDataType::Sketch(SketchAlgorithm::CountSketch, Default::default()), + SketchParams::CountSketch { width, depth }, + ) } fn host(i: usize) -> [String; 1] { @@ -94,7 +110,7 @@ fn host(i: usize) -> [String; 1] { /// Build a plain CountSketch over unit events, one per key index. fn build_plain_count_sketch( - family: &FieldDataType, + family: &Family, events: impl IntoIterator, ) -> Box { build_weighted_frequency(family, events.into_iter().map(|i| (i, 1.0))) @@ -102,14 +118,13 @@ fn build_plain_count_sketch( /// Build a keyed frequency summary over `(key index, weight)` updates. fn build_weighted_frequency( - family: &FieldDataType, + family: &Family, updates: impl IntoIterator, ) -> Box { use asap_executor::planner::ir::schema::SummaryInputExpr; let mut update = SummaryUpdate::column(ColumnRef::SampleValue); update.item = Some(SummaryInputExpr::Column(ColumnRef::Named("host".into()))); - let mut operator = - create_planner_accumulator(family, None, &update, &Default::default()).unwrap(); + let mut operator = accumulator(family, &update).unwrap(); for (at, (i, weight)) in updates.into_iter().enumerate() { operator.validate_single_input(weight).unwrap(); operator.update_keyed(&host(i), weight, at as i64).unwrap(); @@ -137,7 +152,7 @@ fn zipf_events(keys: usize) -> Vec { /// Decode, for `keys` keys, the column each row of `family` (a keyed /// frequency sketch of `width` x `depth`) puts the key in, and assert every /// pair of rows shares a column for few keys, as independent rows do. -fn assert_rows_hash_independently(family: &FieldDataType, width: usize, depth: usize, keys: usize) { +fn assert_rows_hash_independently(family: &Family, width: usize, depth: usize, keys: usize) { // Key `base + p` of a batch carries weight 3^p, so a cell holds a sum // whose balanced-ternary digits name the keys that hit it (CountSketch // signs give digits -1 or 1, Count-Min only 1). @@ -195,13 +210,8 @@ fn deep_plain_count_sketch_rows_hash_independently() { assert_rows_hash_independently(&plain_count_sketch(30_000, 15), 30_000, 15, 2_000); } -fn heap_sketch( - algorithm: SketchAlgorithm, - width: u32, - depth: u32, - heap_size: u32, -) -> FieldDataType { - let _params = match algorithm { +fn heap_sketch(algorithm: SketchAlgorithm, width: u32, depth: u32, heap_size: u32) -> Family { + let params = match algorithm { SketchAlgorithm::CmsWithHeap => SketchParams::CmsWithHeap { width, depth, @@ -213,7 +223,7 @@ fn heap_sketch( heap_size, }, }; - FieldDataType::Sketch(algorithm, Default::default()) + (FieldDataType::Sketch(algorithm, Default::default()), params) } // The factory's top-k sketches use the same per-row-seeded kernel: past 20 @@ -253,7 +263,7 @@ fn heap_sketch_states_load_as_dag_summary_values() { let schema = std::sync::Arc::new(Schema { fields: vec![Field { name: "state".into(), - dtype: family.clone(), + dtype: family.0.clone(), nullable: false, }], unique_keys: vec![], @@ -261,7 +271,7 @@ fn heap_sketch_states_load_as_dag_summary_values() { time_index: None, }); let row = vec![Value::Summary { - family, + family: family.0, state: std::sync::Arc::from(state), }]; assert!(Batch::try_new(schema, vec![row]).is_ok()); @@ -318,7 +328,7 @@ fn count_min_heap_rejects_negative_inputs() { let mut update = SummaryUpdate::column(ColumnRef::SampleValue); update.item = Some(SummaryInputExpr::Column(ColumnRef::Named("host".into()))); let family = heap_sketch(SketchAlgorithm::CmsWithHeap, 64, 3, 4); - let operator = create_planner_accumulator(&family, None, &update, &Default::default()).unwrap(); + let operator = accumulator(&family, &update).unwrap(); assert!(operator.validate_single_input(-1.0).is_err()); assert!(operator.validate_single_input(0.5).is_ok()); } @@ -366,7 +376,7 @@ fn plain_count_sketch_merge_equals_union_build() { for i in 0..500 { assert_eq!(plain_estimate(&*merged, i), plain_estimate(&*union, i)); } - // A different shape is a different family and does not merge. + // A different shape (θ) does not merge. assert!(merged .merge_with(&*build_plain_count_sketch( &plain_count_sketch(1_001, 5), @@ -384,8 +394,7 @@ fn keyed_update_overflow_is_an_error_not_a_panic() { let mut update = SummaryUpdate::column(ColumnRef::SampleValue); update.item = Some(SummaryInputExpr::Column(ColumnRef::Named("host".into()))); let family = plain_count_sketch(64, 3); - let mut operator = - create_planner_accumulator(&family, None, &update, &Default::default()).unwrap(); + let mut operator = accumulator(&family, &update).unwrap(); operator.validate_single_input(f64::MAX).unwrap(); operator.update_keyed(&host(0), f64::MAX, 0).unwrap(); let error = operator.update_keyed(&host(0), f64::MAX, 1).unwrap_err(); diff --git a/crates/executor/tests/exact_state_columns.rs b/crates/executor/tests/exact_state_columns.rs index c9fa8aaec..ce9a12e14 100644 --- a/crates/executor/tests/exact_state_columns.rs +++ b/crates/executor/tests/exact_state_columns.rs @@ -95,7 +95,8 @@ fn exact_records_merge_and_evaluate_as_kernel_objects_do() { ] { let family = FieldDataType::ExactAggregate(kind, params); let build = || { - Operator::per_group_summary_build(input(), family.clone(), 2, Some(1), vec![0]).unwrap() + Operator::per_group_summary_build(input(), family.clone(), None, 2, Some(1), vec![0]) + .unwrap() }; let tumbling_windows = [0, 1_000].map(|start| run(build(), vec![tumbling_window_input(start)]).remove(0)); diff --git a/crates/executor/tests/hydra_cms_execution.rs b/crates/executor/tests/hydra_cms_execution.rs index a71188167..cad89458c 100644 --- a/crates/executor/tests/hydra_cms_execution.rs +++ b/crates/executor/tests/hydra_cms_execution.rs @@ -42,6 +42,16 @@ fn grouping() -> GroupingStrategy { } } +/// θ of the HydraCms grid. +fn hydra_params() -> SketchParams { + SketchParams::Hydra(HydraParams::HydraCms { + width: WIDTH, + depth: 3, + shared_rows: 3, + shared_columns: SHARED_COLUMNS, + }) +} + fn family() -> FieldDataType { FieldDataType::Sketch(SketchAlgorithm::Cms, grouping()) } @@ -62,6 +72,7 @@ fn node(id: usize, payload: Payload, output: Schema) -> PhysicalASAPDAGNode { output_state: ExecutionDataState::QUERY_ROWS, lifetime: Lifetime::None, parameters: None, + summary_coverage: None, output_schema: output, guarantee: None, retention_ms: None, @@ -105,7 +116,7 @@ fn unit_count() -> SummaryUpdate { /// The planner contract: item = the counted column, weight = unit count, /// reduction = the group-by keys. fn summary_agg(update: SummaryUpdate, filter: Option>) -> PhysicalASAPDAGNode { - node( + let mut build = node( 1, Payload::ASAP(ASAPOp::SummaryAgg { child: 0, @@ -123,7 +134,9 @@ fn summary_agg(update: SummaryUpdate, filter: Option>) -> Physi nullable: false, }, ]), - ) + ); + build.parameters = Some(hydra_params()); + build } fn estimate(id: usize, query: SketchStatistic, dtype: DataType) -> PhysicalASAPDAGNode { @@ -375,6 +388,7 @@ fn row_weight_serializes_as_one_field_and_rejects_constants_above_i32() { Operator::all_groups_summary_build( input_schema(), family(), + Some(hydra_params()), 1, RowWeight::Constant(std::num::NonZeroU32::new(weight).unwrap()), vec![0], @@ -405,8 +419,13 @@ fn constant_weight_above_i32_is_refused_at_validation() { }, ..unit_count() }; - let error = asap_executor::capability::validate_summary_kernel(&family(), &update, &grouping()) - .expect_err("validation accepted a constant above i32::MAX"); + let error = asap_executor::capability::validate_summary_kernel( + &family(), + Some(&hydra_params()), + &update, + &grouping(), + ) + .expect_err("validation accepted a constant above i32::MAX"); assert!( error.contains("HydraCms") && error.contains("i32::MAX"), "{error}" diff --git a/crates/executor/tests/keyed_summary_row_weight.rs b/crates/executor/tests/keyed_summary_row_weight.rs index 0ab6e245d..11f46a77c 100644 --- a/crates/executor/tests/keyed_summary_row_weight.rs +++ b/crates/executor/tests/keyed_summary_row_weight.rs @@ -30,6 +30,15 @@ const ROWS: [(&str, &str); 6] = [ ("batch", "export"), ]; +/// θ of `heap_family`. +fn heap_params() -> Option { + Some(SketchParams::CmsWithHeap { + width: 1024, + depth: 3, + heap_size: 8, + }) +} + fn heap_family() -> FieldDataType { FieldDataType::Sketch( SketchAlgorithm::CmsWithHeap, @@ -153,7 +162,8 @@ fn query_time_count_heap_binds_a_constant_row_weight() { }), output_state: ExecutionDataState::QUERY_ROWS, lifetime: Lifetime::None, - parameters: None, + parameters: heap_params(), + summary_coverage: None, output_schema: (*schema(vec![ Field::plain("job", DataType::Utf8, false), Field { @@ -184,6 +194,7 @@ fn keyed_build(input: &SchemaRef, row_weight: RowWeight) -> Operator { Operator::per_group_keyed_summary_build( input.clone(), heap_family(), + heap_params(), vec![1], row_weight, vec![0], @@ -247,6 +258,11 @@ fn int64_weight_column_equals_float64_column() { let signed = Operator::per_group_keyed_summary_build( int64.clone(), count_sketch, + Some(SketchParams::CountSketchWithHeap { + width: 1024, + depth: 3, + heap_size: 8, + }), vec![1], RowWeight::Column(2), vec![0], @@ -304,6 +320,7 @@ fn int64_weight_column_equals_float64_column() { let error = Operator::per_group_keyed_summary_build( text, heap_family(), + heap_params(), vec![1], RowWeight::Column(2), vec![0], diff --git a/crates/executor/tests/physical_dag.rs b/crates/executor/tests/physical_dag.rs index 996aecc58..8359d7fdb 100644 --- a/crates/executor/tests/physical_dag.rs +++ b/crates/executor/tests/physical_dag.rs @@ -163,7 +163,8 @@ fn summary_construction_merge_and_evaluation_at_both_phases() { .map(|v| Batch::try_new(schema.clone(), vec![vec![Value::Float64(v as f64)]]).unwrap()) .collect(); let family = FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum); - let build = Operator::per_group_summary_build(schema.clone(), family, 0, None, vec![]).unwrap(); + let build = + Operator::per_group_summary_build(schema.clone(), family, None, 0, None, vec![]).unwrap(); let state = build.schema(); let mut dag = PhysicalDAG::default(); dag.add(0, vec![], Operator::source(schema, batches).unwrap()) @@ -322,6 +323,7 @@ fn binding_rejects_unsupported_operations() { assert!(Operator::per_group_summary_build( schema.clone(), FieldDataType::ExactAggregate(ExactKind::Rate, ExactParams::Rate), + None, 0, None, vec![] @@ -330,6 +332,7 @@ fn binding_rejects_unsupported_operations() { let sum = Operator::per_group_summary_build( schema.clone(), FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), + None, 0, None, vec![], @@ -356,7 +359,18 @@ fn kll_raw_partial_and_precomputed_are_native_dags() { SketchAlgorithm::Kll, GroupingStrategy::PerSubpopulationInstance, ); - let build = Operator::per_group_summary_build(input.clone(), family, 0, None, vec![]).unwrap(); + let build = Operator::per_group_summary_build( + input.clone(), + family, + Some(planner_types::ir::schema::SketchParams::Kll { + k: 512, + failure_probability: None, + }), + 0, + None, + vec![], + ) + .unwrap(); let state = build.schema(); let build_range = |start: u32, end: u32| { let mut dag = PhysicalDAG::default(); @@ -533,6 +547,7 @@ fn bind_physical_asap_dag_before_execution() { output_state: ExecutionDataState::QUERY_ROWS, lifetime: Default::default(), parameters: None, + summary_coverage: None, output_schema: (*schema).clone(), guarantee: None, retention_ms: None, @@ -610,6 +625,7 @@ fn empty_exact_count_is_an_integer_state_evaluation() { let build = Operator::per_group_summary_build( input.clone(), FieldDataType::ExactAggregate(ExactKind::Count, ExactParams::Count), + None, 0, None, vec![], @@ -694,6 +710,7 @@ fn planner_semijoin_sort_limit_contract_at_both_phases() { output_state: ExecutionDataState::QUERY_ROWS, lifetime: Default::default(), parameters: None, + summary_coverage: None, guarantee: None, retention_ms: None, }; @@ -1018,6 +1035,19 @@ fn assert_weighted_rate_topk(count_sketch: bool) { let build = Operator::per_group_keyed_summary_build( rates.schema(), family, + Some(if count_sketch { + planner_types::ir::schema::SketchParams::CountSketchWithHeap { + width: 4096, + depth: 5, + heap_size: 8, + } + } else { + planner_types::ir::schema::SketchParams::CmsWithHeap { + width: 4096, + depth: 5, + heap_size: 8, + } + }), vec![0], asap_executor::operators::RowWeight::Column(3), vec![1], @@ -1116,6 +1146,7 @@ fn grouped_temporal_schema_compiles_and_executes_topk() { output_state: ExecutionDataState::QUERY_ROWS, lifetime: Default::default(), parameters: None, + summary_coverage: None, output_schema: (*input).clone(), guarantee: None, retention_ms: None, @@ -1207,6 +1238,7 @@ fn certified_pruning_rejects_missing_authoritative_values_after_recovery() { output_state: ExecutionDataState::QUERY_ROWS, lifetime: Default::default(), parameters: None, + summary_coverage: None, guarantee: None, retention_ms: None, payload: PhysicalASAPOperatorPayload::NonASAP(planner_types::ir::NonASAPOp::Join { @@ -1313,6 +1345,7 @@ fn compiled_ingestion_binary_preserves_alignment_and_rejects_missing_updates() { output_state: ExecutionDataState::INGESTION_ROWS, lifetime: Default::default(), parameters: None, + summary_coverage: None, guarantee: None, retention_ms: None, payload: PhysicalASAPOperatorPayload::NonASAP(planner_types::ir::NonASAPOp::BinaryOp { diff --git a/crates/executor/tests/physical_semantics.rs b/crates/executor/tests/physical_semantics.rs index 5d002b2bd..291c5788b 100644 --- a/crates/executor/tests/physical_semantics.rs +++ b/crates/executor/tests/physical_semantics.rs @@ -386,6 +386,7 @@ fn global_extrema_bind_with_planner_derived_schema() { output_state: ExecutionDataState::QUERY_ROWS, lifetime: Default::default(), parameters: None, + summary_coverage: None, output_schema: (*output).clone(), guarantee: None, retention_ms: None, @@ -574,12 +575,19 @@ fn kll_partial_merge_and_multiple_evaluations_preserve_population() { dag.add( id + 3, vec![id], - Operator::per_group_summary_build(input.clone(), family.clone(), 0, None, vec![]) - .unwrap(), + Operator::per_group_summary_build( + input.clone(), + family.clone(), + kll_512(), + 0, + None, + vec![], + ) + .unwrap(), ) .unwrap(); } - let state = Operator::per_group_summary_build(input, family, 0, None, vec![]) + let state = Operator::per_group_summary_build(input, family, kll_512(), 0, None, vec![]) .unwrap() .schema(); dag.add(6, vec![3, 4], Operator::union(state.clone(), 2).unwrap()) @@ -652,7 +660,18 @@ fn compacted_kll_tumbling_windows_merge_within_the_calibrated_rank_error() { let input = schema(&[("v", DataType::Float64, false)]); let family = FieldDataType::Sketch(SketchAlgorithm::Kll, Default::default()); let build = || { - Operator::per_group_summary_build(input.clone(), family.clone(), 0, None, vec![]).unwrap() + Operator::per_group_summary_build( + input.clone(), + family.clone(), + Some(planner_types::ir::schema::SketchParams::Kll { + k: K, + failure_probability: None, + }), + 0, + None, + vec![], + ) + .unwrap() }; let state = build().schema(); let mut dag = PhysicalDAG::default(); @@ -736,7 +755,7 @@ fn counter_tumbling_windows_merge_in_any_order() { ]); let family = FieldDataType::ExactAggregate(ExactKind::Increase, ExactParams::Increase); let build = || { - Operator::per_group_summary_build(input.clone(), family.clone(), 0, Some(1), vec![]) + Operator::per_group_summary_build(input.clone(), family.clone(), None, 0, Some(1), vec![]) .unwrap() }; let state = build().schema(); @@ -826,6 +845,7 @@ fn empty_exact_summary_extrema_agree_with_ordinary_aggregation() { let build = Operator::per_group_summary_build( input.clone(), FieldDataType::ExactAggregate(kind, params), + None, 0, None, vec![], @@ -905,6 +925,14 @@ fn printed(rows: &[Vec]) -> Vec { rows } +/// θ of a KLL with k = 512. +fn kll_512() -> Option { + Some(planner_types::ir::schema::SketchParams::Kll { + k: 512, + failure_probability: None, + }) +} + fn keep() -> Option> { Some(Predicate(ScalarExpr::Column(2))) } @@ -913,12 +941,21 @@ fn planner_node( payload: PhysicalASAPOperatorPayload, fields: &[(&str, FieldDataType, bool)], ) -> PhysicalASAPDAGNode { + // A sketch build carries its θ. + let parameters = match &payload { + PhysicalASAPOperatorPayload::ASAP(planner_types::ir::ASAPOp::SummaryAgg { + family: FieldDataType::Sketch(algorithm, _), + .. + }) => asap_executor::physical_planner::test_support::test_parameters(algorithm), + _ => None, + }; PhysicalASAPDAGNode { id: 1, payload, output_state: planner_types::ir::properties::ExecutionDataState::QUERY_ROWS, lifetime: Default::default(), - parameters: None, + parameters, + summary_coverage: None, output_schema: planner_types::ir::schema::Schema::new( fields .iter() @@ -1166,6 +1203,7 @@ fn exact_aggregate_node( output_state: ExecutionDataState::QUERY_ROWS, lifetime: Default::default(), parameters: None, + summary_coverage: None, output_schema: (*schema(&[(name, dtype, false)])).clone(), guarantee: None, retention_ms: None, diff --git a/crates/executor/tests/plan_properties.rs b/crates/executor/tests/plan_properties.rs index 5b39368fc..967ba3197 100644 --- a/crates/executor/tests/plan_properties.rs +++ b/crates/executor/tests/plan_properties.rs @@ -51,6 +51,7 @@ fn sorted_scan(schema: &Schema, identity: &Source) -> PhysicalASAPDAG { output_state: ExecutionDataState::QUERY_ROWS, lifetime: Default::default(), parameters: None, + summary_coverage: None, output_schema: schema.clone(), guarantee: None, retention_ms: None, @@ -129,6 +130,55 @@ fn blocking_inputs_over_a_raw_data_slot_are_finite() { assert_eq!(opens.load(Ordering::SeqCst), 1); } +/// A KLL kernel accepts `k` only up to the 26 602 `asap_sketchlib` caps it at. +#[test] +fn kll_kernel_rejects_a_k_above_the_library_cap() { + use asap_executor::capability::validate_summary_kernel; + use planner_types::ir::scalar::ColumnRef; + use planner_types::ir::schema::{ + GroupingStrategy, SketchAlgorithm, SketchParams, SummaryInputExpr, SummaryUpdate, + }; + let grouping = GroupingStrategy::default(); + let update = SummaryUpdate { + item: None, + weight: SummaryInputExpr::Column(ColumnRef::SampleValue), + weight_domain: Default::default(), + }; + let kll = FieldDataType::Sketch(SketchAlgorithm::Kll, grouping.clone()); + let params = |k| SketchParams::Kll { + k, + failure_probability: None, + }; + assert!(validate_summary_kernel(&kll, Some(¶ms(26_602)), &update, &grouping).is_ok()); + assert!(validate_summary_kernel(&kll, Some(¶ms(26_603)), &update, &grouping).is_err()); +} + +/// A plain CountSketch reads the median of its rows, so plan validation and +/// kernel construction both require a positive odd depth. +#[test] +fn count_sketch_kernel_requires_an_odd_depth() { + use asap_executor::{capability::validate_summary_kernel, factory::create_planner_accumulator}; + use planner_types::ir::scalar::ColumnRef; + use planner_types::ir::schema::{ + GroupingStrategy, SketchAlgorithm, SketchParams, SummaryInputExpr, SummaryUpdate, + }; + let grouping = GroupingStrategy::default(); + let mut update = SummaryUpdate::column(ColumnRef::SampleValue); + update.item = Some(SummaryInputExpr::Column(ColumnRef::Named("host".into()))); + let family = FieldDataType::Sketch(SketchAlgorithm::CountSketch, grouping.clone()); + for (depth, valid) in [(5, true), (4, false)] { + let params = SketchParams::CountSketch { width: 64, depth }; + assert_eq!( + validate_summary_kernel(&family, Some(¶ms), &update, &grouping).is_ok(), + valid + ); + assert_eq!( + create_planner_accumulator(&family, Some(¶ms), &update, &grouping).is_ok(), + valid + ); + } +} + // Kernel support must not be mistaken for executable native state/evaluation support. #[test] fn summary_capability_levels_are_distinct() { @@ -147,7 +197,13 @@ fn summary_capability_levels_are_distinct() { weight: planner_types::ir::schema::SummaryInputExpr::Constant(1.0), weight_domain: Default::default(), }; - assert!(validate_summary_kernel(&cms, &update, &grouping).is_ok()); + let cms_params = planner_types::ir::schema::SketchParams::Cms { + width: 64, + depth: 4, + }; + assert!(validate_summary_kernel(&cms, Some(&cms_params), &update, &grouping).is_ok()); + // A sketch kernel needs its θ. + assert!(validate_summary_kernel(&cms, None, &update, &grouping).is_err()); // Stored Count-Min state reads only its bare count natively. assert!(validate_native_family(&cms).is_ok()); let bare_count = SketchStatistic::PointCount { diff --git a/crates/executor/tests/precompute_population.rs b/crates/executor/tests/precompute_population.rs index abc87ef26..719d49ac5 100644 --- a/crates/executor/tests/precompute_population.rs +++ b/crates/executor/tests/precompute_population.rs @@ -51,6 +51,7 @@ fn view_over_samples( output_state, lifetime: Default::default(), parameters: None, + summary_coverage: None, output_schema, guarantee: None, retention_ms: None, @@ -141,6 +142,7 @@ fn try_state_dag( planner_types::physical::StorageMedium::Memory, ), parameters: None, + summary_coverage: None, output_schema: logical_schema(family.clone()), guarantee: None, retention_ms: None, @@ -155,6 +157,7 @@ fn try_state_dag( output_state: ExecutionDataState::INGESTION_ROWS, lifetime: Default::default(), parameters: None, + summary_coverage: None, output_schema: logical_schema(FieldDataType::Plain(DataType::Float64)), guarantee: None, retention_ms: None, @@ -175,6 +178,7 @@ fn try_state_dag( planner_types::physical::StorageMedium::Memory, ), parameters: None, + summary_coverage: None, output_schema: logical_schema(target), guarantee: None, retention_ms: None, @@ -193,6 +197,7 @@ fn try_state_dag( roots: vec![root], scalar_roots: vec![], }; + asap_executor::physical_planner::test_support::set_test_parameters(&mut dag); let views = views_at(&mut dag, &[0])?; compile(&dag, &views, &[root as u64]) } diff --git a/crates/executor/tests/promql_binary.rs b/crates/executor/tests/promql_binary.rs index 7f868f2c5..b564a076a 100644 --- a/crates/executor/tests/promql_binary.rs +++ b/crates/executor/tests/promql_binary.rs @@ -81,6 +81,7 @@ fn program_for_bool(operator: BinaryOperator, return_bool: bool) -> Operator { output_state: ExecutionDataState::QUERY_ROWS, lifetime: Default::default(), parameters: None, + summary_coverage: None, output_schema: (*schema).clone(), guarantee: None, retention_ms: None, @@ -415,6 +416,7 @@ fn stored_series_evaluations_support_filters_and_sets() { }, lifetime: Default::default(), parameters: None, + summary_coverage: None, output_schema: match id { 0 | 1 => (*state_schema).clone(), 5 | 6 => sample_schema(), diff --git a/crates/executor/tests/raw_scan.rs b/crates/executor/tests/raw_scan.rs index 671ba113d..f6041e115 100644 --- a/crates/executor/tests/raw_scan.rs +++ b/crates/executor/tests/raw_scan.rs @@ -85,6 +85,7 @@ fn plan( output_state: state, lifetime: Default::default(), parameters: None, + summary_coverage: None, output_schema: (**schema).clone(), guarantee: None, retention_ms: None, @@ -506,6 +507,7 @@ fn windowed_plan(shift_ms: Option) -> PhysicalASAPDAG { output_state: state, lifetime: Default::default(), parameters: None, + summary_coverage: None, output_schema: schema.clone(), guarantee: None, retention_ms: None, @@ -839,6 +841,7 @@ fn windowed_count_plan(shift_ms: Option) -> PhysicalASAPDAG { output_state, lifetime: Default::default(), parameters: None, + summary_coverage: None, output_schema: output_schema.clone(), guarantee: None, retention_ms: None, diff --git a/crates/executor/tests/state_column_updater.rs b/crates/executor/tests/state_column_updater.rs index 52f8ce9fc..4c0377ec4 100644 --- a/crates/executor/tests/state_column_updater.rs +++ b/crates/executor/tests/state_column_updater.rs @@ -1,13 +1,13 @@ //! The per-sample `StateColumnUpdater` builds the same states as the //! batch `SummaryAgg` build and as each family's kernel updated row by row, //! for every family a per-group build supports. - +use asap_executor::factory::create_planner_accumulator; use asap_executor::operators::{Operator, RowWeight}; use asap_executor::plan::PhysicalDAG; use asap_executor::planner::ir::scalar::ColumnRef; use asap_executor::planner::ir::schema::{ - DataType, ExactKind, ExactParams, Field, FieldDataType, GroupingStrategy, HydraKind, Schema, - SketchAlgorithm, SketchParams, SketchStatistic, + DataType, ExactKind, ExactParams, Field, FieldDataType, GroupingStrategy, HydraKind, + HydraParams, Schema, SketchAlgorithm, SketchParams, SketchStatistic, SummaryUpdate, }; use asap_executor::runtime::{Limits, RunContext, Scope}; use asap_executor::summary_kernels::weighted_frequency::WeightedFrequency; @@ -90,16 +90,77 @@ fn input_batch() -> Batch { .unwrap() } -fn sketch(algorithm: SketchAlgorithm, _params: SketchParams) -> FieldDataType { - FieldDataType::Sketch(algorithm, Default::default()) +/// A family and its θ (`None` for an exact family), as a build node +/// carries them. +type Family = (FieldDataType, Option); + +fn sketch(algorithm: SketchAlgorithm, params: SketchParams) -> Family { + ( + FieldDataType::Sketch(algorithm, Default::default()), + Some(params), + ) +} + +fn exact(kind: ExactKind, params: ExactParams) -> Family { + (FieldDataType::ExactAggregate(kind, params), None) } -fn exact(kind: ExactKind, params: ExactParams) -> FieldDataType { - FieldDataType::ExactAggregate(kind, params) +/// An updater of `family` with no group. +fn updater(family: &Family) -> StateColumnUpdater { + StateColumnUpdater::new(&family.0, family.1.as_ref()).unwrap() +} + +/// Every family a per-group value build supports. +fn value_families() -> Vec { + vec![ + exact(ExactKind::Sum, ExactParams::Sum), + exact(ExactKind::Count, ExactParams::Count), + exact(ExactKind::Min, ExactParams::Min), + exact(ExactKind::Max, ExactParams::Max), + exact(ExactKind::Rate, ExactParams::Rate), + exact(ExactKind::Increase, ExactParams::Increase), + sketch( + SketchAlgorithm::Kll, + SketchParams::Kll { + // Above any group's sample count (at most 600): no + // compaction, so the random compaction coin never flips. + k: 1024, + failure_probability: None, + }, + ), + sketch( + SketchAlgorithm::DDSketch, + SketchParams::DDSketch { alpha: 0.01 }, + ), + sketch( + SketchAlgorithm::Hll, + SketchParams::Hll { + precision: 10, + relative_error: None, + }, + ), + sketch( + SketchAlgorithm::Kmv, + SketchParams::Kmv { + k: 32, + relative_error: None, + }, + ), + sketch( + SketchAlgorithm::UnivMon, + SketchParams::UnivMon { + heap_size: 64, + sketch_rows: 5, + sketch_cols: 128, + layers: 4, + distinct_failure_probability: None, + }, + ), + ] } /// Every keyed family a per-group keyed build supports. -fn keyed_families() -> Vec { +fn keyed_families() -> Vec { vec![ sketch( SketchAlgorithm::CmsWithHeap, @@ -231,22 +292,248 @@ fn batch_build(build: Operator) -> BTreeMap { /// One test sample: series, timestamp, value, item. type TestSample = (String, i64, f64, String); +/// How a test feeds one sample to an updater. +type FeedSample<'a> = dyn FnMut(&mut StateColumnUpdater, u32, &TestSample) + 'a; + +/// The per-sample build: one updater, groups numbered as series appear. +/// With `sharded`, the first half of the samples go to one updater and the +/// rest to another, merged at the end as a runtime's shards are. +fn per_sample_build( + family: &Family, + sharded: bool, + mut update: impl FnMut(&mut StateColumnUpdater, u32, &(String, i64, f64, String)), +) -> BTreeMap { + let samples = samples(); + let mut groups: BTreeMap = BTreeMap::new(); + for sample in &samples { + let next = groups.len() as u32; + groups.entry(sample.0.clone()).or_insert(next); + } + let shard = |part: &[(String, i64, f64, String)], update: &mut FeedSample<'_>| { + let mut updater = updater(family); + updater.ensure_groups(groups.len()).unwrap(); + for sample in part { + update(&mut updater, groups[&sample.0], sample); + } + updater + }; + let split = if sharded { + samples.len() / 2 + } else { + samples.len() + }; + let (first, second) = samples.split_at(split); + let mut updater = shard(first, &mut update); + if sharded { + updater.merge(shard(second, &mut update)).unwrap(); + } + assert_eq!(updater.group_count(), groups.len()); + let states = updater.seal(); + groups + .into_iter() + .map(|(series, group)| (series, fingerprint(states.state(group as usize).as_ref()))) + .collect() +} + +/// The pre-columnar reference: each family's kernel, one per series, +/// updated row by row. With `sharded`, two halves merged through the +/// kernel's own merge. +fn kernel_build(family: &Family, keyed: bool, sharded: bool) -> BTreeMap { + let update = if keyed { + SummaryUpdate { + item: Some( + asap_executor::planner::ir::schema::SummaryInputExpr::Column(ColumnRef::Named( + "item".into(), + )), + ), + weight: asap_executor::planner::ir::schema::SummaryInputExpr::Constant(1.0), + weight_domain: Default::default(), + } + } else { + SummaryUpdate::column(ColumnRef::SampleValue) + }; + let samples = samples(); + let split = if sharded { + samples.len() / 2 + } else { + samples.len() + }; + let (first, second) = samples.split_at(split); + let half = |part: &[(String, i64, f64, String)]| { + let mut states = BTreeMap::new(); + for (series, ts, value, item) in part { + let state = states.entry(series.clone()).or_insert_with(|| { + if keyed { + let (algorithm, width, depth, capacity) = match family.1.as_ref().unwrap() { + SketchParams::CmsWithHeap { + width, + depth, + heap_size, + } => ( + asap_executor::summary_kernels::weighted_frequency::FrequencyAlgorithm::Cms, + width, + depth, + heap_size, + ), + SketchParams::CountSketchWithHeap { + width, + depth, + heap_size, + } => ( + asap_executor::summary_kernels::weighted_frequency::FrequencyAlgorithm::CountSketch, + width, + depth, + heap_size, + ), + _ => unreachable!(), + }; + Err(Box::new( + WeightedFrequency::new( + algorithm, + *width as usize, + *depth as usize, + *capacity as usize, + ) + .unwrap(), + )) + } else { + Ok(create_planner_accumulator( + &family.0, + family.1.as_ref(), + &update, + &Default::default(), + ).unwrap()) + } + }); + match state { + Ok(updater) => updater.update_value(&Value::Float64(*value), *ts).unwrap(), + Err(keyed) => keyed + .update(&[Value::Utf8(item.as_str().into())], 1.0) + .unwrap(), + } + } + states + .into_iter() + .map(|(series, state)| { + ( + series, + match state { + Ok(updater) => Arc::::from(updater.into_accumulator()), + Err(keyed) => Arc::new(*keyed) as Arc, + }, + ) + }) + .collect::>() + }; + let (mut merged, second) = (half(first), half(second)); + for (series, state) in second { + let combined = match merged.get(&series) { + Some(existing) => Arc::from(existing.merge_with(state.as_ref()).unwrap()), + None => state, + }; + merged.insert(series, combined); + } + merged + .into_iter() + .map(|(series, state)| (series, fingerprint(state.as_ref()))) + .collect() +} + +// For every value family: the per-sample updater, the batch `SummaryAgg` +// build and the family's own kernel build the same state for every group, +// byte for byte; two updater shards merged equal the kernel's merge of the +// same two halves. +#[test] +fn per_sample_updates_build_the_batch_states_for_every_value_family() { + for family in value_families() { + let batch = batch_build( + Operator::per_group_summary_build( + input_schema(), + family.0.clone(), + family.1.clone(), + 2, + Some(1), + vec![0], + ) + .unwrap(), + ); + let update = + |updater: &mut StateColumnUpdater, group, sample: &(String, i64, f64, String)| { + updater.update(group, sample.2, sample.1).unwrap() + }; + assert_eq!(batch.len(), 7, "{family:?}"); + assert_eq!( + per_sample_build(&family, false, update), + batch, + "{family:?}: updater vs batch build" + ); + assert_eq!( + kernel_build(&family, false, false), + batch, + "{family:?}: kernel vs batch build" + ); + assert_eq!( + per_sample_build(&family, true, update), + kernel_build(&family, false, true), + "{family:?}: merged shards vs kernel merge" + ); + } +} + +// The same for the keyed families (an item and a weight per sample). +#[test] +fn per_sample_updates_build_the_batch_states_for_every_keyed_family() { + for family in keyed_families() { + let batch = batch_build( + Operator::per_group_keyed_summary_build( + input_schema(), + family.0.clone(), + family.1.clone(), + vec![3], + RowWeight::Constant(NonZeroU32::new(1).unwrap()), + vec![0], + ) + .unwrap(), + ); + let update = + |updater: &mut StateColumnUpdater, group, sample: &(String, i64, f64, String)| { + updater + .update_item(group, &[Value::Utf8(sample.3.as_str().into())], 1.0) + .unwrap() + }; + assert_eq!(batch.len(), 7, "{family:?}"); + assert_eq!( + per_sample_build(&family, false, update), + batch, + "{family:?}: updater vs batch build" + ); + assert_eq!( + kernel_build(&family, true, false), + batch, + "{family:?}: kernel vs batch build" + ); + assert_eq!( + per_sample_build(&family, true, update), + kernel_build(&family, true, true), + "{family:?}: merged shards vs kernel merge" + ); + } +} // A counter refuses a sample that is not after its group's previous one, // and a value outside the kernel's domain, leaving the state unchanged. #[test] fn per_sample_updates_keep_each_kernels_input_contract() { - let mut rate = StateColumnUpdater::new(&exact(ExactKind::Rate, ExactParams::Rate)).unwrap(); + let mut rate = updater(&exact(ExactKind::Rate, ExactParams::Rate)); let group = rate.add_group().unwrap(); rate.update(group, 1.0, 100).unwrap(); assert!(rate.update(group, 2.0, 100).is_err()); assert!(rate.update(group, f64::INFINITY, 200).is_err()); rate.update(group, 3.0, 200).unwrap(); - let mut dd = StateColumnUpdater::new(&sketch( + let mut dd = updater(&sketch( SketchAlgorithm::DDSketch, SketchParams::DDSketch { alpha: 0.01 }, - )) - .unwrap(); + )); let group = dd.add_group().unwrap(); // A DDSketch takes zero and negative values, not NaN or ±Inf. dd.update(group, -1.0, 0).unwrap(); @@ -254,7 +541,7 @@ fn per_sample_updates_keep_each_kernels_input_contract() { assert!(dd.update(group, f64::NAN, 0).is_err()); assert!(dd.update(group, f64::INFINITY, 0).is_err()); // A keyed family takes items, a value family takes values. - let mut cms = StateColumnUpdater::new(&keyed_families()[0]).unwrap(); + let mut cms = updater(&keyed_families()[0]); let group = cms.add_group().unwrap(); assert!(cms.update(group, 1.0, 0).is_err()); assert!(dd @@ -272,7 +559,7 @@ fn merging_counter_shards_that_overlap_in_time_is_refused() { exact(ExactKind::Increase, ExactParams::Increase), ] { let shard = |samples: &[(i64, f64)]| { - let mut updater = StateColumnUpdater::new(&family).unwrap(); + let mut updater = updater(&family); let group = updater.add_group().unwrap(); for &(ts, value) in samples { updater.update(group, value, ts).unwrap(); @@ -308,13 +595,13 @@ fn merging_counter_shards_that_overlap_in_time_is_refused() { // A group number the updater never added is bad input, not a panic. #[test] fn an_unknown_group_is_bad_input() { - let mut sum = StateColumnUpdater::new(&exact(ExactKind::Sum, ExactParams::Sum)).unwrap(); + let mut sum = updater(&exact(ExactKind::Sum, ExactParams::Sum)); sum.add_group().unwrap(); assert!(matches!( sum.update(1, 1.0, 0), Err(asap_executor::Error::Invalid(_)) )); - let mut cms = StateColumnUpdater::new(&keyed_families()[0]).unwrap(); + let mut cms = updater(&keyed_families()[0]); assert!(matches!( cms.update_item(0, &[Value::Utf8("a".into())], 1.0), Err(asap_executor::Error::Invalid(_)) @@ -325,12 +612,20 @@ fn an_unknown_group_is_bad_input() { } /// A HydraCms family: one shared grid for all groups. -fn hydra_family() -> FieldDataType { - FieldDataType::Sketch( - SketchAlgorithm::Cms, - GroupingStrategy::SharedMultiSubpopulation { - kind: HydraKind::HydraCms, - }, +fn hydra_family() -> Family { + ( + FieldDataType::Sketch( + SketchAlgorithm::Cms, + GroupingStrategy::SharedMultiSubpopulation { + kind: HydraKind::HydraCms, + }, + ), + Some(SketchParams::Hydra(HydraParams::HydraCms { + width: 64, + depth: 3, + shared_rows: 3, + shared_columns: 16, + })), ) } @@ -340,7 +635,7 @@ fn hydra_per_sample_build(sharded: bool) -> BTreeMap { let family = hydra_family(); let samples = samples(); let shard = |part: &[TestSample]| { - let mut updater = StateColumnUpdater::new(&family).unwrap(); + let mut updater = updater(&family); let mut groups = BTreeMap::::new(); for sample in part { let group = match groups.get(&sample.0) { @@ -363,7 +658,7 @@ fn hydra_per_sample_build(sharded: bool) -> BTreeMap { if sharded { // The second shard numbers groups as the first does: the runtime's // shared map from labels to group numbers. - let mut other = StateColumnUpdater::new(&family).unwrap(); + let mut other = self::updater(&family); let mut by_number = groups .iter() .map(|(series, &group)| (group, series.clone())) @@ -407,7 +702,8 @@ fn per_sample_updates_build_the_batch_hydra_grid() { let batch = batch_build( Operator::all_groups_summary_build( input_schema(), - hydra_family(), + hydra_family().0, + hydra_family().1, 3, RowWeight::Constant(NonZeroU32::new(1).unwrap()), vec![0], @@ -433,7 +729,7 @@ fn per_sample_updates_build_the_batch_hydra_grid() { #[test] fn hydra_updates_keep_the_kernel_input_contract() { let family = hydra_family(); - let mut updater = StateColumnUpdater::new(&family).unwrap(); + let mut updater = updater(&family); assert!(matches!( updater.add_group(), Err(asap_executor::Error::Invalid(_)) @@ -470,7 +766,7 @@ fn hydra_updates_keep_the_kernel_input_contract() { assert_eq!(updater.group_count(), 1); updater.add_group_with_key(&[Value::Null]).unwrap(); assert_eq!(updater.group_count(), 2); - let mut other = StateColumnUpdater::new(&family).unwrap(); + let mut other = self::updater(&family); other .add_group_with_key(&[Value::Utf8("web".into())]) .unwrap(); diff --git a/crates/executor/tests/stored_tumbling_window_memory_limit.rs b/crates/executor/tests/stored_tumbling_window_memory_limit.rs index d02cf1e46..6cc7686fc 100644 --- a/crates/executor/tests/stored_tumbling_window_memory_limit.rs +++ b/crates/executor/tests/stored_tumbling_window_memory_limit.rs @@ -5,7 +5,9 @@ use asap_executor::arrow::array::{DictionaryArray, UInt32Array}; use asap_executor::arrow::datatypes::UInt32Type; use asap_executor::operators::{Operator, SortKey}; use asap_executor::plan::PhysicalDAG; -use asap_executor::planner::ir::schema::{DataType, Field, FieldDataType, Schema, SketchAlgorithm}; +use asap_executor::planner::ir::schema::{ + DataType, Field, FieldDataType, Schema, SketchAlgorithm, SketchParams, +}; use asap_executor::runtime::{Limits, RunContext, Scope}; use asap_executor::values::{Batch, BatchColumn}; use asap_executor::{Error, StateColumnUpdater}; @@ -43,7 +45,14 @@ fn tumbling_window_of(series_count: u32) -> (Arc, Batch) { unique_keys: vec![], closed: false, }); - let mut updater = StateColumnUpdater::new(&family).unwrap(); + let mut updater = StateColumnUpdater::new( + &family, + Some(&SketchParams::Kll { + k: 200, + failure_probability: None, + }), + ) + .unwrap(); updater.ensure_groups(series_count as usize).unwrap(); for series in 0..series_count { for sample in 0..SAMPLES { @@ -317,6 +326,7 @@ fn merge_over_a_stored_tumbling_window( output_state, lifetime: Default::default(), parameters: None, + summary_coverage: None, guarantee: None, retention_ms: None, }; diff --git a/crates/executor/tests/summary_build_memory_limit.rs b/crates/executor/tests/summary_build_memory_limit.rs index d4c54924b..741878140 100644 --- a/crates/executor/tests/summary_build_memory_limit.rs +++ b/crates/executor/tests/summary_build_memory_limit.rs @@ -3,7 +3,9 @@ //! adds them, a bounded number of rows at a time. use asap_executor::operators::Operator; use asap_executor::plan::PhysicalDAG; -use asap_executor::planner::ir::schema::{DataType, Field, FieldDataType, Schema, SketchAlgorithm}; +use asap_executor::planner::ir::schema::{ + DataType, Field, FieldDataType, Schema, SketchAlgorithm, SketchParams, +}; use asap_executor::runtime::{Limits, RunContext, Scope}; use asap_executor::values::{Batch, Value}; use asap_executor::Error; @@ -65,7 +67,18 @@ fn one_large_batch_fails_near_the_memory_limit() { dag.add( 1, vec![0], - Operator::per_group_summary_build(input, family, 1, None, vec![0]).unwrap(), + Operator::per_group_summary_build( + input, + family, + Some(SketchParams::Kll { + k: 200, + failure_probability: None, + }), + 1, + None, + vec![0], + ) + .unwrap(), ) .unwrap(); let context = RunContext::new( diff --git a/crates/executor/tests/summary_projection.rs b/crates/executor/tests/summary_projection.rs index d291eb3b0..b8507dece 100644 --- a/crates/executor/tests/summary_projection.rs +++ b/crates/executor/tests/summary_projection.rs @@ -54,6 +54,7 @@ fn raw_node(id: usize, payload: NonASAPOp) -> PhysicalASAPDAGNode { output_state: ExecutionDataState::QUERY_ROWS, lifetime: Default::default(), parameters: None, + summary_coverage: None, guarantee: None, retention_ms: None, } @@ -124,6 +125,7 @@ fn summary_projection_survives_recovery() { output_state: ExecutionDataState::INGESTION_SUMMARY, lifetime: Default::default(), parameters: None, + summary_coverage: None, guarantee: None, retention_ms: None, }, @@ -152,6 +154,7 @@ fn summary_projection_survives_recovery() { output_state: ExecutionDataState::INGESTION_SUMMARY, lifetime: Default::default(), parameters: None, + summary_coverage: None, guarantee: None, retention_ms: None, }, @@ -261,6 +264,7 @@ fn summary_merge_without_inputs_is_rejected() { output_state: ExecutionDataState::INGESTION_SUMMARY, lifetime: Default::default(), parameters: None, + summary_coverage: None, guarantee: None, retention_ms: None, }], diff --git a/crates/executor/tests/univmon_execution.rs b/crates/executor/tests/univmon_execution.rs index 730d5fb02..a697f5d33 100644 --- a/crates/executor/tests/univmon_execution.rs +++ b/crates/executor/tests/univmon_execution.rs @@ -21,6 +21,17 @@ use planner_types::ir::schema::{ use planner_types::ir::ASAPOp; use std::sync::Arc; +/// θ of `family`. +fn params() -> Option { + Some(planner_types::ir::schema::SketchParams::UnivMon { + heap_size: 64, + sketch_rows: 5, + sketch_cols: 128, + layers: 4, + distinct_failure_probability: None, + }) +} + fn family() -> FieldDataType { FieldDataType::Sketch(SketchAlgorithm::UnivMon, GroupingStrategy::default()) } @@ -92,7 +103,7 @@ fn one_univmon_state_answers_distinct_l2_and_entropy() -> Result<(), Error> { .map(|value| vec![Value::Float64(value)]) .collect(), )?; - let build = Operator::per_group_summary_build(input, family(), 0, None, vec![])?; + let build = Operator::per_group_summary_build(input, family(), params(), 0, None, vec![])?; let [distinct, l2, entropy] = three_statistics(batch, build)?[..] else { unreachable!() }; @@ -135,7 +146,7 @@ fn typed_frequency_keys_preserve_identity() -> Result<(), Error> { vec![Value::Null], ], )?; - let build = Operator::per_group_summary_build(input, family(), 0, None, vec![])?; + let build = Operator::per_group_summary_build(input, family(), params(), 0, None, vec![])?; for (found, expected) in three_statistics(batch, build)? .into_iter() @@ -179,7 +190,8 @@ fn planner_unit_frequency_update_binds_as_a_value_frequency_build() -> Result<() }), output_state: ExecutionDataState::QUERY_ROWS, lifetime: Default::default(), - parameters: None, + parameters: params(), + summary_coverage: None, output_schema: (*state).clone(), guarantee: None, retention_ms: None, diff --git a/crates/executor/tests/weighted_topk_binding.rs b/crates/executor/tests/weighted_topk_binding.rs index 4e878920a..f3b3cac0f 100644 --- a/crates/executor/tests/weighted_topk_binding.rs +++ b/crates/executor/tests/weighted_topk_binding.rs @@ -1,6 +1,7 @@ //! Planner output binds directly to the shared runtime over raw counter samples. use planner_types::ir::schema::*; +// The untyped heap updater must not silently round a Planner rate update. #[test] fn rate_updates_cannot_enter_integer_heap_factory() { let family = FieldDataType::Sketch(SketchAlgorithm::CmsWithHeap, Default::default()); @@ -15,7 +16,11 @@ fn rate_updates_cannot_enter_integer_heap_factory() { }; assert!(asap_executor::factory::create_planner_accumulator( &family, - None, + Some(&SketchParams::CmsWithHeap { + width: 272, + depth: 5, + heap_size: 100, + }), &input, &Default::default() ) From 9c614917e1e2d46380169c9dad5a101dad6a54e7 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Sat, 7 Nov 2026 01:37:24 +0000 Subject: [PATCH 4/8] feat(stage2): realize a selected DAG as the exported physical plan (#1207 PR 6) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stage2AssignmentRealizer builds each selected node, producers first, as its candidate's operator over its selected producers; a tumbling-window layout becomes a SummaryTumblingWindowChain read through a merge of the whole chain. Each node is exported with its candidate's timing, lifetime, θ and its group's summary coverage; update weights the data proves non-negative are confirmed. The realizer keeps only entries and data (original_roots and query_time_results_retained deleted). --- Cargo.lock | 1 + .../stage2-physical-optimization/Cargo.toml | 7 +- .../src/catalog/realize.rs | 298 +++++++++++++++++- .../src/catalog/tests.rs | 226 +++++++++++++ 4 files changed, 515 insertions(+), 17 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 130fc7200..a111d5bdc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -466,6 +466,7 @@ dependencies = [ name = "asap-stage2-physical-optimization" version = "0.1.0" dependencies = [ + "asap-cost", "asap-types", ] diff --git a/crates/stage2-physical-optimization/Cargo.toml b/crates/stage2-physical-optimization/Cargo.toml index b641486e3..5a3587afe 100644 --- a/crates/stage2-physical-optimization/Cargo.toml +++ b/crates/stage2-physical-optimization/Cargo.toml @@ -5,7 +5,10 @@ edition = "2021" # #509 Stage 2, physical ASAP-aware optimization: the physical properties # (timing, lifetime) of each Stage 1 candidate, the physical edges, and the -# PhysicalASAPMemoCandidates Stage 3 selects from (`catalog`). Depends only on -# asap-types, never on Stage 1's crate, plan selection or the executor. +# PhysicalASAPMemoCandidates Stage 3 selects from (`catalog`), and the realizer +# of a selected DAG (`catalog::realize`). Depends only on asap-types and the +# shared cost model, never on Stage 1's crate, plan selection or the executor. [dependencies] asap-types = { path = "../types" } +# The tumbling-window retention of a realized plan (`AssignmentRealizer::realize`). +asap-cost = { path = "../cost" } diff --git a/crates/stage2-physical-optimization/src/catalog/realize.rs b/crates/stage2-physical-optimization/src/catalog/realize.rs index 9abbc6b8b..d1d2f20bd 100644 --- a/crates/stage2-physical-optimization/src/catalog/realize.rs +++ b/crates/stage2-physical-optimization/src/catalog/realize.rs @@ -1,34 +1,302 @@ -//! Realize a selected DAG through Stage 2: each node's physical properties -//! come from its `PhysicalCandidate` (concepts doc, section 3, -//! "Realizer/export"). This validates execution legality, not cost estimates -//! or accuracy targets. +//! Realize a selected DAG through Stage 2: one operator node per selected +//! node, over its selected producers, each exported with its +//! `PhysicalCandidate`'s timing and lifetime, its θ and its memo group's +//! summary coverage (concepts doc, section 3, "Stage 3: baseline selection +//! and realization"). This validates execution legality, not cost +//! estimates or accuracy targets. -use asap_types::ir::QueryRoot; +use std::collections::HashMap; +use std::rc::Rc; + +use asap_types::ir::materialized_view::ViewCoverage; +use asap_types::ir::physical_export::{physical_asap_dag_with_assignments, ExportedNodeAssignment}; +use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode, QueryRoot, TimeRangeKind}; +use asap_types::physical::{ExecutionTiming, Lifetime, PhysicalProperties, PhysicalWorkloadPlan}; use asap_types::stages::{ - AssignmentRealizer, DagOfSelectedCandidates, PhysicalASAPMemoCandidates, - PhysicalCandidateError, RealizedPlan, + AssignmentRealizer, DagOfSelectedCandidates, MemoGroupId, PhysicalASAPMemoCandidates, + PhysicalCandidateError, RealizedPlan, WindowLayout, }; +use asap_types::workload::bounds::confirmed_weights; use asap_types::workload::{DataWorkload, QueryWorkloadEntry}; /// Stage 2's [`AssignmentRealizer`], over the planning context: the -/// workload's original query roots (one per entry of `entries`), its data, -/// and whether the deployment keeps query-time results across -/// evaluations. The facade builds it, passes it to Stage 3, and builds the -/// selected plan with it. +/// workload's entries (one per query, for the range evaluations a +/// tumbling-window chain's retention covers) and its data (which proves +/// summary update weights non-negative). The facade builds it, passes it +/// to Stage 3, and builds the selected plan with it. pub struct Stage2AssignmentRealizer<'a> { - pub original_roots: &'a [QueryRoot], pub entries: &'a [QueryWorkloadEntry], pub data: &'a DataWorkload, - pub query_time_results_retained: bool, } impl AssignmentRealizer for Stage2AssignmentRealizer<'_> { + /// Each selected node, producers first, becomes the operator node of + /// its candidate's expression over the nodes of its selected producers, + /// in input order. A candidate with a tumbling-window layout of width + /// $w$ over a range `TimeRange(W)` becomes a `SummaryTumblingWindowChain` + /// of $W / w$ windows over the range's input, read through a + /// `SummaryMerge` of the whole chain: the chain has the candidate's + /// timing, lifetime and θ; the merge runs at query time unless every + /// consumer of the node runs at ingestion time. A summary build's update + /// weights the data proves non-negative are confirmed + /// (`bounds::confirmed_weights`). The query roots are the workload's + /// roots over groups with each output reference replaced by its node. + /// + /// Err [`PhysicalCandidateError::RejectedAssignment`] when `selected` is + /// not a valid selection over `physical` or the plan fails the physical + /// DAG validator; [`PhysicalCandidateError::InvalidOrUnsupported`] when a + /// candidate's operator does not rebuild over its producers. fn realize( &self, physical: &PhysicalASAPMemoCandidates, selected: &DagOfSelectedCandidates, ) -> Result { - let _ = (physical, selected); - todo!("#1207 PR 6: build the selected DAG, each node at its PhysicalCandidate's properties") + selected + .validate(physical) + .map_err(PhysicalCandidateError::RejectedAssignment)?; + let memo = &physical.logical_candidates.memo; + let node_count = selected.candidate_of_node.len(); + // Per node, per input: the node producing it; and per node, its consumers. + let mut producers: Vec> = selected + .candidate_of_node + .iter() + .map(|&id| { + let logical = + memo.candidate(physical.physical_candidates[id.0].logical_candidate_id); + vec![usize::MAX; logical.expression.operator.children().len()] + }) + .collect(); + let mut consumers: Vec> = vec![Vec::new(); node_count]; + for edge in &selected.edges { + producers[edge.consumer.0][edge.input_index_of_consumer] = edge.producer.0; + consumers[edge.producer.0].push(edge.consumer.0); + } + let mut built: Vec>> = vec![None; node_count]; + let mut assignments: HashMap<*const OperatorNode, ExportedNodeAssignment> = HashMap::new(); + for node in producers_first(&producers) { + let candidate = &physical.physical_candidates[selected.candidate_of_node[node].0]; + let logical = memo.candidate(candidate.logical_candidate_id); + let group = &memo.groups()[candidate.logical_candidate_id.group.0 as usize]; + let children: Vec> = producers[node] + .iter() + .map(|&producer| Rc::clone(built[producer].as_ref().expect("producers first"))) + .collect(); + let operator = with_confirmed_weights( + over_children(&logical.expression.operator, &children)?, + self.data, + ); + let summary_coverage = group + .group_properties + .summary_coverage + .as_ref() + .map(ViewCoverage::of); + let assignment = ExportedNodeAssignment { + physical_properties: candidate.physical_properties, + parameters: logical.logical_properties.parameters().cloned(), + summary_coverage: summary_coverage.clone(), + }; + let operator_node = match logical.logical_properties.window_layout() { + None => new_node(operator)?, + Some(&WindowLayout::Tumbling { width_ms }) => { + let chain = new_node(tumbling_window_chain(operator, width_ms)?)?; + assignments.insert(Rc::as_ptr(&chain), assignment); + let merge_at_ingestion = candidate.physical_properties.timing() + == ExecutionTiming::IngestionTime + && !consumers[node].is_empty() + && consumers[node].iter().all(|&consumer| { + physical.physical_candidates[selected.candidate_of_node[consumer].0] + .physical_properties + .timing() + == ExecutionTiming::IngestionTime + }); + let merge = new_node(Operator::ASAP(ASAPOp::SummaryMerge { + children: vec![chain], + newest_tumbling_windows: None, + group_by: None, + }))?; + assignments.insert( + Rc::as_ptr(&merge), + ExportedNodeAssignment { + physical_properties: PhysicalProperties::new( + match merge_at_ingestion { + true => ExecutionTiming::IngestionTime, + false => ExecutionTiming::QueryTime, + }, + Lifetime::None, + ) + .expect("a result that is not materialized is valid at either time"), + parameters: None, + summary_coverage, + }, + ); + built[node] = Some(merge); + continue; + } + }; + assignments.insert(Rc::as_ptr(&operator_node), assignment); + built[node] = Some(operator_node); + } + let operator_node_of_selected_node: Vec> = built + .into_iter() + .map(|node| node.expect("every node is built")) + .collect(); + let mut outputs = selected + .node_of_output_reference + .iter() + .map(|node| Rc::clone(&operator_node_of_selected_node[node.0])); + let mut next_output = || { + outputs + .next() + .expect("validated: one node per output reference") + }; + let roots: Vec = physical + .query_roots + .iter() + .map(|root| match root { + QueryRoot::Operator(_) => QueryRoot::Operator(next_output()), + QueryRoot::Scalar(expression) => QueryRoot::Scalar( + expression.map_operator_refs(&mut |_: &MemoGroupId| next_output()), + ), + }) + .collect(); + let (dag, _) = physical_asap_dag_with_assignments(&roots, &|node| { + assignments + .get(&Rc::as_ptr(node)) + .cloned() + .expect("every node of the plan is a selected node, a chain or its merge") + }) + .map_err(|error| PhysicalCandidateError::RejectedAssignment(error.to_string()))?; + let range_evaluations: Vec<_> = self + .entries + .iter() + .map(|entry| entry.range_evaluation) + .collect(); + let tumbling_window_retention_ms = + asap_cost::tumbling_window_retention_ms_of_plan(&dag, &range_evaluations); + Ok(RealizedPlan { + plan: PhysicalWorkloadPlan { + id: "selected".into(), + from_logical: "memo".into(), + roots, + dag, + }, + tumbling_window_retention_ms, + operator_node_of_selected_node, + }) + } +} + +/// The nodes of a selected DAG, each after its producers (`producers[n]`: +/// node `n`'s producer per input); the DAG is acyclic (validated). +fn producers_first(producers: &[Vec]) -> Vec { + let mut outstanding: Vec = producers.iter().map(Vec::len).collect(); + let mut consumers_of = vec![Vec::new(); producers.len()]; + for (consumer, inputs) in producers.iter().enumerate() { + for &producer in inputs { + consumers_of[producer].push(consumer); + } + } + let mut ready: Vec = (0..producers.len()) + .filter(|&node| outstanding[node] == 0) + .collect(); + let mut order = Vec::with_capacity(producers.len()); + while let Some(node) = ready.pop() { + order.push(node); + for &consumer in &consumers_of[node] { + outstanding[consumer] -= 1; + if outstanding[consumer] == 0 { + ready.push(consumer); + } + } + } + order +} + +/// `operator` over `children`, its `i`-th child (in [`Operator::children`] +/// order) replaced by `children[i]`. +fn over_children( + operator: &Operator, + children: &[Rc], +) -> Result>, PhysicalCandidateError> { + let in_order: Vec = operator.children().into_iter().copied().collect(); + let mut visited = Vec::with_capacity(in_order.len()); + let _ = operator.map_children(|group| visited.push(*group)); + if visited != in_order || in_order.len() != children.len() { + return Err(PhysicalCandidateError::InvalidOrUnsupported(format!( + "the operator's inputs do not match its selected producers: {operator:?}" + ))); } + let mut next = children.iter(); + Ok(operator.map_children(|_| Rc::clone(next.next().expect("one child per input")))) +} + +/// `operator` with a summary build's update weights confirmed by `data` +/// (`bounds::confirmed_weights`): an open non-negativity requirement the +/// data proves becomes a proof the executor runs. +fn with_confirmed_weights( + mut operator: Operator>, + data: &DataWorkload, +) -> Operator> { + if let Operator::ASAP( + ASAPOp::SummaryAgg { input, .. } | ASAPOp::SummaryTumblingWindowChain { input, .. }, + ) = &mut operator + { + input.weight_domain = confirmed_weights(&input.weight_domain, data); + } + operator +} + +/// The tumbling-window chain of a build `SummaryAgg` over a range +/// `TimeRange(W)`: windows `width_ms` wide, `W / width_ms` of them, over the +/// range's input. +fn tumbling_window_chain( + build: Operator>, + width_ms: u64, +) -> Result>, PhysicalCandidateError> { + let Operator::ASAP(ASAPOp::SummaryAgg { + child, + family, + input, + reduction, + grouping, + filter, + }) = build + else { + return Err("a tumbling-window layout on an operator other than a summary build".into()); + }; + let Operator::NonASAP(NonASAPOp::TimeRange { + range, + kind: TimeRangeKind::Range, + child: below, + }) = &child.operator + else { + return Err("a tumbling-window layout on a build that does not read a range".into()); + }; + let window_ms = u64::try_from(range.as_millis()).unwrap_or(u64::MAX); + if width_ms == 0 || window_ms % width_ms != 0 { + return Err(format!( + "tumbling windows {width_ms} ms wide do not divide the range of {window_ms} ms" + ) + .into()); + } + let tumbling_window_count = u32::try_from(window_ms / width_ms) + .map_err(|_| PhysicalCandidateError::from("too many tumbling windows"))?; + Ok(Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { + child: Rc::clone(below), + family, + input, + reduction, + grouping, + filter, + tumbling_window_width_ms: width_ms, + tumbling_window_count, + })) +} + +/// The operator node of `operator`, its schema derived from its children. +fn new_node( + operator: Operator>, +) -> Result, PhysicalCandidateError> { + OperatorNode::new_shared(operator) + .map_err(|error| PhysicalCandidateError::InvalidOrUnsupported(error.to_string())) } diff --git a/crates/stage2-physical-optimization/src/catalog/tests.rs b/crates/stage2-physical-optimization/src/catalog/tests.rs index d43bc71df..12034d002 100644 --- a/crates/stage2-physical-optimization/src/catalog/tests.rs +++ b/crates/stage2-physical-optimization/src/catalog/tests.rs @@ -721,3 +721,229 @@ fn a_consumer_without_a_producer_for_one_input_cannot_be_built() { vec![properties(QT_NONE), properties(QT_ONE)] ); } + +// ── The realizer ───────────────────────────────────────────────────────── + +/// A KLL state group over a one-minute range of the scan, with an +/// unwindowed build and one of 10 s tumbling windows, read by the p95 +/// readout, the query's root: the physical candidates and the readout's +/// logical candidate. +fn ranged_example() -> (PhysicalASAPMemoCandidates, Vec) { + use asap_types::ir::TimeRangeKind; + let mut memo = Memo::default(); + let scan = group(&mut memo, scan_operator()); + let range = group( + &mut memo, + Operator::NonASAP(NonASAPOp::TimeRange { + range: std::time::Duration::from_secs(60), + kind: TimeRangeKind::Range, + child: scan.group, + }), + ); + let build = Operator::ASAP(ASAPOp::SummaryAgg { + child: range.group, + family: FieldDataType::Sketch(SketchAlgorithm::Kll, GroupingStrategy::default()), + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::PerEntity, + grouping: GroupingStrategy::default(), + filter: None, + }); + let states = memo.find_or_add_group(build.clone()).unwrap(); + let parameters = Some(parameter_list(SketchAlgorithm::Kll)[1].clone()); + let unwindowed = candidate(&mut memo, states, build.clone(), parameters.clone(), None); + let windowed = candidate( + &mut memo, + states, + build, + parameters, + Some(WindowLayout::Tumbling { width_ms: 10_000 }), + ); + let readout = group( + &mut memo, + Operator::ASAP(ASAPOp::SummaryEstimate { + summary_input: states, + query: SketchStatistic::Quantile { q: 0.95 }, + }), + ); + memo.add_query(QueryRoot::Operator(readout.group)); + let edge = |producer, consumer| LogicalEdge { + producer_candidate_id: producer, + consumer_candidate_id: consumer, + input_index_of_consumer: 0, + satisfies_logical_requirement: true, + }; + let edges = vec![ + edge(scan, range), + edge(range, unwindowed), + edge(range, windowed), + edge(unwindowed, readout), + edge(windowed, readout), + ]; + let physical = stage2(&LogicalASAPMemoCandidates { memo, edges }); + (physical, vec![scan, range, unwindowed, windowed, readout]) +} + +/// The selected DAG running `chosen` (each a logical candidate with its +/// physical properties), each reading the next as its one input; the first +/// is the query's output. +fn chain_of( + physical: &PhysicalASAPMemoCandidates, + chosen: &[(CandidateForMemoGroupId, (ExecutionTiming, Lifetime))], +) -> asap_types::stages::DagOfSelectedCandidates { + use asap_types::stages::{DagOfSelectedCandidates, SelectedEdge, SelectedNodeId}; + let candidate_of_node: Vec = chosen + .iter() + .map(|&(logical, state)| { + let id = physical + .physical_candidates + .iter() + .position(|candidate| { + candidate.logical_candidate_id == logical + && candidate.physical_properties == properties(state) + }) + .unwrap_or_else(|| panic!("{logical:?} {state:?} is offered")); + PhysicalCandidateId(id) + }) + .collect(); + DagOfSelectedCandidates { + edges: (1..chosen.len()) + .map(|node| SelectedEdge { + producer: SelectedNodeId(node), + consumer: SelectedNodeId(node - 1), + input_index_of_consumer: 0, + }) + .collect(), + candidate_of_node, + node_of_output_reference: vec![SelectedNodeId(0)], + } +} + +fn realize( + physical: &PhysicalASAPMemoCandidates, + selected: &asap_types::stages::DagOfSelectedCandidates, +) -> Result { + use asap_types::stages::AssignmentRealizer; + let entries = [asap_types::workload::QueryWorkloadEntry { + query: Default::default(), + requirements: Default::default(), + predictability: Default::default(), + recurrence: QueryRecurrence::Repeated(RepeatedDemand::FixedIntervalAt { + interval: RepetitionInterval(10_000), + evaluation_phase: TimestampMs(0), + }), + range_evaluation: None, + }]; + let data = ingesting(); + super::Stage2AssignmentRealizer { + entries: &entries, + data: &data, + } + .realize(physical, selected) +} + +/// An unwindowed selection exports one node per selected node, each with +/// its candidate's timing and lifetime; the build carries its θ and its +/// group's summary coverage, the scan and the readout neither. +#[test] +fn the_realizer_exports_each_node_with_its_candidates_properties_and_theta() { + let (physical, ids) = ranged_example(); + let selected = chain_of( + &physical, + &[ + (ids[4], QT_NONE), + (ids[2], QT_ONE), + (ids[1], QT_NONE), + (ids[0], QT_NONE), + ], + ); + let realized = realize(&physical, &selected).unwrap(); + let dag = &realized.plan.dag; + assert_eq!(dag.nodes.len(), 4); + assert_eq!(realized.operator_node_of_selected_node.len(), 4); + let build = dag + .nodes + .iter() + .find(|node| matches!(node.payload, Operator::ASAP(ASAPOp::SummaryAgg { .. }))) + .unwrap(); + assert_eq!(build.lifetime, Lifetime::OneEvaluation(MEMORY)); + assert_eq!(build.output_state.timing, QueryTime); + assert_eq!( + build.parameters.as_ref(), + Some(¶meter_list(SketchAlgorithm::Kll)[1]) + ); + assert!(build.summary_coverage.is_some()); + for node in dag.nodes.iter().filter(|node| node.id != build.id) { + assert_eq!(node.parameters, None, "{:?}", node.payload); + assert_eq!(node.lifetime, Lifetime::None); + } + assert!(realized.tumbling_window_retention_ms.is_empty()); +} + +/// A windowed build materialized across evaluations at ingestion time +/// becomes a tumbling-window chain of 6 windows of 10 s over the range's +/// input, with the candidate's properties, θ and coverage, read by the +/// readout through a query-time merge of the whole chain; the chain's +/// retention is its one-minute window. +#[test] +fn a_windowed_build_is_realized_as_a_chain_read_through_a_merge() { + let (physical, ids) = ranged_example(); + let selected = chain_of( + &physical, + &[ + (ids[4], QT_NONE), + (ids[3], IT_ACROSS), + (ids[1], IT_NONE), + (ids[0], IT_NONE), + ], + ); + let realized = realize(&physical, &selected).unwrap(); + let dag = &realized.plan.dag; + let chain = dag + .nodes + .iter() + .find(|node| { + matches!( + node.payload, + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { + tumbling_window_width_ms: 10_000, + tumbling_window_count: 6, + .. + }) + ) + }) + .expect("a chain of six 10 s windows"); + assert_eq!(chain.lifetime, Lifetime::AcrossEvaluations(MEMORY)); + assert_eq!(chain.output_state.timing, IngestionTime); + assert!(chain.parameters.is_some() && chain.summary_coverage.is_some()); + let merge = dag + .nodes + .iter() + .find(|node| matches!(node.payload, Operator::ASAP(ASAPOp::SummaryMerge { .. }))) + .unwrap(); + assert_eq!(merge.output_state.timing, QueryTime); + assert_eq!(merge.lifetime, Lifetime::None); + assert!(dag + .edges + .iter() + .any(|e| e.producer == chain.id && e.consumer == merge.id)); + // The range the build read is not part of the plan: the chain reads + // the scan below it. + assert!(!dag + .nodes + .iter() + .any(|node| matches!(node.payload, Operator::NonASAP(NonASAPOp::TimeRange { .. })))); + assert_eq!(realized.tumbling_window_retention_ms[&chain.id], 60_000); +} + +/// A selection that is not valid over the candidates (an input without a +/// producer) is rejected, not realized. +#[test] +fn the_realizer_rejects_an_invalid_selection() { + let (physical, ids) = ranged_example(); + let mut selected = chain_of(&physical, &[(ids[4], QT_NONE), (ids[2], QT_NONE)]); + selected.edges.clear(); + assert!(matches!( + realize(&physical, &selected), + Err(asap_types::stages::PhysicalCandidateError::RejectedAssignment(_)) + )); +} From 2ebc3b9e1ab575b23a4338a5a347393ffcd09b72 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Sat, 7 Nov 2026 01:37:31 +0000 Subject: [PATCH 5/8] feat(stage3): baseline selection MILP over x, y, z, t with constraints (A)-(I) (#1207 PR 6) select_plan prices every physical candidate once (w_v under cost model 1, candidate_cost.rs: asap-cost's node_usage over one representative producer per input at a fixed computation rate), builds x binary per candidate, y on admissible edges whose producer delivers on its consumer's query evaluations, z per query output group, t on candidates, and constraints (A)-(F) as named functions ((E) only inside strongly connected components), solves to optimality with HiGHS, decodes into DagOfSelectedCandidates and realizes it. Uncostable candidates are pruned (UsageNotCostable). The objective is inexact until execution contexts (#1207 PR 7) and reported as such. Selection drops nodes_with_proven_non_negative_weights (the realizer confirms weights). Tests per constraint and a reference enumeration (concepts doc Appendix A) that agrees with the MILP. --- .../src/candidate_cost.rs | 369 +++++++++++ crates/stage3-plan-selection/src/lib.rs | 1 + .../stage3-plan-selection/src/milp/model.rs | 594 ++++++++++++++++-- .../src/milp/model/tests.rs | 469 ++++++++++++++ crates/stage3-plan-selection/src/report.rs | 12 +- crates/stage3-plan-selection/src/select.rs | 262 +++++++- 6 files changed, 1636 insertions(+), 71 deletions(-) create mode 100644 crates/stage3-plan-selection/src/candidate_cost.rs create mode 100644 crates/stage3-plan-selection/src/milp/model/tests.rs diff --git a/crates/stage3-plan-selection/src/candidate_cost.rs b/crates/stage3-plan-selection/src/candidate_cost.rs new file mode 100644 index 000000000..81975669d --- /dev/null +++ b/crates/stage3-plan-selection/src/candidate_cost.rs @@ -0,0 +1,369 @@ +//! $w_v$: the fixed cost per second of each physical candidate under cost +//! model 1, from `asap-cost` (concepts doc, section 3, "Stage 3: baseline +//! selection and realization"). Each candidate is priced as one node of a +//! plan ([`asap_cost::node_usage`]) over one representative producer per +//! input, at a fixed number of computations per second: +//! +//! - at ingestion time, over one second of arriving rows; +//! - at query time, materialized across evaluations, once per request of +//! the queries reaching its group (a materialized tumbling-window chain +//! builds only its newest window per request); +//! - at query time otherwise, once per evaluation of each schedule of the +//! queries reaching its group (`asap_cost::execution_demand::evaluation_rate`). +//! +//! It is the baseline's fixed-cost assumption: a candidate's cost does not +//! depend on which consumers select it. It is exact only when that holds; +//! execution contexts (#1207 PR 7) make the cost per context, and the +//! retention amount $h_v$ the cost of an `AcrossEvaluations` candidate per +//! selected consumer. The cost includes the candidate's own computation and +//! the raw data a query-time scan keeps on disk; the memory a query-time +//! result holds while its evaluation runs is left out, since it depends on +//! the latency of the queries reading it. + +use std::collections::BTreeSet; + +use asap_cost::execution_demand::evaluation_rate; +use asap_cost::{node_usage, CostInputs, CostModel, NodeUsage, ProducerOutput}; +use asap_types::ir::physical_export::{PhysicalASAPDAGNode, ScanWindow}; +use asap_types::ir::properties::{DataPrimitive, ExecutionDataState}; +use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorResultKind}; +use asap_types::physical::{ExecutionTiming, Lifetime}; +use asap_types::stages::{ + Memo, MemoGroupId, PhysicalASAPMemoCandidates, PhysicalCandidateId, PhysicalEdge, WindowLayout, +}; +use asap_types::workload::{DataWorkload, QueryWorkloadEntry}; + +/// $w_v$ of every physical candidate, by [`PhysicalCandidateId`]: its cost +/// per second, or why `asap-cost` could not cost it (the selection MILP +/// leaves such a candidate out). A candidate one of whose inputs has no +/// costable producer is not costable either. +pub(crate) fn candidate_costs( + physical: &PhysicalASAPMemoCandidates, + entries: &[QueryWorkloadEntry], + data: &DataWorkload, + cost_inputs: &CostInputs, + formula_scale: f64, +) -> Vec> { + let memo = &physical.logical_candidates.memo; + let count = physical.physical_candidates.len(); + let edges: Vec = physical.edges().map(|(_, edge)| edge).collect(); + let input_count = |id: usize| { + memo.candidate(physical.physical_candidates[id].logical_candidate_id) + .expression + .operator + .children() + .len() + }; + // A representative producer per input, chosen among costed candidates + // only, so the choices never form a cycle: each round costs the + // candidates whose every input has a costed admissible producer. + let mut representative: Vec>> = vec![None; count]; + let mut costs: Vec>> = vec![None; count]; + let mut nodes: Vec> = vec![None; count]; + let mut order: Vec = Vec::new(); + loop { + let mut progressed = false; + for id in 0..count { + if costs[id].is_some() { + continue; + } + let producers: Option> = (0..input_count(id)) + .map(|input| { + physical + .admissible_edges_into(PhysicalCandidateId(id), input) + .iter() + .map(|&edge| edges[edge.0].producer_candidate_id.0) + .find(|&producer| matches!(costs[producer], Some(Ok(_)))) + }) + .collect(); + let Some(producers) = producers else { + continue; + }; + let node = candidate_node(physical, id, &producers); + nodes[id] = Some(node); + representative[id] = Some(producers); + // Placeholder until the usage pass below; marks it costable for + // the choice of its consumers' representatives. + costs[id] = Some(Ok(0.0)); + order.push(id); + progressed = true; + } + if !progressed { + break; + } + } + let nodes: Vec> = nodes; + let mut usages: Vec>> = (0..count).map(|_| None).collect(); + let mut result: Vec> = (0..count) + .map(|_| Err("one of its inputs has no producer that can be costed".to_string())) + .collect(); + for &id in &order { + let node = nodes[id] + .as_ref() + .expect("built with its representative producers"); + let producers: Vec = representative[id].clone().unwrap_or_default(); + // A tumbling-window chain reads the input below its range: one + // level further down its range's representative producer. + let producers: Vec = match &node.payload { + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) => producers + .iter() + .map(|&range| { + representative[range] + .as_ref() + .map_or(range, |below| below[0]) + }) + .collect(), + _ => producers, + }; + let Some(producer_outputs) = producers + .iter() + .map(|&producer| { + let usage = usages[producer].as_ref()?; + let producer_node = nodes[producer].as_ref()?; + Some(ProducerOutput { + payload: &producer_node.payload, + output_schema: &producer_node.output_schema, + output: usage.output, + sources: usage.sources.clone(), + family: usage.family, + unchanged_columns: usage.unchanged_columns.clone(), + }) + }) + .collect::>>() + else { + continue; + }; + let candidate = &physical.physical_candidates[id]; + let group = candidate.logical_candidate_id.group; + let per_second = match computations_per_second(memo, group, candidate, entries, cost_inputs) + { + Ok(per_second) => per_second, + Err(error) => { + result[id] = Err(error); + continue; + } + }; + let window_ms = window_ms_below(memo, group); + let usage = node_usage( + node, + &producer_outputs, + (1, per_second), + scan_window_of(memo, group), + asap_cost::tumbling_windows_of(node), + (candidate.physical_properties.timing() == ExecutionTiming::IngestionTime) + .then_some(window_ms) + .flatten(), + data, + cost_inputs, + formula_scale, + ); + match usage { + Ok(usage) => { + result[id] = cost_per_second(&usage, cost_inputs); + usages[id] = Some(usage); + } + Err(error) => result[id] = Err(error.to_string()), + } + } + result +} + +/// Candidate `id` as one exported plan node over `producers` (its input +/// `i` read from the node of candidate `producers[i]`): its operator, or +/// for a tumbling-window layout the chain the realizer builds; its timing, +/// lifetime and θ; its group's schema. +fn candidate_node( + physical: &PhysicalASAPMemoCandidates, + id: usize, + producers: &[usize], +) -> PhysicalASAPDAGNode { + let memo = &physical.logical_candidates.memo; + let candidate = &physical.physical_candidates[id]; + let logical = memo.candidate(candidate.logical_candidate_id); + let group = &memo.groups()[candidate.logical_candidate_id.group.0 as usize]; + let mut next = producers.iter(); + let operator = logical + .expression + .operator + .map_children(|_| *next.next().expect("one producer per input")); + let payload = match (logical.logical_properties.window_layout(), operator) { + ( + Some(&WindowLayout::Tumbling { width_ms }), + Operator::ASAP(ASAPOp::SummaryAgg { + child, + family, + input, + reduction, + grouping, + filter, + }), + ) => { + let window_ms = + window_ms_below(memo, candidate.logical_candidate_id.group).unwrap_or(width_ms); + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { + child, + family, + input, + reduction, + grouping, + filter, + tumbling_window_width_ms: width_ms, + tumbling_window_count: u32::try_from(window_ms / width_ms.max(1)) + .unwrap_or(u32::MAX) + .max(1), + }) + } + (_, operator) => operator, + }; + // As the realizer's export sets it: a summary state or a maintained + // population is state, everything else raw values. + let primitive = match &payload { + Operator::ASAP( + ASAPOp::SummaryAgg { .. } + | ASAPOp::SummaryTumblingWindowChain { .. } + | ASAPOp::SummaryJoin { .. } + | ASAPOp::MaintainPopulation { .. }, + ) => DataPrimitive::SummaryState, + _ if group.group_properties.result_kind == OperatorResultKind::State => { + DataPrimitive::SummaryState + } + _ => DataPrimitive::Raw, + }; + PhysicalASAPDAGNode { + id, + payload, + output_state: ExecutionDataState { + timing: candidate.physical_properties.timing(), + primitive, + }, + lifetime: candidate.physical_properties.lifetime(), + parameters: logical.logical_properties.parameters().cloned(), + output_schema: group.group_properties.schema.clone(), + guarantee: None, + retention_ms: None, + summary_coverage: None, + } +} + +/// How often `candidate` of `group` is computed per second (module doc). +fn computations_per_second( + memo: &Memo, + group: MemoGroupId, + candidate: &asap_types::stages::PhysicalCandidate, + entries: &[QueryWorkloadEntry], + cost_inputs: &CostInputs, +) -> Result { + let properties = candidate.physical_properties; + if properties.timing() == ExecutionTiming::IngestionTime { + return Ok(1.0); + } + let per_request = matches!(properties.lifetime(), Lifetime::AcrossEvaluations(_)); + evaluation_rate( + memo.groups()[group.0 as usize] + .queries + .iter() + .filter_map(|&query| entries.get(query)) + .map(|entry| { + ( + &entry.recurrence, + if per_request { + None + } else { + entry.range_evaluation + }, + ) + }), + cost_inputs.non_repeating_query_horizon_seconds, + ) + .map_err(|error| error.to_string()) +} + +/// $w_v$ from a candidate's usage: its cost per second under cost model 1 +/// plus the raw data a query-time scan keeps on disk. +fn cost_per_second(usage: &NodeUsage<'_>, cost_inputs: &CostInputs) -> Result { + let CostModel::UsageOverTime(unit_cost) = &cost_inputs.cost_model else { + return Err("the selection MILP prices under cost model 1 only".into()); + }; + let own = usage + .cost + .cost_per_second + .ok_or("cost model 1 gives every node a cost per second")?; + let raw_data = usage.raw_scan.map_or(0.0, |(_, bytes)| { + unit_cost.cost_per_disk_byte_second * bytes as f64 + }); + let cost = own + raw_data; + if cost.is_finite() && cost >= 0.0 { + Ok(cost) + } else { + Err(format!( + "its cost per second {cost} is not a finite non-negative number" + )) + } +} + +/// The rows a query-time `Scan` of `group` reads, back from the evaluation +/// time: the union of the windows of the ranges over it, through time +/// shifts without an `@` anchor (`PhysicalASAPDAG::scan_window` on the +/// memo's groups instead of a plan's nodes). `Unbounded` when another +/// operator reads it or nothing does. +fn scan_window_of(memo: &Memo, group: MemoGroupId) -> ScanWindow { + fn windows( + memo: &Memo, + group: MemoGroupId, + end_offset_ms: i64, + seen: &mut BTreeSet, + ) -> ScanWindow { + if !seen.insert(group) { + return ScanWindow::Unbounded; + } + let parents = &memo.groups()[group.0 as usize].parents; + let mut union: Option = None; + for &parent in parents { + let window = match &memo.groups()[parent.0 as usize].expression { + Operator::NonASAP(NonASAPOp::TimeRange { range, .. }) => ScanWindow::Within { + end_offset_ms, + width_ms: i64::try_from(range.as_millis()).unwrap_or(i64::MAX), + }, + Operator::NonASAP(NonASAPOp::TimeShift { shift, .. }) if shift.at.is_none() => { + windows( + memo, + parent, + end_offset_ms.saturating_add(shift.offset_ms), + seen, + ) + } + _ => ScanWindow::Unbounded, + }; + union = Some(match union { + None => window, + Some(known) => known.union(window), + }); + } + union.unwrap_or(ScanWindow::Unbounded) + } + windows(memo, group, 0, &mut BTreeSet::new()) +} + +/// The window, in ms, of the first range below `group`'s operator, +/// through filters, projections and time shifts +/// (`asap_cost::maintained_window_ms` on the memo's groups): what an +/// ingestion-time grouping maintains, and a tumbling-window chain covers. +fn window_ms_below(memo: &Memo, group: MemoGroupId) -> Option { + let range_ms = |range: &std::time::Duration| u64::try_from(range.as_millis()).ok(); + let expression = &memo.groups()[group.0 as usize].expression; + if let Operator::NonASAP(NonASAPOp::TimeRange { range, .. }) = expression { + return range_ms(range); + } + let mut below = **expression.children().first()?; + for _ in 0..memo.groups().len() { + let expression = &memo.groups()[below.0 as usize].expression; + match expression { + Operator::NonASAP(NonASAPOp::TimeRange { range, .. }) => return range_ms(range), + Operator::NonASAP( + NonASAPOp::Filter { .. } | NonASAPOp::Project { .. } | NonASAPOp::TimeShift { .. }, + ) => below = **expression.children().first()?, + _ => return None, + } + } + None +} diff --git a/crates/stage3-plan-selection/src/lib.rs b/crates/stage3-plan-selection/src/lib.rs index a4168acc7..aef09d579 100644 --- a/crates/stage3-plan-selection/src/lib.rs +++ b/crates/stage3-plan-selection/src/lib.rs @@ -14,6 +14,7 @@ //! per-candidate error bounds $E_c$ and $D_c$ (section 4). Costs come from //! `asap-cost`. +mod candidate_cost; mod milp; mod models; mod report; diff --git a/crates/stage3-plan-selection/src/milp/model.rs b/crates/stage3-plan-selection/src/milp/model.rs index e976556c0..8f1e78417 100644 --- a/crates/stage3-plan-selection/src/milp/model.rs +++ b/crates/stage3-plan-selection/src/milp/model.rs @@ -7,25 +7,36 @@ //! | execution contexts (section 5) | $x^{\mathrm{mat}}_v$, $x_{\kappa,v}$, $y_{\kappa,u,c,i}$, $h_v$ ([`ExecutionContextVariables`]) | (R) [`retention_constraints`] | //! | accuracy (section 4) | $E_c$, $F_{c,i}$, $D_c$, $H_{c,i}$ ([`AccuracyVariables`]) | (K) [`accuracy_constraints`] | //! -//! Every function body is `todo!()` until #1207 PRs 6 (baseline), 7 -//! (contexts) and 8 (accuracy) fill them in; until then nothing calls -//! them, hence the `dead_code` allowance below. -#![allow(dead_code)] +//! The baseline (section 3) is built and solved by [`solve_selection`]: +//! $x_v$ and $y_{u,c,i}$ binary, $z_{q,v}$ binary per query output group, +//! $t_v$ continuous in $[0, |\mathcal C| - 1]$, the objective +//! $\min \sum_v w_v x_v$ with $w_v$ from [`crate::candidate_cost`]. The +//! execution-context and accuracy families are `todo!()` until #1207 PRs 7 +//! (contexts) and 8 (accuracy); nothing calls them yet, hence their +//! `dead_code` allowances. -use std::collections::HashMap; +use std::collections::{BTreeSet, HashMap}; +use std::time::Instant; -use good_lp::{Constraint, Variable}; +use good_lp::solvers::highs::highs; +use good_lp::{ + constraint, variable, Constraint, Expression, ProblemVariables, SolverModel, Variable, +}; +use asap_types::physical::{ExecutionTiming, Lifetime}; use asap_types::stages::{ - ExecutionContextId, PhysicalASAPMemoCandidates, PhysicalCandidateId, PhysicalEdgeId, + output_references, DagOfSelectedCandidates, ExecutionContext, ExecutionContextId, MemoGroupId, + PhysicalASAPMemoCandidates, PhysicalCandidateId, PhysicalEdge, PhysicalEdgeId, SelectedEdge, + SelectedNodeId, }; +use asap_types::workload::QueryWorkloadEntry; /// A query, by index in the workload. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] pub struct QueryId(pub usize); /// The baseline variables (section 3). -struct SelectionVariables { +pub(crate) struct SelectionVariables { /// $x_v$, by [`PhysicalCandidateId`]. x: Vec, /// $y_{u,c,i}$, on admissible edges only. @@ -36,76 +47,557 @@ struct SelectionVariables { t: Vec, } -/// The execution-context variables (section 5), which replace $x$ and the -/// unindexed $y$ of the baseline. -struct ExecutionContextVariables { - /// $x^{\mathrm{mat}}_v$, for an `AcrossEvaluations` candidate. - x_materialized: HashMap, - /// $x_{\kappa,v}$, for a candidate with $\ell$ = None or `OneEvaluation`. - x_in_execution_context: HashMap<(ExecutionContextId, PhysicalCandidateId), Variable>, - /// $y_{\kappa,u,c,i}$, indexed by the consumer's context. - y_in_execution_context: HashMap<(ExecutionContextId, PhysicalEdgeId), Variable>, - /// $h_v$, for an `AcrossEvaluations` producer. - retention_amount: HashMap, +/// What the baseline MILP is built over: Stage 2's candidates and +/// admissible edges, each candidate's $w_v$ (`None`: not costable, so +/// $x_v = 0$), and $G_q$ for each query's output groups. +pub(crate) struct SelectionInputs<'a> { + pub(crate) physical: &'a PhysicalASAPMemoCandidates, + /// $w_v$ per second, by [`PhysicalCandidateId`]. + pub(crate) cost_per_second: &'a [Option], + /// Each query's workload entry, whose recurrence and range evaluation + /// are its evaluation context. + pub(crate) entries: &'a [QueryWorkloadEntry], } -/// The accuracy variables of constraint (K) (section 4), all continuous. -struct AccuracyVariables { - /// $E_c$, by [`PhysicalCandidateId`]. - error_bound: Vec, - /// $F_{c,i}$: the chosen producer's error bound on input $i$ of $c$. - error_bound_of_input: HashMap<(PhysicalCandidateId, usize), Variable>, - /// $D_c$, by [`PhysicalCandidateId`]. - failure_probability: Vec, - /// $H_{c,i}$: the chosen producer's failure probability on input $i$ of $c$. - failure_probability_of_input: HashMap<(PhysicalCandidateId, usize), Variable>, +impl SelectionInputs<'_> { + /// $\mathcal E$, by [`PhysicalEdgeId`]. + fn edges(&self) -> Vec { + self.physical.edges().map(|(_, edge)| edge).collect() + } + + /// $|\mathcal I(c)|$: the inputs of candidate `c`. + fn input_count(&self, c: PhysicalCandidateId) -> usize { + self.physical + .logical_candidates + .memo + .candidate(self.physical.physical_candidates[c.0].logical_candidate_id) + .expression + .operator + .children() + .len() + } + + /// Whether edge `edge`'s producer delivers on every evaluation context + /// its consumer serves: a producer materialized across evaluations read + /// by a query-time consumer must deliver on the evaluation of each + /// query reaching the consumer's group (`PhysicalCandidate::delivers_on`). + /// The baseline has no execution contexts, so it requires every one; + /// with contexts (#1207 PR 7) the check is per context $\kappa$ of + /// $y_{\kappa,u,c,i}$. A consumer at ingestion time is not checked + /// here (its pipeline context is #1207 PR 7's). + fn delivers_to_consumer(&self, edge: &PhysicalEdge) -> bool { + let producer = &self.physical.physical_candidates[edge.producer_candidate_id.0]; + let consumer = &self.physical.physical_candidates[edge.consumer_candidate_id.0]; + if !matches!( + producer.physical_properties.lifetime(), + Lifetime::AcrossEvaluations(_) + ) || consumer.physical_properties.timing() != ExecutionTiming::QueryTime + { + return true; + } + let memo = &self.physical.logical_candidates.memo; + memo.groups()[consumer.logical_candidate_id.group.0 as usize] + .queries + .iter() + .filter_map(|&query| self.entries.get(query)) + .all(|entry| { + producer.delivers_on( + memo, + &ExecutionContext::Evaluation { + recurrence: entry.recurrence.clone(), + range_evaluation: entry.range_evaluation, + }, + ) + }) + } + + /// The physical candidates of memo group `group`, ascending. + fn candidates_of_group(&self, group: MemoGroupId) -> Vec { + (0..self.physical.physical_candidates.len()) + .map(PhysicalCandidateId) + .filter(|id| { + self.physical.physical_candidates[id.0] + .logical_candidate_id + .group + == group + }) + .collect() + } + + /// Each query's output groups $G_q$, each pair once, in output-reference + /// order: a scalar query reads one per operator reference. + fn query_output_groups(&self) -> Vec<(QueryId, MemoGroupId)> { + let mut pairs: Vec<(QueryId, MemoGroupId)> = Vec::new(); + for (query, group) in output_references(&self.physical.query_roots) { + if !pairs.contains(&(QueryId(query), group)) { + pairs.push((QueryId(query), group)); + } + } + pairs + } } -/// (A) $\sum_{u \in P_{c,i}} y_{u,c,i} = x_c$. +/// The baseline MILP's variables over `inputs` (constraint (I), domains): +/// $x_v$ binary, fixed at 0 for a candidate that cannot be costed; $y$ +/// binary on each admissible edge ((G) and (H)) whose producer and consumer +/// can be costed and whose producer delivers on its consumer's evaluations +/// ([`SelectionInputs::delivers_to_consumer`]); $z_{q,v}$ binary for each costable $v \in G_q$; +/// $t_v \in [0, |\mathcal C| - 1]$ continuous. +fn selection_variables( + inputs: &SelectionInputs<'_>, + problem: &mut ProblemVariables, +) -> SelectionVariables { + let count = inputs.physical.physical_candidates.len(); + let costable = |id: PhysicalCandidateId| inputs.cost_per_second[id.0].is_some(); + let x = (0..count) + .map(|id| match costable(PhysicalCandidateId(id)) { + true => problem.add(variable().binary()), + false => problem.add(variable().min(0).max(0)), + }) + .collect(); + let mut y = HashMap::new(); + for (id, edge) in inputs.physical.edges() { + if edge.satisfies_logical_requirement + && edge.satisfies_physical_requirement + && inputs.delivers_to_consumer(&edge) + && costable(edge.producer_candidate_id) + && costable(edge.consumer_candidate_id) + { + y.insert(id, problem.add(variable().binary())); + } + } + let mut z = HashMap::new(); + for (query, group) in inputs.query_output_groups() { + for v in inputs.candidates_of_group(group) { + if costable(v) { + z.entry((query, v)) + .or_insert_with(|| problem.add(variable().binary())); + } + } + } + let rank_bound = count.saturating_sub(1) as f64; + let t = (0..count) + .map(|_| problem.add(variable().min(0).max(rank_bound))) + .collect(); + SelectionVariables { x, y, z, t } +} + +/// (A) $\sum_{u \in P_{c,i}} y_{u,c,i} = x_c$: every input of a selected +/// consumer has exactly one selected producer, and an unselected consumer +/// none. fn input_satisfaction_constraints( - physical: &PhysicalASAPMemoCandidates, + inputs: &SelectionInputs<'_>, variables: &SelectionVariables, ) -> Vec { - let _ = (physical, variables); - todo!("#1207 PR 6: constraint (A)") + let mut constraints = Vec::new(); + for c in (0..inputs.physical.physical_candidates.len()).map(PhysicalCandidateId) { + for i in 0..inputs.input_count(c) { + let producers: Expression = inputs + .physical + .admissible_edges_into(c, i) + .iter() + .filter_map(|edge| variables.y.get(edge)) + .sum(); + constraints.push(constraint!(producers == variables.x[c.0])); + } + } + constraints } -/// (B) $y_{u,c,i} \le x_u$. +/// (B) $y_{u,c,i} \le x_u$: a chosen producer is selected. fn producer_selection_constraints( - physical: &PhysicalASAPMemoCandidates, + inputs: &SelectionInputs<'_>, variables: &SelectionVariables, ) -> Vec { - let _ = (physical, variables); - todo!("#1207 PR 6: constraint (B)") + let edges = inputs.edges(); + let mut by_edge: Vec<(&PhysicalEdgeId, &Variable)> = variables.y.iter().collect(); + by_edge.sort_by_key(|(edge, _)| **edge); + by_edge + .into_iter() + .map(|(edge, &y)| { + let producer = edges[edge.0].producer_candidate_id; + constraint!(y <= variables.x[producer.0]) + }) + .collect() } -/// (C) $\sum_{v \in G_q} z_{q,v} = 1$; (D) $z_{q,v} \le x_v$. +/// (C) $\sum_{v \in G_q} z_{q,v} = 1$, for each output group of each query; +/// (D) $z_{q,v} \le x_v$. fn query_coverage_constraints( - physical: &PhysicalASAPMemoCandidates, + inputs: &SelectionInputs<'_>, variables: &SelectionVariables, ) -> Vec { - let _ = (physical, variables); - todo!("#1207 PR 6: constraints (C) and (D)") + let mut constraints = Vec::new(); + for (query, group) in inputs.query_output_groups() { + let outputs: Vec<(PhysicalCandidateId, Variable)> = inputs + .candidates_of_group(group) + .into_iter() + .filter_map(|v| variables.z.get(&(query, v)).map(|&z| (v, z))) + .collect(); + let covered: Expression = outputs.iter().map(|&(_, z)| z).sum(); + constraints.push(constraint!(covered == 1)); + for (v, z) in outputs { + constraints.push(constraint!(z <= variables.x[v.0])); + } + } + constraints } -/// (E) $t_u + 1 \le t_c + |\mathcal C|(1 - y_{u,c,i})$. +/// (E) $t_u + 1 \le t_c + |\mathcal C|(1 - y_{u,c,i})$: no selected cycle. +/// Only edges inside a strongly connected component of the admissible +/// edges can close a cycle, so (E) is written for those only (a producer +/// reading itself included, which (E) forbids). fn acyclicity_constraints( - physical: &PhysicalASAPMemoCandidates, + inputs: &SelectionInputs<'_>, variables: &SelectionVariables, ) -> Vec { - let _ = (physical, variables); - todo!("#1207 PR 6: constraint (E)") + let edges = inputs.edges(); + let component = strongly_connected_components( + inputs.physical.physical_candidates.len(), + variables.y.keys().map(|edge| { + ( + edges[edge.0].producer_candidate_id.0, + edges[edge.0].consumer_candidate_id.0, + ) + }), + ); + let candidate_count = inputs.physical.physical_candidates.len() as f64; + let mut on_cycles: Vec<(&PhysicalEdgeId, &Variable)> = variables + .y + .iter() + .filter(|(edge, _)| { + let edge = &edges[edge.0]; + component[edge.producer_candidate_id.0] == component[edge.consumer_candidate_id.0] + }) + .collect(); + on_cycles.sort_by_key(|(edge, _)| **edge); + on_cycles + .into_iter() + .map(|(edge, &y)| { + let edge = &edges[edge.0]; + let t_u = variables.t[edge.producer_candidate_id.0]; + let t_c = variables.t[edge.consumer_candidate_id.0]; + constraint!(t_u + 1 <= t_c + candidate_count * (1 - y)) + }) + .collect() } -/// (F) $x_v \le \sum_{(v,c,i)} y_{v,c,i} + \sum_q z_{q,v}$. +/// (F) $x_v \le \sum_{(v,c,i)} y_{v,c,i} + \sum_q z_{q,v}$: a selected +/// candidate supplies a selected input or a query output. fn utilization_constraints( - physical: &PhysicalASAPMemoCandidates, + inputs: &SelectionInputs<'_>, variables: &SelectionVariables, ) -> Vec { - let _ = (physical, variables); - todo!("#1207 PR 6: constraint (F)") + let mut outputs_of: HashMap = HashMap::new(); + for (&(_, v), &z) in &variables.z { + *outputs_of.entry(v).or_default() += z; + } + (0..inputs.physical.physical_candidates.len()) + .map(PhysicalCandidateId) + .map(|v| { + let consumers: Expression = inputs + .physical + .admissible_edges_out_of(v) + .iter() + .filter_map(|edge| variables.y.get(edge)) + .sum(); + let used = consumers + outputs_of.remove(&v).unwrap_or_default(); + constraint!(variables.x[v.0] <= used) + }) + .collect() } +/// The component of each of `node_count` nodes in the graph of `arcs` +/// (`(from, to)`), numbered so equal numbers are one strongly connected +/// component (Kosaraju's algorithm, iterative). +fn strongly_connected_components( + node_count: usize, + arcs: impl Iterator, +) -> Vec { + let mut forward = vec![Vec::new(); node_count]; + let mut backward = vec![Vec::new(); node_count]; + for (from, to) in arcs { + forward[from].push(to); + backward[to].push(from); + } + // Nodes by finishing time of a depth-first search over `forward`. + let mut finished = Vec::with_capacity(node_count); + let mut visited = vec![false; node_count]; + for start in 0..node_count { + if visited[start] { + continue; + } + visited[start] = true; + let mut stack = vec![(start, 0)]; + while let Some((node, next)) = stack.pop() { + if let Some(&to) = forward[node].get(next) { + stack.push((node, next + 1)); + if !visited[to] { + visited[to] = true; + stack.push((to, 0)); + } + } else { + finished.push(node); + } + } + } + let mut component = vec![usize::MAX; node_count]; + for (number, &start) in finished.iter().rev().enumerate() { + if component[start] != usize::MAX { + continue; + } + component[start] = number; + let mut stack = vec![start]; + while let Some(node) = stack.pop() { + for &from in &backward[node] { + if component[from] == usize::MAX { + component[from] = number; + stack.push(from); + } + } + } + } + component +} + +/// The solved baseline: the selected DAG and what the solve took. +#[derive(Debug, Clone)] +pub(crate) struct SolvedSelection { + pub(crate) selected_dag: DagOfSelectedCandidates, + /// $\sum_v w_v x_v$ at the optimum, per second. + pub(crate) objective_per_second: f64, + /// The solver's lower bound on it; NaN when it reports none. + pub(crate) lower_bound_per_second: f64, + pub(crate) variable_count: usize, + pub(crate) constraint_count: usize, + pub(crate) construction_ms: u64, + pub(crate) solver_ms: u64, +} + +/// Why the baseline MILP selected nothing. +#[derive(Debug, Clone, PartialEq)] +pub(crate) enum SelectionMilpError { + /// No assignment meets (A)–(I): some query output has no candidate + /// whose inputs all have costable admissible producers. + Infeasible, + /// HiGHS failed. + Solver(String), +} + +/// Build the baseline MILP over `inputs` ((A)–(I), $\min \sum_v w_v x_v$), +/// solve it to proven optimality with HiGHS, and decode $\{x = 1\}$, +/// $\{y = 1\}$ and $\{z = 1\}$ into a [`DagOfSelectedCandidates`]: one node +/// per selected candidate, in candidate order; one edge per selected $y$; +/// each output reference's node the selected $z$ of its query and group. +pub(crate) fn solve_selection( + inputs: &SelectionInputs<'_>, +) -> Result { + // (C) cannot hold for a query output group without a costable + // candidate. + if inputs.query_output_groups().into_iter().any(|(_, group)| { + inputs + .candidates_of_group(group) + .into_iter() + .all(|v| inputs.cost_per_second[v.0].is_none()) + }) { + return Err(SelectionMilpError::Infeasible); + } + let started = Instant::now(); + let mut problem = ProblemVariables::new(); + let variables = selection_variables(inputs, &mut problem); + let constraints: Vec = [ + input_satisfaction_constraints(inputs, &variables), + producer_selection_constraints(inputs, &variables), + query_coverage_constraints(inputs, &variables), + acyclicity_constraints(inputs, &variables), + utilization_constraints(inputs, &variables), + ] + .into_iter() + .flatten() + .collect(); + // Scaled so the largest cost is 1,000: HiGHS's tolerances are absolute. + let largest = inputs + .cost_per_second + .iter() + .flatten() + .fold(0.0_f64, |largest, &cost| largest.max(cost)); + let scale = if largest > 0.0 && largest.is_finite() { + 1_000.0 / largest + } else { + 1.0 + }; + let objective: Expression = inputs + .cost_per_second + .iter() + .zip(&variables.x) + .filter_map(|(cost, &x)| cost.map(|cost| cost * scale * x)) + .sum(); + let variable_count = problem.len(); + // HiGHS's columns are the variables in the order they were added. + let column_of: HashMap = problem + .iter_variables_with_def() + .enumerate() + .map(|(column, (variable, _))| (variable, column)) + .collect(); + let constraint_count = constraints.len(); + let mut model = problem + .minimise(objective.clone()) + .using(highs) + .with_all(constraints) + .try_into_inner() + .map_err(|error| SelectionMilpError::Solver(error.to_string()))?; + model.set_option("mip_rel_gap", 0.0); + model.set_option("mip_abs_gap", 0.0); + model.set_option("output_flag", false); + model.set_option("threads", 1); + let construction_ms = started.elapsed().as_millis() as u64; + let started = Instant::now(); + let solved = model + .try_solve() + .map_err(|status| SelectionMilpError::Solver(format!("{status:?}")))?; + let solver_ms = started.elapsed().as_millis() as u64; + use highs::{HighsModelStatus, HighsSolutionStatus}; + if solved.status() == HighsModelStatus::ModelEmpty { + // No constraint: no query reads an operator (literal-only scalar + // roots), so nothing is selected. + return Ok(SolvedSelection { + selected_dag: decode(inputs, &variables, |_| false), + objective_per_second: 0.0, + lower_bound_per_second: 0.0, + variable_count, + constraint_count, + construction_ms, + solver_ms, + }); + } + match solved.status() { + HighsModelStatus::Optimal => {} + HighsModelStatus::Infeasible | HighsModelStatus::UnboundedOrInfeasible => { + return Err(SelectionMilpError::Infeasible) + } + other => return Err(SelectionMilpError::Solver(format!("{other:?}"))), + } + if solved.primal_solution_status() != HighsSolutionStatus::Feasible { + return Err(SelectionMilpError::Solver( + "HiGHS reported an optimal status without a feasible solution".into(), + )); + } + let lower_bound_per_second = solved + .double_info_value(c"mip_dual_bound") + .unwrap_or(f64::NAN) + / scale; + let columns = solved.get_solution().columns().to_vec(); + let chosen = |variable: Variable| columns[column_of[&variable]] > 0.5; + let objective_per_second: f64 = inputs + .cost_per_second + .iter() + .zip(&variables.x) + .filter(|(_, &x)| chosen(x)) + .filter_map(|(cost, _)| *cost) + .sum(); + let selected_dag = decode(inputs, &variables, chosen); + Ok(SolvedSelection { + selected_dag, + objective_per_second, + lower_bound_per_second, + variable_count, + constraint_count, + construction_ms, + solver_ms, + }) +} + +/// $\{x = 1\}$, $\{y = 1\}$ and $\{z = 1\}$ as a [`DagOfSelectedCandidates`] +/// (`decode.rs` in the formulation): one node per selected candidate, in +/// candidate order; each selected edge between their nodes; each output +/// reference's node the candidate its query's $z$ selects in its group. +fn decode( + inputs: &SelectionInputs<'_>, + variables: &SelectionVariables, + chosen: impl Fn(Variable) -> bool, +) -> DagOfSelectedCandidates { + let candidate_of_node: Vec = (0..variables.x.len()) + .filter(|&v| chosen(variables.x[v])) + .map(PhysicalCandidateId) + .collect(); + let node_of: HashMap = candidate_of_node + .iter() + .enumerate() + .map(|(node, &candidate)| (candidate, SelectedNodeId(node))) + .collect(); + let edges = inputs.edges(); + let mut selected_edges: BTreeSet<(usize, usize, usize)> = BTreeSet::new(); + for (edge, &y) in &variables.y { + if chosen(y) { + let edge = &edges[edge.0]; + if let (Some(producer), Some(consumer)) = ( + node_of.get(&edge.producer_candidate_id), + node_of.get(&edge.consumer_candidate_id), + ) { + selected_edges.insert((consumer.0, edge.input_index_of_consumer, producer.0)); + } + } + } + let node_of_output_reference = output_references(&inputs.physical.query_roots) + .into_iter() + .map(|(query, group)| { + inputs + .candidates_of_group(group) + .into_iter() + .find(|v| { + variables + .z + .get(&(QueryId(query), *v)) + .is_some_and(|&z| chosen(z)) + }) + .and_then(|v| node_of.get(&v).copied()) + .unwrap_or(SelectedNodeId(usize::MAX)) + }) + .collect(); + DagOfSelectedCandidates { + candidate_of_node, + edges: selected_edges + .into_iter() + .map( + |(consumer, input_index_of_consumer, producer)| SelectedEdge { + producer: SelectedNodeId(producer), + consumer: SelectedNodeId(consumer), + input_index_of_consumer, + }, + ) + .collect(), + node_of_output_reference, + } +} + +#[allow(dead_code)] // #1207 PR 7. +/// The execution-context variables (section 5), which replace $x$ and the +/// unindexed $y$ of the baseline. +struct ExecutionContextVariables { + /// $x^{\mathrm{mat}}_v$, for an `AcrossEvaluations` candidate. + x_materialized: HashMap, + /// $x_{\kappa,v}$, for a candidate with $\ell$ = None or `OneEvaluation`. + x_in_execution_context: HashMap<(ExecutionContextId, PhysicalCandidateId), Variable>, + /// $y_{\kappa,u,c,i}$, indexed by the consumer's context. + y_in_execution_context: HashMap<(ExecutionContextId, PhysicalEdgeId), Variable>, + /// $h_v$, for an `AcrossEvaluations` producer. + retention_amount: HashMap, +} + +#[allow(dead_code)] // #1207 PR 8. +/// The accuracy variables of constraint (K) (section 4), all continuous. +struct AccuracyVariables { + /// $E_c$, by [`PhysicalCandidateId`]. + error_bound: Vec, + /// $F_{c,i}$: the chosen producer's error bound on input $i$ of $c$. + error_bound_of_input: HashMap<(PhysicalCandidateId, usize), Variable>, + /// $D_c$, by [`PhysicalCandidateId`]. + failure_probability: Vec, + /// $H_{c,i}$: the chosen producer's failure probability on input $i$ of $c$. + failure_probability_of_input: HashMap<(PhysicalCandidateId, usize), Variable>, +} + +#[allow(dead_code)] // #1207 PR 7. /// (R) $h_v \ge \operatorname{need}_{\kappa,v,c,i}\, y_{\kappa,v,c,i}$. /// /// Assumptions (decided 2026-11-06): an edge's need is the time horizon @@ -121,6 +613,7 @@ fn retention_constraints( todo!("#1207 PR 7: constraint (R)") } +#[allow(dead_code)] // #1207 PR 8. /// (K): $E_c$ and $D_c$ per candidate, big-M on selected edges, and the /// root checks $E_v \le \varepsilon_q + M_q(1 - z_{q,v})$, /// $D_v \le \delta_q + (1 - z_{q,v})$. @@ -132,3 +625,6 @@ fn accuracy_constraints( let _ = (physical, selection, accuracy); todo!("#1207 PR 8: constraint (K)") } + +#[cfg(test)] +mod tests; diff --git a/crates/stage3-plan-selection/src/milp/model/tests.rs b/crates/stage3-plan-selection/src/milp/model/tests.rs new file mode 100644 index 000000000..68ab6194d --- /dev/null +++ b/crates/stage3-plan-selection/src/milp/model/tests.rs @@ -0,0 +1,469 @@ +//! The baseline MILP's variables and constraints (A)–(I), each on a small +//! memo built through `Memo::find_or_add_group` and `Memo::add_candidate` +//! with hand-made physical candidates, edges and costs $w_v$; and the +//! reference enumeration of the concepts doc's Appendix A against the MILP. +use std::collections::BTreeMap; + +use asap_types::ir::operator::operator_properties::{Reduction, Source}; +use asap_types::ir::scalar::ColumnRef; +use asap_types::ir::schema::{ + DataType, Field, FieldDataType, GroupingStrategy, Schema, SketchAlgorithm, SketchParams, + SketchStatistic, SummaryUpdate, +}; +use asap_types::ir::{ASAPOp, NonASAPOp, Operator, QueryRoot}; +use asap_types::physical::{ExecutionTiming, Lifetime, PhysicalProperties, StorageMedium}; +use asap_types::stages::{ + parameter_list, CandidateExpression, CandidateForMemoGroup, CandidateForMemoGroupId, + LogicalASAPMemoCandidates, LogicalProperties, LogicalRequirements, Memo, MemoGroupId, + OriginOfCandidateForMemoGroup, PhysicalASAPMemoCandidates, PhysicalCandidate, + PhysicalCandidateId, PhysicalEdge, WindowLayout, +}; +use asap_types::workload::{ + QueryRecurrence, QueryRequirements, QueryWorkloadEntry, RepeatedDemand, RepetitionInterval, + TimestampMs, +}; + +use super::*; + +const QT_NONE: (ExecutionTiming, Lifetime) = (ExecutionTiming::QueryTime, Lifetime::None); +const IT_ACROSS: (ExecutionTiming, Lifetime) = ( + ExecutionTiming::IngestionTime, + Lifetime::AcrossEvaluations(StorageMedium::Memory), +); + +fn add( + memo: &mut Memo, + group: MemoGroupId, + operator: Operator, + parameters: Option, + window_layout: Option, +) -> CandidateForMemoGroupId { + let expression = CandidateExpression { operator }; + let logical_properties = + LogicalProperties::new(&expression, parameters, window_layout).unwrap(); + let input_requirements = + vec![LogicalRequirements::new(None); expression.operator.children().len()]; + memo.add_candidate( + group, + CandidateForMemoGroup { + expression, + logical_properties, + input_requirements, + fused_groups: Vec::new(), + origin: OriginOfCandidateForMemoGroup::Pass1, + }, + ) + .unwrap() +} + +/// The running example's query A over one scan: a scan group (candidate +/// 0), a KLL state group with three sizes θ (candidates 1–3) and one of +/// 10 s tumbling windows (candidate 4), and the p95 readout group, the +/// query's root (candidate 5), every candidate at query time and not +/// materialized except the windowed state, at ingestion time and +/// materialized across evaluations. Every edge is admissible unless +/// `inadmissible` names its (producer, consumer, input). +fn example(inadmissible: &[(usize, usize, usize)]) -> PhysicalASAPMemoCandidates { + let mut memo = Memo::default(); + let scan_operator = Operator::NonASAP(NonASAPOp::Scan { + source: Source::TimeSeries { + metric: "latency".into(), + }, + predicates: vec![], + schema: Schema::with_time_index( + vec![ + Field::plain("ts", DataType::Timestamp, false), + Field::plain("value", DataType::Float64, false), + ], + 0, + vec![], + ), + }); + let scan_group = memo.find_or_add_group(scan_operator.clone()).unwrap(); + let scan = add(&mut memo, scan_group, scan_operator, None, None); + let build = Operator::ASAP(ASAPOp::SummaryAgg { + child: scan_group, + family: FieldDataType::Sketch(SketchAlgorithm::Kll, GroupingStrategy::default()), + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::PerEntity, + grouping: GroupingStrategy::default(), + filter: None, + }); + let states = memo.find_or_add_group(build.clone()).unwrap(); + let list = parameter_list(SketchAlgorithm::Kll); + let mut logical = vec![scan]; + for parameters in &list[..3] { + logical.push(add( + &mut memo, + states, + build.clone(), + Some(parameters.clone()), + None, + )); + } + logical.push(add( + &mut memo, + states, + build, + Some(list[0].clone()), + Some(WindowLayout::Tumbling { width_ms: 10_000 }), + )); + let readout_operator = Operator::ASAP(ASAPOp::SummaryEstimate { + summary_input: states, + query: SketchStatistic::Quantile { q: 0.95 }, + }); + let readout_group = memo.find_or_add_group(readout_operator.clone()).unwrap(); + logical.push(add(&mut memo, readout_group, readout_operator, None, None)); + memo.add_query(QueryRoot::Operator(readout_group)); + let physical_candidates: Vec = logical + .iter() + .enumerate() + .map(|(id, &logical_candidate_id)| { + let (timing, lifetime) = if id == 4 { IT_ACROSS } else { QT_NONE }; + PhysicalCandidate { + logical_candidate_id, + physical_properties: PhysicalProperties::new(timing, lifetime).unwrap(), + } + }) + .collect(); + let mut edges = Vec::new(); + // Grouped by (consumer, input). + for state in 1..=4 { + edges.push((0, state, 0)); + } + for state in 1..=4 { + edges.push((state, 5, 0)); + } + let edges = edges + .into_iter() + .map(|(producer, consumer, input)| PhysicalEdge { + producer_candidate_id: PhysicalCandidateId(producer), + consumer_candidate_id: PhysicalCandidateId(consumer), + input_index_of_consumer: input, + satisfies_logical_requirement: !inadmissible.contains(&(producer, consumer, input)), + satisfies_physical_requirement: true, + }) + .collect(); + let query_roots = memo.query_roots.clone(); + PhysicalASAPMemoCandidates::new( + LogicalASAPMemoCandidates { + memo, + edges: Vec::new(), + }, + physical_candidates, + query_roots, + edges, + ) +} + +/// One query every 10 s at `phase_ms`. +fn entry(phase_ms: u64) -> QueryWorkloadEntry { + QueryWorkloadEntry { + query: Default::default(), + requirements: QueryRequirements::default(), + predictability: Default::default(), + recurrence: QueryRecurrence::Repeated(RepeatedDemand::FixedIntervalAt { + interval: RepetitionInterval(10_000), + evaluation_phase: TimestampMs(phase_ms), + }), + range_evaluation: None, + } +} + +fn solve( + physical: &PhysicalASAPMemoCandidates, + costs: &[Option], + entries: &[QueryWorkloadEntry], +) -> Result { + solve_selection(&SelectionInputs { + physical, + cost_per_second: costs, + entries, + }) +} + +/// The selected candidates, ascending. +fn selected(solved: &SolvedSelection) -> Vec { + solved + .selected_dag + .candidate_of_node + .iter() + .map(|id| id.0) + .collect() +} + +/// (A), (B), (C)/(D) and the objective: the readout (C) reads exactly one +/// state (A), the cheapest, which is selected with the scan it reads (B); +/// the decoded DAG validates and its objective is the selected costs' sum. +#[test] +fn one_producer_per_input_the_cheapest_and_its_inputs_selected() { + let physical = example(&[]); + let costs = [ + Some(1.0), + Some(5.0), + Some(2.0), + Some(9.0), + Some(7.0), + Some(1.0), + ]; + let solved = solve(&physical, &costs, &[entry(0)]).unwrap(); + assert_eq!(selected(&solved), [0, 2, 5]); + solved.selected_dag.validate(&physical).unwrap(); + assert_eq!(solved.selected_dag.edges.len(), 2); + assert_eq!(solved.objective_per_second, 4.0); + assert_eq!( + solved.selected_dag.node_of_output_reference, + [SelectedNodeId(2)] + ); +} + +/// (G) and (H): $y$ exists only on admissible edges, so the cheapest state +/// is not chosen when its edge into the readout fails $\alpha^{\log}$. +#[test] +fn an_inadmissible_edge_is_never_selected() { + let physical = example(&[(2, 5, 0)]); + let costs = [ + Some(1.0), + Some(5.0), + Some(2.0), + Some(9.0), + Some(7.0), + Some(1.0), + ]; + let solved = solve(&physical, &costs, &[entry(0)]).unwrap(); + assert_eq!(selected(&solved), [0, 1, 5]); +} + +/// A candidate `asap-cost` could not cost has $x_v = 0$; with no costable +/// root candidate, (C) is infeasible. +#[test] +fn an_uncostable_candidate_is_not_selected_and_an_uncostable_root_is_infeasible() { + let physical = example(&[]); + let mut costs = [Some(1.0), Some(5.0), None, Some(9.0), Some(7.0), Some(1.0)]; + assert_eq!( + selected(&solve(&physical, &costs, &[entry(0)]).unwrap()), + [0, 1, 5] + ); + costs[5] = None; + assert_eq!( + solve(&physical, &costs, &[entry(0)]).unwrap_err(), + SelectionMilpError::Infeasible + ); +} + +/// (F): a candidate that costs nothing is still not selected when no edge +/// or query output uses it, so every decoded node is reached. +#[test] +fn a_free_candidate_nothing_reads_is_not_selected() { + let physical = example(&[]); + let costs = [ + Some(1.0), + Some(5.0), + Some(2.0), + Some(0.0), + Some(0.0), + Some(1.0), + ]; + // At phase 5 s the windowed state (candidate 4) cannot be read: free, + // it is not selected; candidate 3, also free, is read. + let solved = solve(&physical, &costs, &[entry(5_000)]).unwrap(); + assert_eq!(selected(&solved), [0, 3, 5]); + solved.selected_dag.validate(&physical).unwrap(); +} + +/// A tumbling-window state materialized across evaluations, 10 s wide, +/// serves a query every 10 s at phase 0 but not at phase 5 s: its edge +/// gets no $y$ when the producer does not deliver on the consumer's +/// evaluations (`PhysicalCandidate::delivers_on`). +#[test] +fn a_windowed_state_is_read_only_by_queries_it_delivers_on() { + let physical = example(&[]); + let costs = [ + Some(10.0), + Some(5.0), + Some(6.0), + Some(7.0), + Some(1.0), + Some(1.0), + ]; + assert_eq!( + selected(&solve(&physical, &costs, &[entry(0)]).unwrap()), + [0, 4, 5] + ); + assert_eq!( + selected(&solve(&physical, &costs, &[entry(5_000)]).unwrap()), + [0, 1, 5] + ); +} + +/// (E): an edge closing a cycle (the readout feeding a state's input) is +/// cheaper than the scan, but the selection stays acyclic. +#[test] +fn a_cycle_is_never_selected() { + let mut physical = example(&[]); + let mut edges: Vec = physical.edges().map(|(_, edge)| edge).collect(); + edges.push(PhysicalEdge { + producer_candidate_id: PhysicalCandidateId(5), + consumer_candidate_id: PhysicalCandidateId(1), + input_index_of_consumer: 0, + satisfies_logical_requirement: true, + satisfies_physical_requirement: true, + }); + edges.sort_by_key(|edge| (edge.consumer_candidate_id, edge.input_index_of_consumer)); + physical = PhysicalASAPMemoCandidates::new( + physical.logical_candidates.clone(), + physical.physical_candidates.clone(), + physical.query_roots.clone(), + edges, + ); + let costs = [ + Some(100.0), + Some(1.0), + Some(50.0), + Some(50.0), + Some(50.0), + Some(1.0), + ]; + let solved = solve(&physical, &costs, &[entry(0)]).unwrap(); + assert_eq!(selected(&solved), [0, 1, 5]); + solved.selected_dag.validate(&physical).unwrap(); +} + +#[test] +fn strongly_connected_components_group_cycles_only() { + let component = strongly_connected_components(5, [(0, 1), (1, 2), (2, 1), (3, 3)].into_iter()); + assert_eq!(component[1], component[2]); + assert_ne!(component[0], component[1]); + assert_ne!(component[3], component[4]); + assert_ne!(component[0], component[4]); +} + +/// The concepts doc's Appendix A over physical candidates: every DAG with +/// one admissible producer per input of every reached candidate, from +/// each root candidate, without cycles; each DAG's cost the sum of its +/// distinct candidates' $w_v$. Returns the cheapest cost. +fn reference_minimum( + physical: &PhysicalASAPMemoCandidates, + costs: &[Option], + entries: &[QueryWorkloadEntry], +) -> Option { + let inputs = SelectionInputs { + physical, + cost_per_second: costs, + entries, + }; + let edges = inputs.edges(); + let producers = |c: PhysicalCandidateId, i: usize| -> Vec { + physical + .admissible_edges_into(c, i) + .iter() + .map(|edge| &edges[edge.0]) + .filter(|edge| { + inputs.delivers_to_consumer(edge) && costs[edge.producer_candidate_id.0].is_some() + }) + .map(|edge| edge.producer_candidate_id) + .collect() + }; + fn reaches( + dag: &BTreeMap>, + from: PhysicalCandidateId, + to: PhysicalCandidateId, + ) -> bool { + from == to + || dag + .get(&from) + .into_iter() + .flatten() + .any(|&next| reaches(dag, next, to)) + } + fn expand( + inputs: &SelectionInputs<'_>, + producers: &dyn Fn(PhysicalCandidateId, usize) -> Vec, + dag: BTreeMap>, + mut pending: Vec, + best: &mut Option, + ) { + let Some(consumer) = pending.pop() else { + let cost: f64 = dag + .keys() + .map(|v| inputs.cost_per_second[v.0].unwrap()) + .sum(); + *best = Some(best.map_or(cost, |known: f64| known.min(cost))); + return; + }; + let mut choices: Vec> = vec![Vec::new()]; + for i in 0..inputs.input_count(consumer) { + choices = choices + .into_iter() + .flat_map(|prefix| { + producers(consumer, i).into_iter().map(move |u| { + let mut next = prefix.clone(); + next.push(u); + next + }) + }) + .collect(); + } + for choice in choices { + if choice.iter().any(|&u| reaches(&dag, u, consumer)) { + continue; + } + let mut next = dag.clone(); + next.insert(consumer, choice.clone()); + let mut next_pending = pending.clone(); + for &u in &choice { + if !next.contains_key(&u) && !next_pending.contains(&u) { + next_pending.push(u); + } + } + expand(inputs, producers, next, next_pending, best); + } + } + let mut best = None; + for (_, group) in inputs.query_output_groups() { + for root in inputs.candidates_of_group(group) { + if costs[root.0].is_some() { + expand(&inputs, &producers, BTreeMap::new(), vec![root], &mut best); + } + } + } + best +} + +/// On the small example, under many cost assignments and both query +/// phases, the MILP's optimum equals the cheapest DAG the reference +/// enumeration lists (#1207 (c)). +#[test] +fn the_milp_agrees_with_the_reference_enumeration() { + let physical = example(&[(3, 5, 0)]); + // A deterministic pseudo-random sequence of costs in [0, 10). + let mut state = 0x2545_f491_4f6c_dd1d_u64; + let mut next = || { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + (state % 1000) as f64 / 100.0 + }; + for round in 0..50 { + let costs: Vec> = (0..6).map(|id| (round % 7 != id).then(&mut next)).collect(); + for phase_ms in [0, 5_000] { + let entries = [entry(phase_ms)]; + let reference = reference_minimum(&physical, &costs, &entries); + let milp = solve(&physical, &costs, &entries); + match (reference, milp) { + (Some(reference), Ok(solved)) => { + assert!( + (reference - solved.objective_per_second).abs() < 1e-9, + "costs {costs:?} phase {phase_ms}: reference {reference}, MILP {}", + solved.objective_per_second + ); + solved.selected_dag.validate(&physical).unwrap(); + } + (None, Err(SelectionMilpError::Infeasible)) => {} + (reference, milp) => { + panic!("costs {costs:?}: reference {reference:?}, MILP {milp:?}") + } + } + } + } +} diff --git a/crates/stage3-plan-selection/src/report.rs b/crates/stage3-plan-selection/src/report.rs index 7cac6899e..27366573d 100644 --- a/crates/stage3-plan-selection/src/report.rs +++ b/crates/stage3-plan-selection/src/report.rs @@ -7,9 +7,7 @@ use asap_cost::analytical::AnalyticalCostError; use asap_cost::{CostModel, PlanCost}; use asap_types::ir::physical_export::PhysicalASAPNodeId; use asap_types::ir::properties::AccuracyMetric; -use asap_types::stages::{ - DagOfSelectedCandidates, PhysicalCandidateId, PrunedCandidate, SelectedNodeId, -}; +use asap_types::stages::{DagOfSelectedCandidates, PhysicalCandidateId, PrunedCandidate}; use thiserror::Error; /// Peak memory above which a candidate's report carries a warning when the @@ -190,15 +188,11 @@ impl std::fmt::Display for AcceptedAccuracy { pub struct Selection { /// The selected plan: one node per run of a candidate. pub selected_dag: DagOfSelectedCandidates, - /// Nodes of `selected_dag` whose open count-min weight requirement - /// Stage 3 proved; the facade applies the proof to each node's operator - /// node when it builds the plan. - pub nodes_with_proven_non_negative_weights: Vec, pub cost: PlanCost, /// Per query, the accuracy Stage 3 accepted for the selected plan. pub accepted_accuracy_per_query: Vec, - /// The candidates Stage 3 ruled out before the search: dominated ones and - /// those whose usage could not be costed. + /// The candidates Stage 3 ruled out before the search: those whose usage + /// could not be costed. pub pruned_candidates: Vec>, pub search_report: SearchReport, } diff --git a/crates/stage3-plan-selection/src/select.rs b/crates/stage3-plan-selection/src/select.rs index f0468ae33..d4d418044 100644 --- a/crates/stage3-plan-selection/src/select.rs +++ b/crates/stage3-plan-selection/src/select.rs @@ -1,21 +1,50 @@ -//! Stage 3, plan selection: [`select_plan`] builds the selection MILP over -//! Stage 2's physical candidates and admissible edges -//! ([`crate::milp::model`]), solves it, decodes $\{x = 1\}$ and $\{y = 1\}$ -//! into a [`DagOfSelectedCandidates`](asap_types::stages::DagOfSelectedCandidates), +//! Stage 3, plan selection: [`select_plan`] prices every physical candidate +//! ($w_v$, [`crate::candidate_cost`]), builds and solves the baseline +//! selection MILP over Stage 2's candidates and admissible edges +//! ([`crate::milp::model`]), decodes $\{x = 1\}$, $\{y = 1\}$ and +//! $\{z = 1\}$ into a +//! [`DagOfSelectedCandidates`](asap_types::stages::DagOfSelectedCandidates), //! and realizes it once through the injected realizer. -use asap_cost::CostInputs; -use asap_types::stages::{AssignmentRealizer, PhysicalASAPMemoCandidates}; +use std::collections::BTreeMap; + +use asap_accuracy::AccuracyModel; +use asap_cost::{materialized_tumbling_window_chains, node_map, plan_cost, CostInputs, CostModel}; +use asap_types::ir::schema::FieldDataType; +use asap_types::ir::{ASAPOp, OperatorNode, QueryRoot}; +use asap_types::stages::{ + AssignmentRealizer, PhysicalASAPMemoCandidates, PhysicalCandidateError, PhysicalCandidateId, + PruneReason, PrunedCandidate, RealizedPlan, +}; use asap_types::workload::{DataWorkload, QueryWorkloadEntry}; +use crate::candidate_cost::candidate_costs; +use crate::milp::model::{solve_selection, SelectionInputs, SelectionMilpError}; use crate::models::AccuracyCheckInputs; -use crate::report::{ErrorSelectingPlan, Selection}; +use crate::report::{ + AcceptedAccuracy, ErrorSelectingPlan, InexactObjective, Rejection, RejectionCause, + SearchReport, SearchStatus, Selection, +}; + +/// Relative difference under which the MILP's objective and `plan_cost` +/// of the selected plan count as equal. +const COST_EQUALITY_TOLERANCE: f64 = 1e-6; + +/// The name of the inexact term of the baseline objective, with the +/// number of selected candidates it covers in [`InexactObjective`]. +const FIXED_COST_PER_CANDIDATE: &str = + "fixed cost per candidate, whichever consumers select it (execution contexts: #1207 PR 7)"; /// Stage 3, plan selection: select one plan among `physical`'s candidates -/// (module documentation). The selection is realized through `realizer`, -/// checked against every query's accuracy target (`accuracy_checks`) and -/// priced under `cost_inputs`. `entries[q]` is query `q`'s workload entry; -/// `data` the data workload with its bounds. +/// (module documentation). The selection is realized through `realizer` +/// and priced under `cost_inputs` (cost model 1 only). `entries[q]` is +/// query `q`'s workload entry; `data` the data workload with its bounds. +/// +/// Accuracy: until constraint (K) (#1207 PR 8), a query's accuracy rests on +/// the parameter floors of its edges, which Pass 2 derived from the query's +/// target ((G): an edge from a producer below its floor is not admissible); +/// `accuracy_checks.model` may accept an output regardless of its +/// guarantee, which the report records. /// /// Err: invalid or unsupported cost inputs, an infeasible catalog, a /// solver failure, or the realizer rejected the selected plan @@ -31,6 +60,213 @@ pub fn select_plan( cost_inputs .validate() .map_err(ErrorSelectingPlan::InvalidSettings)?; - let _ = (physical, entries, data, accuracy_checks, realizer); - todo!("#1207 PR 6: the baseline MILP over x, y, z, t, its decode and realization") + if !matches!(cost_inputs.cost_model, CostModel::UsageOverTime(_)) { + return Err(ErrorSelectingPlan::UnsupportedCostModel( + "cost models 2 and 3 (the selection MILP implements cost model 1 only)".into(), + )); + } + // The MILP's coefficients and `plan_cost` must use one calibration: a + // measured cost table needs a frozen factor. + if cost_inputs.frozen_calibration_factor.is_none() + && !cost_inputs.measured_operation_costs.operators.is_empty() + { + return Err(ErrorSelectingPlan::UnsupportedCostModel( + "a measured operation cost table without a frozen calibration factor".into(), + )); + } + if entries.len() != physical.query_roots.len() { + return Err(ErrorSelectingPlan::EntryCount { + candidate: "the physical memo".into(), + roots: physical.query_roots.len(), + entries: entries.len(), + }); + } + let formula_scale = cost_inputs.frozen_calibration_factor.unwrap_or(1.0); + let costs = candidate_costs(physical, entries, data, cost_inputs, formula_scale); + let pruned_candidates: Vec> = costs + .iter() + .enumerate() + .filter(|(_, cost)| cost.is_err()) + .map(|(id, _)| PrunedCandidate { + group: physical.physical_candidates[id].logical_candidate_id.group, + candidate: PhysicalCandidateId(id), + reason: PruneReason::UsageNotCostable, + }) + .collect(); + let cost_per_second: Vec> = costs.iter().map(|cost| cost.clone().ok()).collect(); + + let solved = solve_selection(&SelectionInputs { + physical, + cost_per_second: &cost_per_second, + entries, + }) + .map_err(|error| match error { + SelectionMilpError::Infeasible => { + ErrorSelectingPlan::NoValidCandidate(vec![infeasible(&costs)]) + } + SelectionMilpError::Solver(reason) => ErrorSelectingPlan::Solver(reason), + })?; + let validation_started = std::time::Instant::now(); + solved.selected_dag.validate(physical).map_err(|reason| { + ErrorSelectingPlan::Solver(format!("the decoded selection is invalid: {reason}")) + })?; + let realized = + realizer + .realize(physical, &solved.selected_dag) + .map_err(|error| match error { + PhysicalCandidateError::RejectedAssignment(reason) => { + ErrorSelectingPlan::RealizerRejected(reason) + } + error => ErrorSelectingPlan::RealizerRejected(error.to_string()), + })?; + let plan = &realized.plan; + let recurrence: Vec<_> = entries.iter().map(|e| e.recurrence.clone()).collect(); + let ranges: Vec<_> = entries.iter().map(|e| e.range_evaluation).collect(); + let (cost, _) = plan_cost( + &plan.dag, + &node_map(&plan.dag), + &materialized_tumbling_window_chains(&plan.dag), + &recurrence, + &ranges, + data, + cost_inputs, + formula_scale, + ) + .map_err(|(node, error)| ErrorSelectingPlan::Cost { + candidate: plan.id.clone(), + node, + error, + })?; + let plan_cost_per_second = cost.cost_per_second.ok_or_else(|| { + ErrorSelectingPlan::UnsupportedCostModel("a cost model without a cost per second".into()) + })?; + let objective_per_second = solved.objective_per_second; + let difference = (objective_per_second - plan_cost_per_second).abs(); + let tolerance = COST_EQUALITY_TOLERANCE * plan_cost_per_second.abs().max(f64::MIN_POSITIVE); + let inexact_objective = (difference > tolerance).then(|| InexactObjective { + objective_per_second, + plan_cost_per_second, + inexact_terms: BTreeMap::from([( + FIXED_COST_PER_CANDIDATE.to_string(), + solved.selected_dag.candidate_of_node.len(), + )]), + }); + let accepted_accuracy_per_query = accepted_accuracy(&realized, entries, accuracy_checks.model); + let validation_ms = validation_started.elapsed().as_millis() as u64; + Ok(Selection { + selected_dag: solved.selected_dag, + cost, + accepted_accuracy_per_query, + pruned_candidates, + search_report: SearchReport { + status: SearchStatus::Optimal, + cost_lower_bound_per_second: solved + .lower_bound_per_second + .is_finite() + .then_some(solved.lower_bound_per_second), + milp_variables: solved.variable_count, + milp_constraints: solved.constraint_count, + model_construction_ms: solved.construction_ms, + solver_ms: solved.solver_ms, + validation_ms, + inexact_objective, + }, + }) +} + +/// The rejection of an infeasible selection: no plan meets (A)–(I), with +/// the reasons `asap-cost` could not cost the candidates the model left +/// out, each once in candidate order: those candidates may be why no plan +/// is left. +fn infeasible(costs: &[Result]) -> Rejection { + let mut reasons: Vec<&str> = Vec::new(); + for reason in costs.iter().filter_map(|cost| cost.as_ref().err()) { + if !reasons.contains(&reason.as_str()) { + reasons.push(reason); + } + } + let mut reason = "no plan gives every query output a candidate whose every input has an \ + admissible producer (the selection MILP is infeasible)" + .to_string(); + if !reasons.is_empty() { + reason += &format!( + "; candidates left out because they could not be costed: {}", + reasons.join("; ") + ); + } + Rejection { + id: "workload".into(), + cause: RejectionCause::Infeasible, + reason, + } +} + +/// Per query, the accuracy accepted for the selected plan. An output that +/// reads no sketch is exact. Otherwise, until constraint (K) (#1207 PR 8) +/// checks the composed bound at the root, the output is accepted in its +/// target's metric through the parameter floors of its edges (Pass 2's +/// metric rule admits only edges whose metric converts to it), unless the +/// deployment's model accepts it regardless of its guarantee. +fn accepted_accuracy( + realized: &RealizedPlan, + entries: &[QueryWorkloadEntry], + model: &dyn AccuracyModel, +) -> Vec { + realized + .plan + .roots + .iter() + .zip(entries) + .map(|(root, entry)| { + let target = &entry.requirements.accuracy; + let sketches = sketches_read(root); + let exact = sketches.is_empty(); + let regardless = !exact && model.accepts_output_regardless_of_guarantee(root, target); + AcceptedAccuracy { + target_metric: target.accuracy_metric, + accepted_metric: (!exact && !regardless) + .then_some(target.accuracy_metric) + .flatten(), + accepted_regardless_of_guarantee: regardless, + guarantee_check_rejection: None, + sketches, + } + }) + .collect() +} + +/// The sketch algorithms of the summary builds `root` reads, in plan order +/// (depth first, inputs in order), each build once. +fn sketches_read(root: &QueryRoot) -> Vec { + let mut seen = std::collections::HashSet::new(); + let mut sketches = Vec::new(); + let mut pending: Vec> = match root { + QueryRoot::Operator(node) => vec![std::rc::Rc::clone(node)], + QueryRoot::Scalar(expression) => expression + .operator_refs() + .into_iter() + .rev() + .cloned() + .collect(), + }; + while let Some(node) = pending.pop() { + if !seen.insert(std::rc::Rc::as_ptr(&node)) { + continue; + } + if let Some( + ASAPOp::SummaryAgg { + family: FieldDataType::Sketch(algorithm, _), + .. + } + | ASAPOp::SummaryTumblingWindowChain { + family: FieldDataType::Sketch(algorithm, _), + .. + }, + ) = node.asap() + { + sketches.push(format!("{algorithm:?}")); + } + pending.extend(node.children().into_iter().rev().cloned()); + } + sketches } From 8e15e6317b92e905ba323916c83ff641e7a52535 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Sat, 7 Nov 2026 01:37:31 +0000 Subject: [PATCH 6/8] feat(planner): install the realized plan; facade, devtools and test helpers over the realizer (#1207 PR 6) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PlanOutput carries the realized plan (replacing its tumbling-window retention map); plan_installable installs it, expanding its chains with each tumbling window at its chain's properties, θ, own coverage and retention (NoRealizedPlan when a pass realized none). Plan file wire version 14 (summary_coverage). The facade and devtools take the plan to run from the realizer. Integration helpers realize plans from the memo (realize_from_root_candidates); the fuzzer prices a witness through Stages 0-3. Planner test workloads that read chains declare their evaluation phase (delivers_on needs known evaluation times). --- crates/devtools/src/bin/asap_planner.rs | 2 - crates/devtools/src/planner_fuzz/mod.rs | 89 +++++-- crates/devtools/src/planning_stages.rs | 26 +-- crates/integration-tests/src/lib.rs | 220 +++++++++++++++++- .../tests/stage1_logical_asap_optimization.rs | 16 +- crates/planner/src/installable.rs | 161 ++++++++++++- crates/planner/src/pass.rs | 13 +- crates/planner/src/run.rs | 29 +-- crates/planner/tests/columnar_batches.rs | 2 +- crates/planner/tests/runtime_install.rs | 45 +++- crates/planner/tests/sql_table_scans.rs | 2 +- 11 files changed, 509 insertions(+), 96 deletions(-) diff --git a/crates/devtools/src/bin/asap_planner.rs b/crates/devtools/src/bin/asap_planner.rs index 9dbde8e69..5ce3e5cfe 100644 --- a/crates/devtools/src/bin/asap_planner.rs +++ b/crates/devtools/src/bin/asap_planner.rs @@ -734,8 +734,6 @@ fn selection_json(selection: &Result) -> Value { .collect::>(), "node_of_output_reference": nodes(&dag.node_of_output_reference), }, - "nodes_with_proven_non_negative_weights": - nodes(&selection.nodes_with_proven_non_negative_weights), "cost": cost_json(&selection.cost), "accepted_accuracy_per_query": accepted_json(&selection.accepted_accuracy_per_query), "pruned_candidates": selection diff --git a/crates/devtools/src/planner_fuzz/mod.rs b/crates/devtools/src/planner_fuzz/mod.rs index ad5df92fe..457bea70a 100644 --- a/crates/devtools/src/planner_fuzz/mod.rs +++ b/crates/devtools/src/planner_fuzz/mod.rs @@ -34,7 +34,7 @@ pub mod relations; pub mod report; pub mod shrink; -use std::collections::HashMap; +use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::rc::Rc; use serde::{Deserialize, Serialize}; @@ -451,25 +451,86 @@ impl Planner { }) } - /// Stage 2 and Stage 3 over the logical plan `roots` of `workload`'s - /// queries (one root per query, in order): every materialization - /// assignment implemented and priced, the cheapest selected, with the - /// same models as [`plan`](Self::plan). Guarantees are composed again - /// first, so a derived node carries the guarantee of what it reads, and - /// summaries are sized for their consumers as Pass 1 sizes them. - /// With `last_root_is_key_filter`, the last root is a `Filter` on - /// group-key columns only and keeps its input's guarantee (whole groups - /// selected by an exact key keep each remaining row's bound), which - /// Stage 1's composition, giving no guarantee to any filter over - /// approximate rows, does not know. + /// Stages 0–3 over the witness plan `roots` of `workload`'s queries + /// (one root per query, in order), with the same models as + /// [`plan`](Self::plan): the witness's own operators are hash-consed + /// into the memo and Stage 3 selects the cheapest plan of them, each + /// summary at the θ its consumers' parameter floors allow. The cost is + /// that plan's, and `derived_root` the nodes of the realized plan only + /// the last root reads. `last_root_is_key_filter` is accepted for the + /// relations' interface; accuracy is not composed again here (constraint + /// (K), #1207 PR 8). pub fn price( &self, workload: &FuzzWorkload, roots: &[QueryRoot], last_root_is_key_filter: bool, ) -> Result { - let _ = (workload, roots, last_root_is_key_filter); - todo!("#1207 PR 6: price a witness plan through the new Stage 2 and Stage 3") + let _ = last_root_is_key_filter; + let planning = workload.planning_workload(); + let entries: Vec<_> = planning.query_workload.entries().collect(); + let cost_inputs = self.cost_inputs(); + let deployment = workload.deployment(); + let models = PlanningModels::builtin() + .with_cost_inputs(&cost_inputs) + .with_deployment_capabilities(&deployment) + .with_ignore_accuracy_guarantees(!self.settings.enforce_accuracy_guarantees); + let run = run_search_over( + roots.to_vec(), + &entries, + &workload.data, + models, + PlanSearchSettings::default(), + ) + .map_err(|e| format!("witness: {e}"))?; + let selection = run + .run + .selection + .as_ref() + .map_err(|e| format!("witness selection: {e}"))?; + let cost = &selection.cost; + let cost_per_second = cost + .cost_per_second + .ok_or("no cost per second (cost model 3)")?; + let dag = &run + .run + .selected_physical_plan + .as_ref() + .ok_or("a selection has a realized plan")? + .plan + .dag; + let mut inputs: BTreeMap<_, Vec<_>> = BTreeMap::new(); + for edge in &dag.edges { + inputs.entry(edge.consumer).or_default().push(edge.producer); + } + let reached = |root| { + let mut seen = BTreeSet::new(); + let mut stack = vec![root]; + while let Some(node) = stack.pop() { + if seen.insert(node) { + stack.extend(inputs.get(&node).into_iter().flatten().copied()); + } + } + seen + }; + let reached_by_root: Vec> = dag.roots.iter().map(|r| reached(*r)).collect(); + let (last, others) = reached_by_root.split_last().ok_or("no roots")?; + let private: Vec<_> = last + .iter() + .filter(|node| others.iter().all(|set| !set.contains(node))) + .filter_map(|node| cost.per_node.get(node)) + .collect(); + Ok(Priced { + selected: "selected".into(), + cost_per_second, + derived_root: PrivateNodes { + cost_per_second: private.iter().filter_map(|n| n.cost_per_second).sum(), + operators: private + .iter() + .map(|n| format!("{:?}", n.operator)) + .collect(), + }, + }) } /// [`price`](Self::price) of `workload`'s selected logical plan diff --git a/crates/devtools/src/planning_stages.rs b/crates/devtools/src/planning_stages.rs index 8a3600636..8fcd9e678 100644 --- a/crates/devtools/src/planning_stages.rs +++ b/crates/devtools/src/planning_stages.rs @@ -17,14 +17,11 @@ use asap_stage2_physical_optimization::{ use asap_stage3_plan_selection::select_plan; use asap_types::ir::schema::schema_support::with_promql_series_identity; use asap_types::ir::schema::{DataType, Field, Schema}; -use asap_types::ir::{OperatorNode, QueryRoot}; +use asap_types::ir::QueryRoot; use asap_types::physical::PhysicalWorkloadPlan; use asap_types::stages::{AssignmentRealizer, Memo}; -use asap_types::workload::bounds::with_confirmed_weights_at; use asap_types::workload::{DataWorkload, PlanningWorkload, QueryWorkloadEntry}; use serde_json::Value; -use std::collections::HashSet; -use std::rc::Rc; /// The last stage a run runs (`asap_planner --run-until`); the run's /// fields for later stages are left empty. @@ -346,12 +343,7 @@ pub fn run_stages_until( run.physical_asap_candidates = Some(physical); return Ok(stages); } - let realizer = Stage2AssignmentRealizer { - original_roots: &roots, - entries, - data, - query_time_results_retained: models.deployment_capabilities.query_time_results_retained, - }; + let realizer = Stage2AssignmentRealizer { entries, data }; match select_plan( &physical, entries, @@ -363,17 +355,9 @@ pub fn run_stages_until( Ok(selection) => { match realizer.realize(&physical, &selection.selected_dag) { Ok(realized) => { - // As the facade does: the plan to run carries the weight - // proofs Stage 3 made. - let proven: HashSet<*const OperatorNode> = selection - .nodes_with_proven_non_negative_weights - .iter() - .filter_map(|node| realized.operator_node_of_selected_node.get(node.0)) - .map(Rc::as_ptr) - .collect(); - let plan = with_confirmed_weights_at(&realized.plan.roots, data, &|node| { - proven.contains(&Rc::as_ptr(node)) - }); + // As the facade does: the plan to run is the realized + // plan's roots. + let plan = realized.plan.roots.clone(); validate(&plan)?; run.selected_plan = Some(plan); run.selected_physical_plan = Some(realized); diff --git a/crates/integration-tests/src/lib.rs b/crates/integration-tests/src/lib.rs index 1e2b34629..fe8991e41 100644 --- a/crates/integration-tests/src/lib.rs +++ b/crates/integration-tests/src/lib.rs @@ -193,23 +193,223 @@ pub mod planning { } } - /// Exact computation and each explicit single-target rule application, - /// each as one plan for `root`. Pass 1 now adds these ways as memo - /// candidates; realizing one plan per way from the memo is #1207 PR 6. + /// Exact computation and each way Pass 1 and Pass 2 offer for the + /// root group of `root`, each realized as one plan: for each root + /// candidate run at query time and not materialized, each input reads + /// its first admissible producer of the same kind + /// ([`realize_from_root_candidates`]). Accuracy guarantees are ignored, + /// so every summary Pass 1 offers stays in the memo. pub fn explicit_local_candidates(root: Rc) -> Vec> { - let _ = root; - todo!("#1207 PR 6: realize one plan per way to compute from Pass 1's memo") + realize_from_root_candidates(root, |_| false) + .into_iter() + .map(|plan| plan_root(&plan)) + .collect() } - /// One explicit plan using `family` wherever a local target supports it, - /// keeping all other computations exact. Realizing it from Pass 1's - /// memo is #1207 PR 6. + /// One plan of `root` that reads `family` wherever an input can: each + /// input prefers a producer whose plan builds a `family` sketch + /// ([`realize_from_root_candidates`]); the first such plan. pub fn explicit_summary_candidate( root: Rc, family: asap_types::ir::schema::SketchAlgorithm, ) -> Rc { - let _ = (root, family); - todo!("#1207 PR 6: realize the plan of one summary family from Pass 1's memo") + let builds_family = + move |operator: &asap_types::ir::Operator| { + matches!( + operator, + asap_types::ir::Operator::ASAP(asap_types::ir::ASAPOp::SummaryAgg { + family: asap_types::ir::schema::FieldDataType::Sketch(algorithm, _), + .. + }) if *algorithm == family + ) + }; + realize_from_root_candidates(root, builds_family) + .iter() + .map(plan_root) + .find(|plan| { + asap_types::ir::OperatorNode::reachable(plan) + .iter() + .any(|node| { + matches!( + node.asap(), + Some(asap_types::ir::ASAPOp::SummaryAgg { + family: asap_types::ir::schema::FieldDataType::Sketch(algorithm, _), + .. + }) if *algorithm == family + ) + }) + }) + .unwrap_or_else(|| panic!("no plan builds a {family:?} sketch")) + } + + /// The single operator root of a realized plan. + fn plan_root(plan: &asap_types::stages::RealizedPlan) -> Rc { + match plan.plan.roots.as_slice() { + [QueryRoot::Operator(node)] => Rc::clone(node), + other => panic!("one operator root: {other:?}"), + } + } + + /// For each candidate of `root`'s output group run at query time and + /// not materialized, the plan Stage 2's realizer builds when each input + /// reads, among its admissible producers run at query time and not + /// materialized, the first whose own plan contains an operator + /// `preferred` accepts, else the first; each a distinct plan, in + /// candidate order. Stages run with accuracy guarantees ignored and a + /// target of 0.05, so every summary Pass 1 offers stays. + pub fn realize_from_root_candidates( + root: Rc, + preferred: impl Fn(&asap_types::ir::Operator) -> bool, + ) -> Vec { + use asap_types::physical::{ExecutionTiming, Lifetime}; + use asap_types::stages::{ + AssignmentRealizer, DagOfSelectedCandidates, PhysicalASAPMemoCandidates, + PhysicalCandidateId, SelectedEdge, SelectedNodeId, + }; + let models = asap_planner::PlanningModels::builtin().with_ignore_accuracy_guarantees(true); + let data = declared_data(&root); + let entries = [asap_types::workload::QueryWorkloadEntry { + query: Default::default(), + requirements: asap_types::workload::QueryRequirements { + accuracy: AccuracyTarget::max_error(0.05), + max_latency_ms: None, + }, + predictability: Default::default(), + recurrence: Default::default(), + range_evaluation: None, + }]; + let stages = run_all_stages_with(root, entries[0].requirements.accuracy.clone(), models); + let Some(physical) = stages.run.physical_asap_candidates.as_ref() else { + return Vec::new(); + }; + let memo = &physical.logical_candidates.memo; + let edges: Vec<_> = physical.edges().map(|(_, edge)| edge).collect(); + let unmaterialized_at_query_time = |id: PhysicalCandidateId| { + let properties = physical.physical_candidates[id.0].physical_properties; + properties.timing() == ExecutionTiming::QueryTime + && properties.lifetime() == Lifetime::None + }; + let inputs_of = |id: PhysicalCandidateId| { + memo.candidate(physical.physical_candidates[id.0].logical_candidate_id) + .expression + .operator + .children() + .len() + }; + // Whether the plan below `id`, choosing first producers, contains a + // preferred operator (depth-bounded: the memo may have cycles). + fn contains_preferred( + physical: &PhysicalASAPMemoCandidates, + edges: &[asap_types::stages::PhysicalEdge], + id: PhysicalCandidateId, + preferred: &dyn Fn(&asap_types::ir::Operator) -> bool, + depth: usize, + ) -> bool { + let memo = &physical.logical_candidates.memo; + let operator = &memo + .candidate(physical.physical_candidates[id.0].logical_candidate_id) + .expression + .operator; + if preferred(operator) { + return true; + } + depth > 0 + && (0..operator.children().len()).any(|input| { + physical + .admissible_edges_into(id, input) + .iter() + .any(|edge| { + contains_preferred( + physical, + edges, + edges[edge.0].producer_candidate_id, + preferred, + depth - 1, + ) + }) + }) + } + let mut plans: Vec = Vec::new(); + let realizer = asap_stage2_physical_optimization::Stage2AssignmentRealizer { + entries: &entries, + data: &data, + }; + let output_group = match &physical.query_roots[0] { + QueryRoot::Operator(group) => *group, + QueryRoot::Scalar(_) => return Vec::new(), + }; + for root_candidate in (0..physical.physical_candidates.len()) + .map(PhysicalCandidateId) + .filter(|&id| { + physical.physical_candidates[id.0] + .logical_candidate_id + .group + == output_group + && unmaterialized_at_query_time(id) + }) + { + let mut selected = DagOfSelectedCandidates { + candidate_of_node: vec![root_candidate], + edges: Vec::new(), + node_of_output_reference: vec![SelectedNodeId(0)], + }; + let mut pending = vec![(root_candidate, SelectedNodeId(0), 0usize)]; + let mut feasible = true; + while let Some((candidate, node, depth)) = pending.pop() { + for input in 0..inputs_of(candidate) { + let producers: Vec = physical + .admissible_edges_into(candidate, input) + .iter() + .map(|edge| edges[edge.0].producer_candidate_id) + .filter(|&producer| unmaterialized_at_query_time(producer)) + .collect(); + let producer = producers + .iter() + .copied() + .find(|&producer| { + contains_preferred(physical, &edges, producer, &preferred, 8) + }) + .or_else(|| producers.first().copied()); + let (Some(producer), true) = (producer, depth < 64) else { + feasible = false; + break; + }; + let producer_node = match selected + .candidate_of_node + .iter() + .position(|&known| known == producer) + { + Some(known) => SelectedNodeId(known), + None => { + selected.candidate_of_node.push(producer); + let new = SelectedNodeId(selected.candidate_of_node.len() - 1); + pending.push((producer, new, depth + 1)); + new + } + }; + selected.edges.push(SelectedEdge { + producer: producer_node, + consumer: node, + input_index_of_consumer: input, + }); + } + if !feasible { + break; + } + } + if !feasible { + continue; + } + if let Ok(plan) = realizer.realize(physical, &selected) { + if !plans + .iter() + .any(|known| known.plan.roots == plan.plan.roots) + { + plans.push(plan); + } + } + } + plans } /// Logical plans visited by the bounded search for `root`, in archive order. diff --git a/crates/integration-tests/tests/stage1_logical_asap_optimization.rs b/crates/integration-tests/tests/stage1_logical_asap_optimization.rs index dbc5d4ce1..ddf9f5849 100644 --- a/crates/integration-tests/tests/stage1_logical_asap_optimization.rs +++ b/crates/integration-tests/tests/stage1_logical_asap_optimization.rs @@ -321,14 +321,22 @@ fn execute(root: &Rc, rows: &[Vec<(&str, Value)>]) -> Vec (Rc, Rc) { - let _ = (catalog, query); - todo!("#1207 PR 6: realize the sum-and-count plan of `avg` from Pass 1's memo") + let exact = lower_sql(query, catalog, accuracy()).await.unwrap(); + let quotient = asap_integration_tests::planning::explicit_local_candidates(Rc::clone(&exact)) + .into_iter() + .find(|plan| { + OperatorNode::reachable(plan) + .iter() + .any(|node| matches!(node.non_asap(), Some(NonASAPOp::BinaryOp { .. }))) + }) + .expect("a plan computing avg as sum over count"); + (exact, quotient) } /// The candidate of `memo`'s root group that is not its own expression diff --git a/crates/planner/src/installable.rs b/crates/planner/src/installable.rs index 858d575a7..979469d51 100644 --- a/crates/planner/src/installable.rs +++ b/crates/planner/src/installable.rs @@ -24,6 +24,7 @@ //! has every chain expanded into the tumbling windows the executor runs. use std::collections::{HashMap, HashSet}; +use std::rc::Rc; use asap_cost::{NodePriceSource, OperatorKind}; use asap_stage3_plan_selection::{SearchReport, Selection}; @@ -32,12 +33,17 @@ use asap_types::ir::materialized_view::{ IngestionRefreshGroup, InstallableDagError, MaterializedView, RawDataRetention, }; +use asap_types::ir::flat::flatten; +use asap_types::ir::operator::tumbling_window_chain::expand_tumbling_window_chains_in_workload; +use asap_types::ir::physical_export::{physical_asap_dag_with_assignments, ExportedNodeAssignment}; use asap_types::ir::physical_export::{ PhysicalASAPDAG, PhysicalASAPDAGEdge, PhysicalASAPDAGNode, PhysicalASAPDAGValidationError, PhysicalASAPNodeId, }; use asap_types::ir::properties::ExecutionTiming; use asap_types::ir::SchemaDerivationError; +use asap_types::ir::{ASAPOp, OperatorNode}; +use asap_types::physical::{Lifetime, PhysicalProperties}; use asap_types::stages::RealizedPlan; use asap_types::workload::QueryWorkloadEntry; @@ -47,7 +53,7 @@ use crate::{e2e_plan, PlanError, UserInput}; /// The version of the plan-file format [`InstallablePlan`] serializes to: /// the DAG, its nodes' fields and the query texts. -pub const INSTALLABLE_PLAN_WIRE_VERSION: u32 = 13; +pub const INSTALLABLE_PLAN_WIRE_VERSION: u32 = 14; /// The selected plan, ready to install and execute: what /// [`plan_installable`] returns and a runtime writes as its plan file. @@ -153,6 +159,11 @@ pub struct PredictedQueryUsage { #[derive(Debug, thiserror::Error)] #[non_exhaustive] pub enum InstallablePlanError { + /// Unsupported request: the optimization pass selected no plan the + /// realizer built (`PlanOutput::realized_plan` is `None`), so there is no + /// physical plan to install. + #[error("the optimization pass returned no realized plan to install")] + NoRealizedPlan, /// Internal bug: a tumbling-window chain of the selected plan does not expand. #[error("tumbling-window chain expansion: {0}")] TumblingWindowChainExpansion(#[from] SchemaDerivationError), @@ -233,8 +244,12 @@ pub async fn plan_installable( pass: input.pass, }; let output = e2e_plan(input).await?; - let _ = (output, entries); - todo!("#1207 PR 6: install the plan the realizer built from the selected DAG") + let realized = output + .realized_plan + .as_ref() + .ok_or(PlanError::Installable(InstallablePlanError::NoRealizedPlan))?; + installable_plan_of(realized, output.selection.as_ref(), &entries) + .map_err(PlanError::Installable) } /// The installable plan of `realized`, the plan the realizer @@ -616,14 +631,142 @@ struct ExpandedWithRetention { installed_node_ids: HashMap>, } -/// The realized plan's DAG with its tumbling-window chains expanded; each -/// tumbling window takes the retention Stage 2 gives its chain -/// (`RealizedPlan::tumbling_window_retention_ms`). +/// The realized plan's DAG with its tumbling-window chains expanded into +/// the tumbling windows the executor runs. Each tumbling window takes its +/// chain's timing, lifetime and θ, the coverage of its own time slice +/// (`ViewCoverage::of_tumbling_window`) and the retention Stage 2 gives its +/// chain (`RealizedPlan::tumbling_window_retention_ms`); the range and time +/// shift below it run with the chain, not materialized. Every other node +/// keeps what the realizer assigned it. fn tumbling_window_retention_of_expanded_tumbling_windows( realized: &RealizedPlan, ) -> Result { - let _ = realized; - todo!("#1207 PR 6: expand the realized DAG's tumbling-window chains") + let selected = &realized.plan.dag; + // Reject unsupported raw-source shapes before expanding possibly thousands + // of tumbling windows. The complete expanded-plan validation remains authoritative. + selected.check_raw_data_windows()?; + // The realizer exported `roots` with `flatten`'s node ids, so node `i` + // of the selected DAG is `nodes_by_id[i]`. + let (_, nodes_by_id) = flatten(&realized.plan.roots); + let assignment_of_node = |id: PhysicalASAPNodeId| { + let node = &selected.nodes[id]; + ExportedNodeAssignment { + physical_properties: PhysicalProperties::new(node.output_state.timing, node.lifetime) + .expect("a validated plan's nodes have valid physical properties"), + parameters: node.parameters.clone(), + summary_coverage: node.summary_coverage.clone(), + } + }; + let id_of: HashMap<*const OperatorNode, PhysicalASAPNodeId> = nodes_by_id + .iter() + .enumerate() + .map(|(id, node)| (Rc::as_ptr(node), id)) + .collect(); + let mut chain_retention: HashMap<*const OperatorNode, u64> = HashMap::new(); + for (&node_id, &retention_ms) in &realized.tumbling_window_retention_ms { + let chain = nodes_by_id + .get(node_id) + .filter(|node| matches!(node.asap(), Some(ASAPOp::SummaryTumblingWindowChain { .. }))) + .ok_or(InstallablePlanError::TumblingWindowRetentionOfUnknownChain { node_id })?; + chain_retention.insert(Rc::as_ptr(chain), retention_ms); + } + let expansion = expand_tumbling_window_chains_in_workload(&realized.plan.roots) + .map_err(InstallablePlanError::TumblingWindowChainExpansion)?; + let mut assignments: HashMap<*const OperatorNode, ExportedNodeAssignment> = HashMap::new(); + for (original, replacement) in &expansion.expanded_nodes { + if let Some(&id) = id_of.get(&Rc::as_ptr(original)) { + assignments.insert(Rc::as_ptr(replacement), assignment_of_node(id)); + } + } + // Each tumbling window as its chain; the new nodes below it (its range + // and time shift) at the chain's timing, not materialized. + let mut tumbling_windows_of_chain: Vec<(PhysicalASAPNodeId, &[Rc])> = Vec::new(); + for expanded in &expansion.chains { + let chain_id = id_of[&Rc::as_ptr(&expanded.chain)]; + let chain = assignment_of_node(chain_id); + let width_ms = match expanded.chain.asap() { + Some(ASAPOp::SummaryTumblingWindowChain { + tumbling_window_width_ms, + .. + }) => *tumbling_window_width_ms, + _ => unreachable!("only chains are expanded"), + }; + let below = ExportedNodeAssignment { + physical_properties: PhysicalProperties::new( + chain.physical_properties.timing(), + Lifetime::None, + ) + .expect("a result that is not materialized is valid at either time"), + parameters: None, + summary_coverage: None, + }; + for (index, tumbling_window) in expanded.tumbling_windows.iter().enumerate() { + assignments.insert( + Rc::as_ptr(tumbling_window), + ExportedNodeAssignment { + summary_coverage: chain + .summary_coverage + .as_ref() + .map(|coverage| coverage.of_tumbling_window(index as u32, width_ms)), + ..chain.clone() + }, + ); + let mut pending: Vec> = + tumbling_window.children().into_iter().cloned().collect(); + while let Some(node) = pending.pop() { + if assignments.contains_key(&Rc::as_ptr(&node)) { + continue; + } + assignments.insert(Rc::as_ptr(&node), below.clone()); + pending.extend(node.children().into_iter().cloned()); + } + } + tumbling_windows_of_chain.push((chain_id, &expanded.tumbling_windows)); + } + let (dag, expanded_nodes_by_id) = + physical_asap_dag_with_assignments(&expansion.roots, &|node| { + assignments + .get(&Rc::as_ptr(node)) + .cloned() + .expect("every node of the expanded plan has an assignment") + })?; + let installed_id_of: HashMap<*const OperatorNode, PhysicalASAPNodeId> = expanded_nodes_by_id + .iter() + .enumerate() + .map(|(id, node)| (Rc::as_ptr(node), id)) + .collect(); + let mut retention_ms_by_node = HashMap::new(); + let mut installed_node_ids: HashMap> = + HashMap::new(); + for (chain_id, tumbling_windows) in tumbling_windows_of_chain { + let retention_ms = chain_retention.get(&Rc::as_ptr(&nodes_by_id[chain_id])); + let ids: Vec = tumbling_windows + .iter() + .filter_map(|window| installed_id_of.get(&Rc::as_ptr(window)).copied()) + .collect(); + if let Some(&retention_ms) = retention_ms { + for &id in &ids { + retention_ms_by_node.insert(id, retention_ms); + } + } + installed_node_ids.insert(chain_id, ids); + } + // Every other node runs as the node replacing it. + for (original, replacement) in &expansion.expanded_nodes { + if let (Some(&id), Some(&installed)) = ( + id_of.get(&Rc::as_ptr(original)), + installed_id_of.get(&Rc::as_ptr(replacement)), + ) { + installed_node_ids + .entry(id) + .or_insert_with(|| vec![installed]); + } + } + Ok(ExpandedWithRetention { + dag, + retention_ms_by_node, + installed_node_ids, + }) } /// Stage 3's record of its selected plan. @@ -706,7 +849,7 @@ mod tests { let workload: crate::PlanningWorkload = serde_json::from_str( r#"{"query_workload": {"language": "promql", "repeating_queries": [{ "query": "avg by (job) (quantile_over_time(0.99, data[300s]))", - "demand": {"fixed_interval": 1000}, + "demand": {"fixed_interval_at": {"interval": 1000, "evaluation_phase": 0}}, "requirements": {"accuracy": {"max_error": 0.05, "failure_probability": 0.3, "accuracy_scope": "whole_result"}}, "predictability": {"predictable": {"known_at": null}}}]}, diff --git a/crates/planner/src/pass.rs b/crates/planner/src/pass.rs index e02265f92..d8b310c66 100644 --- a/crates/planner/src/pass.rs +++ b/crates/planner/src/pass.rs @@ -29,7 +29,6 @@ use asap_cost::CostInputs; use asap_stage1_pass1_per_computation_candidates::PerComputationCandidateError; use asap_stage3_plan_selection::{AccuracyCheckInputs, Selection}; use asap_types::deployment_capabilities::DeploymentCapabilities; -use asap_types::ir::physical_export::PhysicalASAPNodeId; static BUILTIN_ACCURACY_MODEL: BuiltinAccuracyModel = BuiltinAccuracyModel; static IGNORE_ACCURACY_GUARANTEES: IgnoreAccuracyGuarantees = IgnoreAccuracyGuarantees; @@ -212,11 +211,11 @@ pub struct PlanOutput { /// How plan selection chose the plan, when the pass selects among priced /// candidates. pub selection: Option, - /// For each tumbling-window chain of the selected plan materialized beyond one - /// evaluation, by node id in its physical DAG, how long its completed - /// tumbling windows are retained ([`RealizedPlan::tumbling_window_retention_ms`](asap_types::stages::RealizedPlan::tumbling_window_retention_ms)); - /// empty for a pass that does not set it. - pub tumbling_window_retention_ms: BTreeMap, + /// The selected plan as the realizer built it (its physical DAG, each + /// node with its timing, lifetime, θ and coverage, and its tumbling-window + /// retention): what [`plan_installable`](crate::plan_installable) + /// installs. `None` for a pass that does not realize one. + pub realized_plan: Option, } impl PlanOutput { @@ -226,7 +225,7 @@ impl PlanOutput { scalar_roots: Vec::new(), scalar_plans: Vec::new(), selection: None, - tumbling_window_retention_ms: BTreeMap::new(), + realized_plan: None, } } diff --git a/crates/planner/src/run.rs b/crates/planner/src/run.rs index 3be733dbc..0d473f80b 100644 --- a/crates/planner/src/run.rs +++ b/crates/planner/src/run.rs @@ -5,7 +5,6 @@ use asap_stage0_memo::memo_of_queries; -use std::collections::HashSet; use std::rc::Rc; use std::time::Instant; @@ -24,7 +23,6 @@ use asap_types::stages::{ AssignmentRealizer, LogicalASAPMemoCandidates, Memo, PhysicalASAPMemoCandidates, PhysicalCandidateError, RealizedPlan, }; -use asap_types::workload::bounds::with_confirmed_weights_at; use asap_types::workload::{DataWorkload, QueryLanguage, QueryWorkloadEntry}; use crate::pass::{ @@ -96,8 +94,8 @@ pub struct PlanningRun { pub selection: Result, /// The selected plan as Stage 2 built it, with its tumbling-window retention. pub selected_physical_plan: Option, - /// The plan to run: the selected plan's roots with the weight proofs - /// Stage 3 made applied. + /// The plan to run: the selected plan's roots, as the realizer built + /// them (with the update-weight proofs the data gives applied). pub selected_plan: Option>, pub report: PlanningReport, } @@ -136,12 +134,7 @@ pub fn run_planning( ..PlanningReport::default() }; let pass1_memo = pass1.as_ref().ok().filter(|_| settings.inspect).cloned(); - let realizer = Stage2AssignmentRealizer { - original_roots: &roots, - entries, - data, - query_time_results_retained: models.deployment_capabilities.query_time_results_retained, - }; + let realizer = Stage2AssignmentRealizer { entries, data }; let mut logical_asap_candidates = None; let mut physical_asap_candidates = None; let mut selected_physical_plan = None; @@ -197,17 +190,7 @@ pub fn run_planning( let realized = realizer .realize(physical, &selection.selected_dag) .map_err(SelectionError::Realization)?; - let proven: HashSet<*const OperatorNode> = selection - .nodes_with_proven_non_negative_weights - .iter() - .filter_map(|node| realized.operator_node_of_selected_node.get(node.0)) - .map(Rc::as_ptr) - .collect(); - selected_plan = Some(with_confirmed_weights_at( - &realized.plan.roots, - data, - &|node| proven.contains(&Rc::as_ptr(node)), - )); + selected_plan = Some(realized.plan.roots.clone()); selected_physical_plan = Some(realized); Ok(selection) })(); @@ -300,9 +283,7 @@ impl OptimizationPass for Stage1ToStage3Pass { } } } - if let Some(realized) = run.selected_physical_plan { - output.tumbling_window_retention_ms = realized.tumbling_window_retention_ms; - } + output.realized_plan = run.selected_physical_plan; output.selection = Some(selection); Ok(output) } diff --git a/crates/planner/tests/columnar_batches.rs b/crates/planner/tests/columnar_batches.rs index 658dee91b..54d917993 100644 --- a/crates/planner/tests/columnar_batches.rs +++ b/crates/planner/tests/columnar_batches.rs @@ -36,7 +36,7 @@ async fn a_tumbling_window_is_built_from_arrow_columns_with_one_label_entry_per_ let workload: PlanningWorkload = serde_json::from_str( r#"{"query_workload": {"language": "promql", "repeating_queries": [ {"query": "quantile_over_time(0.99, latency_ms[60m])", - "demand": {"fixed_interval": 600000}, + "demand": {"fixed_interval_at": {"interval": 600000, "evaluation_phase": 0}}, "requirements": {"accuracy": {"accuracy_metric": "rank", "max_error": 0.01, "failure_probability": 0.01, "accuracy_scope": "whole_result"}, "max_latency_ms": 200.0}, diff --git a/crates/planner/tests/runtime_install.rs b/crates/planner/tests/runtime_install.rs index b8087aab7..69af0c94d 100644 --- a/crates/planner/tests/runtime_install.rs +++ b/crates/planner/tests/runtime_install.rs @@ -39,7 +39,7 @@ fn workload(queries: &[&str]) -> PlanningWorkload { .iter() .map(|query| { format!( - r#"{{"query": {query:?}, "demand": {{"fixed_interval": 600000}}, + r#"{{"query": {query:?}, "demand": {{"fixed_interval_at": {{"interval": 600000, "evaluation_phase": 0}}}}, "requirements": {{"accuracy": {{"accuracy_metric": "rank", "max_error": 0.01, "failure_probability": 0.01, "accuracy_scope": "whole_result"}}, "max_latency_ms": 200.0}}, @@ -395,7 +395,7 @@ fn p99_avg_workload(values_non_negative: bool) -> PlanningWorkload { serde_json::from_str(&format!( r#"{{"query_workload": {{"language": "promql", "repeating_queries": [{{ "query": "avg by (region, service, job) (quantile_over_time(0.99, data[100s]))", - "demand": {{"fixed_interval": 1000}}, + "demand": {{"fixed_interval_at": {{"interval": 1000, "evaluation_phase": 0}}}}, "requirements": {{"accuracy": {{"max_error": 0.05, "failure_probability": 0.3, "accuracy_scope": "whole_result"}}}}, "predictability": {{"predictable": {{"known_at": null}}}}}}]}}, @@ -444,7 +444,7 @@ fn counter_topk_workload(function: &str) -> PlanningWorkload { serde_json::from_str(&format!( r#"{{"query_workload": {{"language": "promql", "repeating_queries": [{{ "query": {query:?}, - "demand": {{"fixed_interval": 60000}}, + "demand": {{"fixed_interval_at": {{"interval": 60000, "evaluation_phase": 0}}}}, "requirements": {{"accuracy": {{"max_error": 0.05, "failure_probability": 0.3}}}}, "predictability": {{"predictable": {{"known_at": null}}}}}}]}}, "data_workload": {{"arrival": {{"ingesting": {{"rows_per_second": 100000.0}}}}, @@ -497,3 +497,42 @@ async fn check_compiles_reports_the_executors_error() { // ── Classic-histogram `histogram_quantile` (issue #890) ───────────────────── // ── PromQL absent() (issue #895) ───────────────────────────────────────────── + +/// The plan file round trip of #1207 PR 6: Stage 3 selects, Stage 2's +/// realizer exports each node with its timing, lifetime, θ and coverage, +/// installation expands the tumbling-window chains, and a runtime reads the +/// file back unchanged and its executor compiles it. Every summary build +/// carries its θ, every materialized view a coverage. +#[tokio::test] +async fn the_plan_file_carries_theta_and_coverage_and_reads_back() { + let json = plan_json(&["quantile_over_time(0.99, latency_ms[60m])"]).await; + let plan = InstallablePlan::from_json(&json).unwrap(); + assert_eq!(serde_json::to_string(&plan).unwrap(), json); + plan.check_compiles().unwrap(); + let builds: Vec<_> = plan + .dag + .nodes + .iter() + .filter(|node| { + matches!( + node.payload, + asap_types::ir::Operator::ASAP(asap_types::ir::ASAPOp::SummaryAgg { .. }) + ) + }) + .collect(); + assert!(!builds.is_empty()); + for build in &builds { + assert!(build.parameters.is_some(), "{:?}", build.payload); + } + let views = plan.materialized_views().unwrap(); + assert!(!views.is_empty()); + for view in &views { + let node = plan + .dag + .nodes + .iter() + .find(|n| n.id == view.node_id) + .unwrap(); + assert!(node.summary_coverage.is_some()); + } +} diff --git a/crates/planner/tests/sql_table_scans.rs b/crates/planner/tests/sql_table_scans.rs index 9dd9b41a2..c7a239599 100644 --- a/crates/planner/tests/sql_table_scans.rs +++ b/crates/planner/tests/sql_table_scans.rs @@ -51,7 +51,7 @@ fn workload(queries: &[&str], accuracy: &str, interval_ms: u64) -> PlanningWorkl .iter() .map(|query| { format!( - r#"{{"query": {query:?}, "demand": {{"fixed_interval": {interval_ms}}}, + r#"{{"query": {query:?}, "demand": {{"fixed_interval_at": {{"interval": {interval_ms}, "evaluation_phase": 0}}}}, "requirements": {{"accuracy": {accuracy}}}, "predictability": {{"predictable": {{"known_at": null}}}}}}"# ) From 265a93aa281ddb53283f0ad64f4a838ead5e1bce Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Sat, 7 Nov 2026 01:46:09 +0000 Subject: [PATCH 7/8] fix: review fixes (#1207 PR 6) Installation keeps the assignment of a realized node under an expanded chain (only the new range and time shift run with the chain); a chain's merge runs at ingestion time only when no query reads it. Stage 3 costs candidates producers first in one worklist pass, picking each input's representative among costed producers, and refuses a window layout the realizer would refuse; SelectionInputs indexes candidates by group, the output groups and the edges once. Deleted, unused after the realizer: RealizedPlan::operator_node_of_selected_node, bounds::with_confirmed_weights and with_confirmed_weights_at. Docs and terminology. --- crates/devtools/src/bin/asap_planner.rs | 2 +- crates/planner/src/installable.rs | 14 +- .../src/catalog/realize.rs | 12 +- .../src/catalog/tests.rs | 1 - .../src/candidate_cost.rs | 317 +++++++++++------- .../stage3-plan-selection/src/milp/model.rs | 66 ++-- .../src/milp/model/tests.rs | 12 +- crates/stage3-plan-selection/src/select.rs | 22 +- crates/types/src/ir/schema/sketch.rs | 2 +- crates/types/src/stages/realizer.rs | 29 +- crates/types/src/workload/bounds.rs | 64 ---- 11 files changed, 289 insertions(+), 252 deletions(-) diff --git a/crates/devtools/src/bin/asap_planner.rs b/crates/devtools/src/bin/asap_planner.rs index 5ce3e5cfe..0abacd83f 100644 --- a/crates/devtools/src/bin/asap_planner.rs +++ b/crates/devtools/src/bin/asap_planner.rs @@ -63,7 +63,7 @@ // `P1`, the selected plan as Stage 2 built it; // - stage3_plan_selection (from `stage3`): Stage 3's `Selection`: // `selected_dag` (`candidate_of_node`, `edges`, -// `node_of_output_reference`), `nodes_with_proven_non_negative_weights`, +// `node_of_output_reference`), // the plan's `cost` (its `usage`, `cost_per_second` and per-node // charges), `accepted_accuracy_per_query`, `pruned_candidates` and // `search_report`; `error` when no plan was selected. diff --git a/crates/planner/src/installable.rs b/crates/planner/src/installable.rs index 979469d51..359da15b6 100644 --- a/crates/planner/src/installable.rs +++ b/crates/planner/src/installable.rs @@ -717,8 +717,18 @@ fn tumbling_window_retention_of_expanded_tumbling_windows( if assignments.contains_key(&Rc::as_ptr(&node)) { continue; } - assignments.insert(Rc::as_ptr(&node), below.clone()); - pending.extend(node.children().into_iter().cloned()); + // A node of the realized plan (the chain's input) keeps its + // own assignment; only the new range and time shift run + // with the chain. + match id_of.get(&Rc::as_ptr(&node)) { + Some(&id) => { + assignments.insert(Rc::as_ptr(&node), assignment_of_node(id)); + } + None => { + assignments.insert(Rc::as_ptr(&node), below.clone()); + pending.extend(node.children().into_iter().cloned()); + } + } } } tumbling_windows_of_chain.push((chain_id, &expanded.tumbling_windows)); diff --git a/crates/stage2-physical-optimization/src/catalog/realize.rs b/crates/stage2-physical-optimization/src/catalog/realize.rs index d1d2f20bd..01129daae 100644 --- a/crates/stage2-physical-optimization/src/catalog/realize.rs +++ b/crates/stage2-physical-optimization/src/catalog/realize.rs @@ -14,7 +14,7 @@ use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode, QueryRoot, TimeR use asap_types::physical::{ExecutionTiming, Lifetime, PhysicalProperties, PhysicalWorkloadPlan}; use asap_types::stages::{ AssignmentRealizer, DagOfSelectedCandidates, MemoGroupId, PhysicalASAPMemoCandidates, - PhysicalCandidateError, RealizedPlan, WindowLayout, + PhysicalCandidateError, RealizedPlan, SelectedNodeId, WindowLayout, }; use asap_types::workload::bounds::confirmed_weights; use asap_types::workload::{DataWorkload, QueryWorkloadEntry}; @@ -37,7 +37,7 @@ impl AssignmentRealizer for Stage2AssignmentRealizer<'_> { /// of $W / w$ windows over the range's input, read through a /// `SummaryMerge` of the whole chain: the chain has the candidate's /// timing, lifetime and θ; the merge runs at query time unless every - /// consumer of the node runs at ingestion time. A summary build's update + /// consumer of the node runs at ingestion time and no query reads it. A summary build's update /// weights the data proves non-negative are confirmed /// (`bounds::confirmed_weights`). The query roots are the workload's /// roots over groups with each output reference replaced by its node. @@ -103,6 +103,9 @@ impl AssignmentRealizer for Stage2AssignmentRealizer<'_> { let merge_at_ingestion = candidate.physical_properties.timing() == ExecutionTiming::IngestionTime && !consumers[node].is_empty() + && !selected + .node_of_output_reference + .contains(&SelectedNodeId(node)) && consumers[node].iter().all(|&consumer| { physical.physical_candidates[selected.candidate_of_node[consumer].0] .physical_properties @@ -136,14 +139,14 @@ impl AssignmentRealizer for Stage2AssignmentRealizer<'_> { assignments.insert(Rc::as_ptr(&operator_node), assignment); built[node] = Some(operator_node); } - let operator_node_of_selected_node: Vec> = built + let built: Vec> = built .into_iter() .map(|node| node.expect("every node is built")) .collect(); let mut outputs = selected .node_of_output_reference .iter() - .map(|node| Rc::clone(&operator_node_of_selected_node[node.0])); + .map(|node| Rc::clone(&built[node.0])); let mut next_output = || { outputs .next() @@ -181,7 +184,6 @@ impl AssignmentRealizer for Stage2AssignmentRealizer<'_> { dag, }, tumbling_window_retention_ms, - operator_node_of_selected_node, }) } } diff --git a/crates/stage2-physical-optimization/src/catalog/tests.rs b/crates/stage2-physical-optimization/src/catalog/tests.rs index 12034d002..10f2aa7ca 100644 --- a/crates/stage2-physical-optimization/src/catalog/tests.rs +++ b/crates/stage2-physical-optimization/src/catalog/tests.rs @@ -859,7 +859,6 @@ fn the_realizer_exports_each_node_with_its_candidates_properties_and_theta() { let realized = realize(&physical, &selected).unwrap(); let dag = &realized.plan.dag; assert_eq!(dag.nodes.len(), 4); - assert_eq!(realized.operator_node_of_selected_node.len(), 4); let build = dag .nodes .iter() diff --git a/crates/stage3-plan-selection/src/candidate_cost.rs b/crates/stage3-plan-selection/src/candidate_cost.rs index 81975669d..8901197d5 100644 --- a/crates/stage3-plan-selection/src/candidate_cost.rs +++ b/crates/stage3-plan-selection/src/candidate_cost.rs @@ -1,6 +1,6 @@ //! $w_v$: the fixed cost per second of each physical candidate under cost //! model 1, from `asap-cost` (concepts doc, section 3, "Stage 3: baseline -//! selection and realization"). Each candidate is priced as one node of a +//! selection and realization"). Each candidate is costed as one node of a //! plan ([`asap_cost::node_usage`]) over one representative producer per //! input, at a fixed number of computations per second: //! @@ -16,17 +16,18 @@ //! execution contexts (#1207 PR 7) make the cost per context, and the //! retention amount $h_v$ the cost of an `AcrossEvaluations` candidate per //! selected consumer. The cost includes the candidate's own computation and -//! the raw data a query-time scan keeps on disk; the memory a query-time +//! the raw data a query-time scan retains on disk; the memory a query-time //! result holds while its evaluation runs is left out, since it depends on //! the latency of the queries reading it. use std::collections::BTreeSet; use asap_cost::execution_demand::evaluation_rate; -use asap_cost::{node_usage, CostInputs, CostModel, NodeUsage, ProducerOutput}; +use asap_cost::{node_usage, CostInputs, CostModel, NodeUsage, ProducerOutput, StateFamily}; use asap_types::ir::physical_export::{PhysicalASAPDAGNode, ScanWindow}; use asap_types::ir::properties::{DataPrimitive, ExecutionDataState}; -use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorResultKind}; +use asap_types::ir::schema::{ExactKind, SketchParams}; +use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorResultKind, TimeRangeKind}; use asap_types::physical::{ExecutionTiming, Lifetime}; use asap_types::stages::{ Memo, MemoGroupId, PhysicalASAPMemoCandidates, PhysicalCandidateId, PhysicalEdge, WindowLayout, @@ -35,8 +36,12 @@ use asap_types::workload::{DataWorkload, QueryWorkloadEntry}; /// $w_v$ of every physical candidate, by [`PhysicalCandidateId`]: its cost /// per second, or why `asap-cost` could not cost it (the selection MILP -/// leaves such a candidate out). A candidate one of whose inputs has no -/// costable producer is not costable either. +/// leaves such a candidate out). Candidates are costed producers first +/// (Kahn's order over the admissible edges): a candidate is costed once +/// each of its inputs has a costed producer, and its representative +/// producer per input is the first costed one, so the representatives +/// never form a cycle. A candidate one of whose inputs never gets a +/// costed producer is not costable either. pub(crate) fn candidate_costs( physical: &PhysicalASAPMemoCandidates, entries: &[QueryWorkloadEntry], @@ -54,129 +59,186 @@ pub(crate) fn candidate_costs( .children() .len() }; - // A representative producer per input, chosen among costed candidates - // only, so the choices never form a cycle: each round costs the - // candidates whose every input has a costed admissible producer. - let mut representative: Vec>> = vec![None; count]; - let mut costs: Vec>> = vec![None; count]; - let mut nodes: Vec> = vec![None; count]; - let mut order: Vec = Vec::new(); - loop { - let mut progressed = false; - for id in 0..count { - if costs[id].is_some() { - continue; - } - let producers: Option> = (0..input_count(id)) - .map(|input| { - physical - .admissible_edges_into(PhysicalCandidateId(id), input) - .iter() - .map(|&edge| edges[edge.0].producer_candidate_id.0) - .find(|&producer| matches!(costs[producer], Some(Ok(_)))) - }) - .collect(); - let Some(producers) = producers else { - continue; - }; - let node = candidate_node(physical, id, &producers); - nodes[id] = Some(node); - representative[id] = Some(producers); - // Placeholder until the usage pass below; marks it costable for - // the choice of its consumers' representatives. - costs[id] = Some(Ok(0.0)); - order.push(id); - progressed = true; - } - if !progressed { - break; - } - } - let nodes: Vec> = nodes; - let mut usages: Vec>> = (0..count).map(|_| None).collect(); let mut result: Vec> = (0..count) .map(|_| Err("one of its inputs has no producer that can be costed".to_string())) .collect(); - for &id in &order { - let node = nodes[id] - .as_ref() - .expect("built with its representative producers"); - let producers: Vec = representative[id].clone().unwrap_or_default(); - // A tumbling-window chain reads the input below its range: one - // level further down its range's representative producer. - let producers: Vec = match &node.payload { - Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) => producers - .iter() - .map(|&range| { - representative[range] - .as_ref() - .map_or(range, |below| below[0]) - }) - .collect(), - _ => producers, - }; - let Some(producer_outputs) = producers - .iter() - .map(|&producer| { - let usage = usages[producer].as_ref()?; - let producer_node = nodes[producer].as_ref()?; - Some(ProducerOutput { - payload: &producer_node.payload, - output_schema: &producer_node.output_schema, - output: usage.output, - sources: usage.sources.clone(), - family: usage.family, - unchanged_columns: usage.unchanged_columns.clone(), - }) + let mut costed = vec![false; count]; + // Per candidate, its inputs that have no costed producer yet. + let mut input_without_producer: Vec> = + (0..count).map(|id| vec![true; input_count(id)]).collect(); + let mut missing: Vec = (0..count).map(input_count).collect(); + let mut ready: Vec = (0..count).filter(|&id| missing[id] == 0).collect(); + // Costed nodes and usages, in costing order, by candidate. + let mut representative: Vec> = vec![Vec::new(); count]; + let mut node_of: Vec> = vec![None; count]; + let mut nodes: Vec = Vec::new(); + let mut usage_outputs: Vec = Vec::new(); + while let Some(id) = ready.pop() { + let producers: Vec = (0..input_count(id)) + .map(|input| { + physical + .admissible_edges_into(PhysicalCandidateId(id), input) + .iter() + .map(|&edge| edges[edge.0].producer_candidate_id.0) + .find(|&producer| costed[producer]) + .expect("ready: every input has a costed producer") }) - .collect::>>() - else { - continue; - }; - let candidate = &physical.physical_candidates[id]; - let group = candidate.logical_candidate_id.group; - let per_second = match computations_per_second(memo, group, candidate, entries, cost_inputs) - { - Ok(per_second) => per_second, + .collect(); + let costing = cost_candidate( + physical, + id, + &producers, + &representative, + &node_of, + &nodes, + &usage_outputs, + entries, + data, + cost_inputs, + formula_scale, + ); + representative[id] = producers; + let (node, usage) = match costing { + Ok(costed_node) => costed_node, Err(error) => { result[id] = Err(error); continue; } }; - let window_ms = window_ms_below(memo, group); - let usage = node_usage( - node, - &producer_outputs, - (1, per_second), - scan_window_of(memo, group), - asap_cost::tumbling_windows_of(node), - (candidate.physical_properties.timing() == ExecutionTiming::IngestionTime) - .then_some(window_ms) - .flatten(), - data, - cost_inputs, - formula_scale, - ); - match usage { - Ok(usage) => { - result[id] = cost_per_second(&usage, cost_inputs); - usages[id] = Some(usage); + result[id] = Ok(usage.cost_per_second); + node_of[id] = Some(nodes.len()); + nodes.push(node); + usage_outputs.push(usage); + costed[id] = true; + for &edge in physical.admissible_edges_out_of(PhysicalCandidateId(id)) { + let edge = &edges[edge.0]; + let (consumer, input) = (edge.consumer_candidate_id.0, edge.input_index_of_consumer); + if std::mem::take(&mut input_without_producer[consumer][input]) { + missing[consumer] -= 1; + if missing[consumer] == 0 { + ready.push(consumer); + } } - Err(error) => result[id] = Err(error.to_string()), } } result } +/// What a costed candidate's consumers read of it ([`ProducerOutput`] +/// without its borrowed node), and its $w_v$. +struct UsageOutput { + output: asap_cost::EdgeStatistics, + sources: BTreeSet, + family: OwnedStateFamily, + unchanged_columns: BTreeSet, + cost_per_second: f64, +} + +/// Cost candidate `id` over `producers` (its input `i` read from candidate +/// `producers[i]`, already costed: `nodes[node_of[u]]` and +/// `usage_outputs[node_of[u]]`): its node and what its consumers read of it. +#[allow(clippy::too_many_arguments)] +fn cost_candidate( + physical: &PhysicalASAPMemoCandidates, + id: usize, + producers: &[usize], + representative: &[Vec], + node_of: &[Option], + nodes: &[PhysicalASAPDAGNode], + usage_outputs: &[UsageOutput], + entries: &[QueryWorkloadEntry], + data: &DataWorkload, + cost_inputs: &CostInputs, + formula_scale: f64, +) -> Result<(PhysicalASAPDAGNode, UsageOutput), String> { + let memo = &physical.logical_candidates.memo; + let node = candidate_node(physical, id, producers)?; + // A tumbling-window chain reads the input below its range: the range's + // own representative producer, as the realizer builds it. + let producers: Vec = match &node.payload { + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) => producers + .iter() + .map(|&range| representative[range].first().copied().unwrap_or(range)) + .collect(), + _ => producers.to_vec(), + }; + let producer_outputs: Vec> = producers + .iter() + .map(|&producer| { + let index = node_of[producer].expect("costed"); + let output = &usage_outputs[index]; + ProducerOutput { + payload: &nodes[index].payload, + output_schema: &nodes[index].output_schema, + output: output.output, + sources: output.sources.iter().map(String::as_str).collect(), + family: match &output.family { + OwnedStateFamily::None => StateFamily::None, + OwnedStateFamily::Sketch(parameters) => StateFamily::Sketch(parameters), + OwnedStateFamily::Exact(kind) => StateFamily::Exact(kind), + }, + unchanged_columns: output + .unchanged_columns + .iter() + .map(String::as_str) + .collect(), + } + }) + .collect(); + let candidate = &physical.physical_candidates[id]; + let group = candidate.logical_candidate_id.group; + let per_second = computations_per_second(memo, group, candidate, entries, cost_inputs)?; + let usage = node_usage( + &node, + &producer_outputs, + (1, per_second), + scan_window_of(memo, group), + asap_cost::tumbling_windows_of(&node), + (candidate.physical_properties.timing() == ExecutionTiming::IngestionTime) + .then(|| window_ms_below(memo, group)) + .flatten(), + data, + cost_inputs, + formula_scale, + ) + .map_err(|error| error.to_string())?; + let cost_per_second = cost_per_second(&usage, cost_inputs)?; + let output = UsageOutput { + output: usage.output, + sources: usage.sources.iter().map(|s| s.to_string()).collect(), + family: match usage.family { + StateFamily::None => OwnedStateFamily::None, + StateFamily::Sketch(parameters) => OwnedStateFamily::Sketch(parameters.clone()), + StateFamily::Exact(kind) => OwnedStateFamily::Exact(*kind), + }, + unchanged_columns: usage + .unchanged_columns + .iter() + .map(|s| s.to_string()) + .collect(), + cost_per_second, + }; + Ok((node, output)) +} + +/// [`StateFamily`] owned, so a costed candidate outlives its node's borrow. +enum OwnedStateFamily { + None, + Sketch(SketchParams), + Exact(ExactKind), +} + /// Candidate `id` as one exported plan node over `producers` (its input /// `i` read from the node of candidate `producers[i]`): its operator, or /// for a tumbling-window layout the chain the realizer builds; its timing, -/// lifetime and θ; its group's schema. +/// lifetime and θ; its group's schema. Err for a tumbling-window layout the +/// realizer would refuse: not on a build over a range whose window its width +/// divides. fn candidate_node( physical: &PhysicalASAPMemoCandidates, id: usize, producers: &[usize], -) -> PhysicalASAPDAGNode { +) -> Result { let memo = &physical.logical_candidates.memo; let candidate = &physical.physical_candidates[id]; let logical = memo.candidate(candidate.logical_candidate_id); @@ -198,8 +260,27 @@ fn candidate_node( filter, }), ) => { - let window_ms = - window_ms_below(memo, candidate.logical_candidate_id.group).unwrap_or(width_ms); + let range_input = match &logical.expression.operator { + Operator::ASAP(ASAPOp::SummaryAgg { child, .. }) => *child, + _ => unreachable!("matched a build"), + }; + let window_ms = match &memo.groups()[range_input.0 as usize].expression { + Operator::NonASAP(NonASAPOp::TimeRange { + range, + kind: TimeRangeKind::Range, + .. + }) => u64::try_from(range.as_millis()).unwrap_or(u64::MAX), + _ => { + return Err( + "a tumbling-window layout on a build that does not read a range".into(), + ) + } + }; + if width_ms == 0 || window_ms % width_ms != 0 { + return Err(format!( + "tumbling windows {width_ms} ms wide do not divide the range of {window_ms} ms" + )); + } Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { child, family, @@ -208,12 +289,14 @@ fn candidate_node( grouping, filter, tumbling_window_width_ms: width_ms, - tumbling_window_count: u32::try_from(window_ms / width_ms.max(1)) - .unwrap_or(u32::MAX) - .max(1), + tumbling_window_count: u32::try_from(window_ms / width_ms) + .map_err(|_| "too many tumbling windows".to_string())?, }) } - (_, operator) => operator, + (Some(_), _) => { + return Err("a tumbling-window layout on an operator other than a summary build".into()) + } + (None, operator) => operator, }; // As the realizer's export sets it: a summary state or a maintained // population is state, everything else raw values. @@ -229,7 +312,7 @@ fn candidate_node( } _ => DataPrimitive::Raw, }; - PhysicalASAPDAGNode { + Ok(PhysicalASAPDAGNode { id, payload, output_state: ExecutionDataState { @@ -242,7 +325,7 @@ fn candidate_node( guarantee: None, retention_ms: None, summary_coverage: None, - } + }) } /// How often `candidate` of `group` is computed per second (module doc). @@ -279,10 +362,10 @@ fn computations_per_second( } /// $w_v$ from a candidate's usage: its cost per second under cost model 1 -/// plus the raw data a query-time scan keeps on disk. +/// plus the raw data a query-time scan retains on disk. fn cost_per_second(usage: &NodeUsage<'_>, cost_inputs: &CostInputs) -> Result { let CostModel::UsageOverTime(unit_cost) = &cost_inputs.cost_model else { - return Err("the selection MILP prices under cost model 1 only".into()); + return Err("the selection MILP costs under cost model 1 only".into()); }; let own = usage .cost diff --git a/crates/stage3-plan-selection/src/milp/model.rs b/crates/stage3-plan-selection/src/milp/model.rs index 8f1e78417..767aee908 100644 --- a/crates/stage3-plan-selection/src/milp/model.rs +++ b/crates/stage3-plan-selection/src/milp/model.rs @@ -49,7 +49,8 @@ pub(crate) struct SelectionVariables { /// What the baseline MILP is built over: Stage 2's candidates and /// admissible edges, each candidate's $w_v$ (`None`: not costable, so -/// $x_v = 0$), and $G_q$ for each query's output groups. +/// $x_v = 0$), each query's entry, and indexes built once +/// ([`Self::new`]). pub(crate) struct SelectionInputs<'a> { pub(crate) physical: &'a PhysicalASAPMemoCandidates, /// $w_v$ per second, by [`PhysicalCandidateId`]. @@ -57,12 +58,48 @@ pub(crate) struct SelectionInputs<'a> { /// Each query's workload entry, whose recurrence and range evaluation /// are its evaluation context. pub(crate) entries: &'a [QueryWorkloadEntry], + /// $\mathcal E$, by [`PhysicalEdgeId`]. + edges: Vec, + /// The physical candidates of each memo group, ascending. + candidates_of_group: HashMap>, + /// Each query's output groups $G_q$, each pair once, in + /// output-reference order. + query_output_groups: Vec<(QueryId, MemoGroupId)>, } -impl SelectionInputs<'_> { +impl<'a> SelectionInputs<'a> { + pub(crate) fn new( + physical: &'a PhysicalASAPMemoCandidates, + cost_per_second: &'a [Option], + entries: &'a [QueryWorkloadEntry], + ) -> Self { + let mut candidates_of_group: HashMap> = + HashMap::new(); + for (id, candidate) in physical.physical_candidates.iter().enumerate() { + candidates_of_group + .entry(candidate.logical_candidate_id.group) + .or_default() + .push(PhysicalCandidateId(id)); + } + let mut seen = std::collections::HashSet::new(); + let query_output_groups = output_references(&physical.query_roots) + .into_iter() + .map(|(query, group)| (QueryId(query), group)) + .filter(|pair| seen.insert(*pair)) + .collect(); + Self { + physical, + cost_per_second, + entries, + edges: physical.edges().map(|(_, edge)| edge).collect(), + candidates_of_group, + query_output_groups, + } + } + /// $\mathcal E$, by [`PhysicalEdgeId`]. - fn edges(&self) -> Vec { - self.physical.edges().map(|(_, edge)| edge).collect() + fn edges(&self) -> &[PhysicalEdge] { + &self.edges } /// $|\mathcal I(c)|$: the inputs of candidate `c`. @@ -113,27 +150,16 @@ impl SelectionInputs<'_> { /// The physical candidates of memo group `group`, ascending. fn candidates_of_group(&self, group: MemoGroupId) -> Vec { - (0..self.physical.physical_candidates.len()) - .map(PhysicalCandidateId) - .filter(|id| { - self.physical.physical_candidates[id.0] - .logical_candidate_id - .group - == group - }) - .collect() + self.candidates_of_group + .get(&group) + .cloned() + .unwrap_or_default() } /// Each query's output groups $G_q$, each pair once, in output-reference /// order: a scalar query reads one per operator reference. fn query_output_groups(&self) -> Vec<(QueryId, MemoGroupId)> { - let mut pairs: Vec<(QueryId, MemoGroupId)> = Vec::new(); - for (query, group) in output_references(&self.physical.query_roots) { - if !pairs.contains(&(QueryId(query), group)) { - pairs.push((QueryId(query), group)); - } - } - pairs + self.query_output_groups.clone() } } diff --git a/crates/stage3-plan-selection/src/milp/model/tests.rs b/crates/stage3-plan-selection/src/milp/model/tests.rs index 68ab6194d..cc5b4ee31 100644 --- a/crates/stage3-plan-selection/src/milp/model/tests.rs +++ b/crates/stage3-plan-selection/src/milp/model/tests.rs @@ -175,11 +175,7 @@ fn solve( costs: &[Option], entries: &[QueryWorkloadEntry], ) -> Result { - solve_selection(&SelectionInputs { - physical, - cost_per_second: costs, - entries, - }) + solve_selection(&SelectionInputs::new(physical, costs, entries)) } /// The selected candidates, ascending. @@ -347,11 +343,7 @@ fn reference_minimum( costs: &[Option], entries: &[QueryWorkloadEntry], ) -> Option { - let inputs = SelectionInputs { - physical, - cost_per_second: costs, - entries, - }; + let inputs = SelectionInputs::new(physical, costs, entries); let edges = inputs.edges(); let producers = |c: PhysicalCandidateId, i: usize| -> Vec { physical diff --git a/crates/stage3-plan-selection/src/select.rs b/crates/stage3-plan-selection/src/select.rs index d4d418044..d3ea7cec4 100644 --- a/crates/stage3-plan-selection/src/select.rs +++ b/crates/stage3-plan-selection/src/select.rs @@ -1,4 +1,4 @@ -//! Stage 3, plan selection: [`select_plan`] prices every physical candidate +//! Stage 3, plan selection: [`select_plan`] costs every physical candidate //! ($w_v$, [`crate::candidate_cost`]), builds and solves the baseline //! selection MILP over Stage 2's candidates and admissible edges //! ([`crate::milp::model`]), decodes $\{x = 1\}$, $\{y = 1\}$ and @@ -37,7 +37,7 @@ const FIXED_COST_PER_CANDIDATE: &str = /// Stage 3, plan selection: select one plan among `physical`'s candidates /// (module documentation). The selection is realized through `realizer` -/// and priced under `cost_inputs` (cost model 1 only). `entries[q]` is +/// and costed under `cost_inputs` (cost model 1 only). `entries[q]` is /// query `q`'s workload entry; `data` the data workload with its bounds. /// /// Accuracy: until constraint (K) (#1207 PR 8), a query's accuracy rests on @@ -95,17 +95,13 @@ pub fn select_plan( .collect(); let cost_per_second: Vec> = costs.iter().map(|cost| cost.clone().ok()).collect(); - let solved = solve_selection(&SelectionInputs { - physical, - cost_per_second: &cost_per_second, - entries, - }) - .map_err(|error| match error { - SelectionMilpError::Infeasible => { - ErrorSelectingPlan::NoValidCandidate(vec![infeasible(&costs)]) - } - SelectionMilpError::Solver(reason) => ErrorSelectingPlan::Solver(reason), - })?; + let solved = solve_selection(&SelectionInputs::new(physical, &cost_per_second, entries)) + .map_err(|error| match error { + SelectionMilpError::Infeasible => { + ErrorSelectingPlan::NoValidCandidate(vec![infeasible(&costs)]) + } + SelectionMilpError::Solver(reason) => ErrorSelectingPlan::Solver(reason), + })?; let validation_started = std::time::Instant::now(); solved.selected_dag.validate(physical).map_err(|reason| { ErrorSelectingPlan::Solver(format!("the decoded selection is invalid: {reason}")) diff --git a/crates/types/src/ir/schema/sketch.rs b/crates/types/src/ir/schema/sketch.rs index b5b80cb7c..0288713ec 100644 --- a/crates/types/src/ir/schema/sketch.rs +++ b/crates/types/src/ir/schema/sketch.rs @@ -589,7 +589,7 @@ pub enum WeightDomain { /// `metric`, or plain sums of them (no rate, no other arithmetic), so they /// are non-negative when every raw data sample of `metric` is: `metric` is /// declared a counter, or declared non-negative. Logical optimization - /// records it without reading the data workload; plan selection confirms + /// records it without reading the data workload; the realizer confirms /// it from the declared source facts /// ([`SourceFacts::metric_type`](crate::workload::SourceFacts::metric_type), /// [`SourceFacts::values_non_negative`](crate::workload::SourceFacts::values_non_negative)). diff --git a/crates/types/src/stages/realizer.rs b/crates/types/src/stages/realizer.rs index 96ad555a1..3ae8025dc 100644 --- a/crates/types/src/stages/realizer.rs +++ b/crates/types/src/stages/realizer.rs @@ -2,10 +2,8 @@ //! selected candidates, and Stage 2 realizes it. Stage 3 receives a `&dyn AssignmentRealizer` from the facade, //! so it never depends on the Stage 2 crate. use std::collections::BTreeMap; -use std::rc::Rc; use crate::ir::physical_export::PhysicalASAPNodeId; -use crate::ir::OperatorNode; use crate::physical::PhysicalWorkloadPlan; use super::{DagOfSelectedCandidates, PhysicalASAPMemoCandidates}; @@ -34,11 +32,13 @@ impl From<&str> for PhysicalCandidateError { /// A realized selection (was `SelectedPhysicalCandidate`): the physical /// plan Stage 2 built for it, how long each materialized tumbling-window chain of that plan -/// keeps its tumbling windows, and the operator node each selected node became. +/// retains its tumbling windows. #[derive(Debug, Clone)] pub struct RealizedPlan { - /// The selection rebuilt as one physical plan, one operator node per - /// selected node, each with its candidate's timing and lifetime. + /// The selection rebuilt as one physical plan: one operator node per + /// selected node (a tumbling-window layout as its chain read through a + /// merge), each exported with its candidate's timing, lifetime, θ and + /// its group's summary coverage. pub plan: PhysicalWorkloadPlan, /// For each tumbling-window chain of `plan` materialized beyond one evaluation /// (maintained at ingestion time, or built at query time and materialized @@ -51,24 +51,17 @@ pub struct RealizedPlan { /// its tumbling windows; without an offset or range evaluations, just the window. /// Empty when the plan keeps no tumbling windows. pub tumbling_window_retention_ms: BTreeMap, - /// The operator node of `plan` each selected node became, indexed by - /// [`SelectedNodeId`](super::SelectedNodeId). - pub operator_node_of_selected_node: Vec>, } /// Realizes a [`DagOfSelectedCandidates`] through Stage 2, one plan node /// per selected node. Implemented by the -/// Stage 2 crate, which holds the planning context (the workload's -/// original query roots, its entries and data, and whether query-time -/// results are retained); the facade builds it, passes it to Stage 3, and -/// calls the same method to build the selected plan. +/// Stage 2 crate, which holds the planning context (the workload's entries +/// and data); the facade builds it, passes it to Stage 3, and calls the +/// same method to build the selected plan. /// -/// Guarantees: a returned result has one operator node per selected node, each -/// rebuilt over its selected producers with its identity kept, guarantees -/// recomposed, and every output contract (result kind and schema) met; a -/// realized assignment also passes Stage 2's timing and dependency rules -/// and the physical DAG validator, with every node keeping its -/// candidate's timing and lifetime. +/// Guarantees: a returned plan has one operator node per selected node, +/// each rebuilt over its selected producers, and passes the physical DAG +/// validator, with every node keeping its candidate's timing and lifetime. /// /// It checks structure and execution legality only. It does not check cost, /// and it does not check accuracy targets: Stage 3 does both. diff --git a/crates/types/src/workload/bounds.rs b/crates/types/src/workload/bounds.rs index 85b33b75c..380a6ee16 100644 --- a/crates/types/src/workload/bounds.rs +++ b/crates/types/src/workload/bounds.rs @@ -436,70 +436,6 @@ fn declared_counter(data: &DataWorkload, metric: &str) -> bool { .is_some_and(|(_, series_type)| series_type == Some(PrometheusMetricType::Counter)) } -/// `roots` with every open [`WeightDomain::RawDataSamplesOfMetric`] requirement -/// replaced by its answer from `data` ([`confirmed_weights`]): the form of a -/// selected plan the executor runs, which reads no data workload. The same -/// `Rc` where nothing changes. -pub fn with_confirmed_weights(roots: &[QueryRoot], data: &DataWorkload) -> Vec { - with_confirmed_weights_at(roots, data, &|_| true) -} - -/// [`with_confirmed_weights`] at the summaries `confirm_at` names only: the -/// selected plan with the weight proofs Stage 3 made for its candidates -/// applied (`Selection::nodes_with_proven_non_negative_weights`). -pub fn with_confirmed_weights_at( - roots: &[QueryRoot], - data: &DataWorkload, - confirm_at: &dyn Fn(&Rc) -> bool, -) -> Vec { - fn confirm( - node: &Rc, - data: &DataWorkload, - confirm_at: &dyn Fn(&Rc) -> bool, - cache: &mut HashMap<*const OperatorNode, Rc>, - ) -> Rc { - if let Some(done) = cache.get(&Rc::as_ptr(node)) { - return Rc::clone(done); - } - let mut changed = false; - let mut operator = node.operator.map_children(|child| { - let rebuilt = confirm(child, data, confirm_at, cache); - changed |= !Rc::ptr_eq(&rebuilt, child); - rebuilt - }); - if let Operator::ASAP( - ASAPOp::SummaryAgg { input, .. } | ASAPOp::SummaryTumblingWindowChain { input, .. }, - ) = &mut operator - { - let confirmed = confirmed_weights(&input.weight_domain, data); - if confirmed != input.weight_domain && confirm_at(node) { - input.weight_domain = confirmed; - changed = true; - } - } - let rebuilt = if changed { - Rc::new( - OperatorNode::with_schema(operator, node.schema.clone()) - .with_guarantee(node.guarantee.clone()), - ) - } else { - Rc::clone(node) - }; - cache.insert(Rc::as_ptr(node), Rc::clone(&rebuilt)); - rebuilt - } - let mut cache = HashMap::new(); - roots - .iter() - .map(|root| match root { - QueryRoot::Operator(n) => QueryRoot::Operator(confirm(n, data, confirm_at, &mut cache)), - QueryRoot::Scalar(expr) => QueryRoot::Scalar( - expr.map_operator_refs(&mut |n| confirm(n, data, confirm_at, &mut cache)), - ), - }) - .collect() -} - /// Rows per millisecond the sources in `sources` receive together, each /// at its own rate: `series_count` series each sampled every /// `sample_interval`, or, for a source that does not declare both, its From 09d69f1efa712e4f7e7e7cd3630c8f39bec1a6f0 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Sat, 7 Nov 2026 01:51:47 +0000 Subject: [PATCH 8/8] =?UTF-8?q?fix:=20the=20solver's=20bound=20only=20boun?= =?UTF-8?q?ds=20an=20exact=20objective;=20integration=20export=20gives=20e?= =?UTF-8?q?ach=20sketch=20build=20a=20listed=20=CE=B8=20(#1207=20PR=206)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- crates/integration-tests/src/lib.rs | 33 +++++++++++++++++++--- crates/stage3-plan-selection/src/select.rs | 9 +++--- 2 files changed, 34 insertions(+), 8 deletions(-) diff --git a/crates/integration-tests/src/lib.rs b/crates/integration-tests/src/lib.rs index fe8991e41..00a463638 100644 --- a/crates/integration-tests/src/lib.rs +++ b/crates/integration-tests/src/lib.rs @@ -101,10 +101,34 @@ pub mod physical_export { use asap_types::ir::OperatorNode; use std::rc::Rc; - /// Export the physical DAG of `root`. + /// Export the physical DAG of `root`, every node at query time and not + /// materialized. An operator tree carries no θ, so each sketch build + /// gets the first θ of its family's predefined list + /// (`asap_types::stages::parameter_list`), none for a family without a + /// list; a plan the realizer exported carries its selected θ instead. pub fn physical_asap_dag(root: &Rc) -> PhysicalASAPDAG { - asap_types::test_support::physical_asap_dag_at_query_time(root) - .expect("physical ASAP DAG export failed") + use asap_types::ir::schema::FieldDataType; + use asap_types::ir::{ASAPOp, Operator}; + let mut dag = asap_types::test_support::physical_asap_dag_at_query_time(root) + .expect("physical ASAP DAG export failed"); + for node in &mut dag.nodes { + if let Operator::ASAP( + ASAPOp::SummaryAgg { + family: FieldDataType::Sketch(algorithm, _), + .. + } + | ASAPOp::SummaryTumblingWindowChain { + family: FieldDataType::Sketch(algorithm, _), + .. + }, + ) = &node.payload + { + node.parameters = asap_types::stages::parameter_list(*algorithm) + .first() + .cloned(); + } + } + dag } } @@ -419,7 +443,8 @@ pub mod planning { root: Rc, accuracy: AccuracyTarget, ) -> Vec> { - require_composed_operator_candidates(run_all_stages(root, accuracy).logical_candidates()) + let stages = run_all_stages(root, accuracy); + require_composed_operator_candidates(stages.logical_candidates()) } /// [`logical_candidates`] with every output accepted whatever its diff --git a/crates/stage3-plan-selection/src/select.rs b/crates/stage3-plan-selection/src/select.rs index d3ea7cec4..caab227f4 100644 --- a/crates/stage3-plan-selection/src/select.rs +++ b/crates/stage3-plan-selection/src/select.rs @@ -156,10 +156,11 @@ pub fn select_plan( pruned_candidates, search_report: SearchReport { status: SearchStatus::Optimal, - cost_lower_bound_per_second: solved - .lower_bound_per_second - .is_finite() - .then_some(solved.lower_bound_per_second), + // The solver's bound is on the MILP's objective; it bounds the + // plan's cost only when the objective equals it. + cost_lower_bound_per_second: (inexact_objective.is_none() + && solved.lower_bound_per_second.is_finite()) + .then_some(solved.lower_bound_per_second), milp_variables: solved.variable_count, milp_constraints: solved.constraint_count, model_construction_ms: solved.construction_ms,