diff --git a/crates/planner/tests/execution_contexts.rs b/crates/planner/tests/execution_contexts.rs new file mode 100644 index 000000000..c1b22f292 --- /dev/null +++ b/crates/planner/tests/execution_contexts.rs @@ -0,0 +1,122 @@ +//! #1207 PR 7: Stage 3's objective over execution contexts against +//! `asap_cost::plan_cost` of the realized plan, end to end through the +//! facade. On small workloads whose plan meets none of the terms the +//! formulation leaves out (memory a query-time result holds while its +//! evaluation runs; see `SearchReport::inexact_objective`), the objective +//! equals `plan_cost`. + +use asap_cost::{CostInputs, CostModel, ResourceUnitCost}; +use asap_frontend_promql::promql_metric_names; +use asap_planner::{e2e_plan, FrontendInput, PlanOutput, PlanningModels, UserInput}; +use asap_types::workload::{ + DataArrival, DataWorkload, DurationMs, PlanningWorkload, Predictability, Query, QueryLanguage, + QueryRequirements, QueryWorkload, RepeatedDemand, RepeatingEntry, RepetitionInterval, + TimestampMs, +}; + +/// Exact PromQL queries, each `(query, interval in ms, phase in ms)`, over +/// stored samples (no ingestion pipeline): 1,000 series sampled every 15 s. +fn workload(queries: &[(&str, u32, u64)]) -> PlanningWorkload { + PlanningWorkload { + query_workload: QueryWorkload { + language: QueryLanguage::PromQL, + query_batch: None, + repeating_queries: Some( + queries + .iter() + .map(|&(query, interval_ms, phase_ms)| RepeatingEntry { + query: Query(query.into()), + demand: RepeatedDemand::FixedIntervalAt { + interval: RepetitionInterval(interval_ms), + evaluation_phase: TimestampMs(phase_ms), + }, + requirements: QueryRequirements::default(), + predictability: Predictability::Unknown, + range_evaluation: None, + }) + .collect(), + ), + }, + data_workload: Some(DataWorkload { + arrival: DataArrival::AtRest, + ..DataWorkload::sampled_every( + queries + .iter() + .flat_map(|(query, ..)| promql_metric_names(query).unwrap_or_default()), + DurationMs(15_000), + ) + .with_series_count(1_000) + }), + } +} + +async fn plan(queries: &[(&str, u32, u64)], cost_inputs: &CostInputs) -> PlanOutput { + let workload = workload(queries); + let mut models = PlanningModels::builtin(); + models.cost_inputs = cost_inputs; + let input = UserInput::new(&workload, FrontendInput::Promql, models); + e2e_plan(input).await.expect("workload plans") +} + +/// Cost model 1 with the illustrative unit costs, memory free: the memory a +/// query-time result holds while its evaluation runs, which needs query +/// latency (out of scope), costs nothing. +fn memory_free() -> CostInputs { + CostInputs { + cost_model: CostModel::UsageOverTime(ResourceUnitCost { + cost_per_memory_byte_second: 0.0, + ..ResourceUnitCost::illustrative() + }), + ..CostInputs::default() + } +} + +/// One query; two queries in two contexts (every 10 s and every 15 s) +/// over one scan of `latency`, each context running its own: in each, the +/// objective equals `plan_cost` (no `inexact_objective`) and the solver's +/// bound is reported. +#[tokio::test] +async fn the_objective_equals_plan_cost_on_small_workloads() { + let cost_inputs = memory_free(); + for queries in [ + vec![("sum(rate(latency[1m]))", 10_000, 0)], + vec![ + ("sum(rate(latency[1m]))", 10_000, 0), + ("max(rate(latency[5m]))", 15_000, 0), + ], + ] { + let output = plan(&queries, &cost_inputs).await; + let selection = output.selection.expect("a plan is selected"); + let report = &selection.search_report; + assert_eq!(report.inexact_objective, None, "{queries:?}: {report:?}"); + assert!( + report.cost_lower_bound_per_second.is_some(), + "{queries:?}: {report:?}" + ); + } +} + +/// With memory costed, the memory a query-time result holds while its +/// evaluation runs is in `plan_cost` but not in the objective: the +/// selection reports the objective as inexact and names that term. +#[tokio::test] +async fn held_memory_is_reported_as_an_inexact_term() { + let output = plan( + &[("sum(rate(latency[1m]))", 10_000, 0)], + &CostInputs::default(), + ) + .await; + let selection = output.selection.expect("a plan is selected"); + let inexact = selection + .search_report + .inexact_objective + .expect("held memory is costed"); + assert!(inexact.objective_per_second < inexact.plan_cost_per_second); + assert!( + inexact + .inexact_terms + .keys() + .any(|term| term.starts_with("memory a query-time result holds")), + "{inexact:?}" + ); +} diff --git a/crates/planner/tests/main.rs b/crates/planner/tests/main.rs index 9d06c8c46..59c6f63fb 100644 --- a/crates/planner/tests/main.rs +++ b/crates/planner/tests/main.rs @@ -6,6 +6,7 @@ mod time_series; mod columnar_batches; mod e2e_plan; +mod execution_contexts; mod runtime_install; mod sql_table_scans; mod summary_sharing; diff --git a/crates/stage3-plan-selection/src/candidate_cost.rs b/crates/stage3-plan-selection/src/candidate_cost.rs deleted file mode 100644 index 8901197d5..000000000 --- a/crates/stage3-plan-selection/src/candidate_cost.rs +++ /dev/null @@ -1,452 +0,0 @@ -//! $w_v$: the fixed cost per second of each physical candidate under cost -//! model 1, from `asap-cost` (concepts doc, section 3, "Stage 3: baseline -//! selection and realization"). Each candidate is costed as one node of a -//! plan ([`asap_cost::node_usage`]) over one representative producer per -//! input, at a fixed number of computations per second: -//! -//! - at ingestion time, over one second of arriving rows; -//! - at query time, materialized across evaluations, once per request of -//! the queries reaching its group (a materialized tumbling-window chain -//! builds only its newest window per request); -//! - at query time otherwise, once per evaluation of each schedule of the -//! queries reaching its group (`asap_cost::execution_demand::evaluation_rate`). -//! -//! It is the baseline's fixed-cost assumption: a candidate's cost does not -//! depend on which consumers select it. It is exact only when that holds; -//! execution contexts (#1207 PR 7) make the cost per context, and the -//! retention amount $h_v$ the cost of an `AcrossEvaluations` candidate per -//! selected consumer. The cost includes the candidate's own computation and -//! the raw data a query-time scan retains on disk; the memory a query-time -//! result holds while its evaluation runs is left out, since it depends on -//! the latency of the queries reading it. - -use std::collections::BTreeSet; - -use asap_cost::execution_demand::evaluation_rate; -use asap_cost::{node_usage, CostInputs, CostModel, NodeUsage, ProducerOutput, StateFamily}; -use asap_types::ir::physical_export::{PhysicalASAPDAGNode, ScanWindow}; -use asap_types::ir::properties::{DataPrimitive, ExecutionDataState}; -use asap_types::ir::schema::{ExactKind, SketchParams}; -use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorResultKind, TimeRangeKind}; -use asap_types::physical::{ExecutionTiming, Lifetime}; -use asap_types::stages::{ - Memo, MemoGroupId, PhysicalASAPMemoCandidates, PhysicalCandidateId, PhysicalEdge, WindowLayout, -}; -use asap_types::workload::{DataWorkload, QueryWorkloadEntry}; - -/// $w_v$ of every physical candidate, by [`PhysicalCandidateId`]: its cost -/// per second, or why `asap-cost` could not cost it (the selection MILP -/// leaves such a candidate out). Candidates are costed producers first -/// (Kahn's order over the admissible edges): a candidate is costed once -/// each of its inputs has a costed producer, and its representative -/// producer per input is the first costed one, so the representatives -/// never form a cycle. A candidate one of whose inputs never gets a -/// costed producer is not costable either. -pub(crate) fn candidate_costs( - physical: &PhysicalASAPMemoCandidates, - entries: &[QueryWorkloadEntry], - data: &DataWorkload, - cost_inputs: &CostInputs, - formula_scale: f64, -) -> Vec> { - let memo = &physical.logical_candidates.memo; - let count = physical.physical_candidates.len(); - let edges: Vec = physical.edges().map(|(_, edge)| edge).collect(); - let input_count = |id: usize| { - memo.candidate(physical.physical_candidates[id].logical_candidate_id) - .expression - .operator - .children() - .len() - }; - let mut result: Vec> = (0..count) - .map(|_| Err("one of its inputs has no producer that can be costed".to_string())) - .collect(); - let mut costed = vec![false; count]; - // Per candidate, its inputs that have no costed producer yet. - let mut input_without_producer: Vec> = - (0..count).map(|id| vec![true; input_count(id)]).collect(); - let mut missing: Vec = (0..count).map(input_count).collect(); - let mut ready: Vec = (0..count).filter(|&id| missing[id] == 0).collect(); - // Costed nodes and usages, in costing order, by candidate. - let mut representative: Vec> = vec![Vec::new(); count]; - let mut node_of: Vec> = vec![None; count]; - let mut nodes: Vec = Vec::new(); - let mut usage_outputs: Vec = Vec::new(); - while let Some(id) = ready.pop() { - let producers: Vec = (0..input_count(id)) - .map(|input| { - physical - .admissible_edges_into(PhysicalCandidateId(id), input) - .iter() - .map(|&edge| edges[edge.0].producer_candidate_id.0) - .find(|&producer| costed[producer]) - .expect("ready: every input has a costed producer") - }) - .collect(); - let costing = cost_candidate( - physical, - id, - &producers, - &representative, - &node_of, - &nodes, - &usage_outputs, - entries, - data, - cost_inputs, - formula_scale, - ); - representative[id] = producers; - let (node, usage) = match costing { - Ok(costed_node) => costed_node, - Err(error) => { - result[id] = Err(error); - continue; - } - }; - result[id] = Ok(usage.cost_per_second); - node_of[id] = Some(nodes.len()); - nodes.push(node); - usage_outputs.push(usage); - costed[id] = true; - for &edge in physical.admissible_edges_out_of(PhysicalCandidateId(id)) { - let edge = &edges[edge.0]; - let (consumer, input) = (edge.consumer_candidate_id.0, edge.input_index_of_consumer); - if std::mem::take(&mut input_without_producer[consumer][input]) { - missing[consumer] -= 1; - if missing[consumer] == 0 { - ready.push(consumer); - } - } - } - } - result -} - -/// What a costed candidate's consumers read of it ([`ProducerOutput`] -/// without its borrowed node), and its $w_v$. -struct UsageOutput { - output: asap_cost::EdgeStatistics, - sources: BTreeSet, - family: OwnedStateFamily, - unchanged_columns: BTreeSet, - cost_per_second: f64, -} - -/// Cost candidate `id` over `producers` (its input `i` read from candidate -/// `producers[i]`, already costed: `nodes[node_of[u]]` and -/// `usage_outputs[node_of[u]]`): its node and what its consumers read of it. -#[allow(clippy::too_many_arguments)] -fn cost_candidate( - physical: &PhysicalASAPMemoCandidates, - id: usize, - producers: &[usize], - representative: &[Vec], - node_of: &[Option], - nodes: &[PhysicalASAPDAGNode], - usage_outputs: &[UsageOutput], - entries: &[QueryWorkloadEntry], - data: &DataWorkload, - cost_inputs: &CostInputs, - formula_scale: f64, -) -> Result<(PhysicalASAPDAGNode, UsageOutput), String> { - let memo = &physical.logical_candidates.memo; - let node = candidate_node(physical, id, producers)?; - // A tumbling-window chain reads the input below its range: the range's - // own representative producer, as the realizer builds it. - let producers: Vec = match &node.payload { - Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) => producers - .iter() - .map(|&range| representative[range].first().copied().unwrap_or(range)) - .collect(), - _ => producers.to_vec(), - }; - let producer_outputs: Vec> = producers - .iter() - .map(|&producer| { - let index = node_of[producer].expect("costed"); - let output = &usage_outputs[index]; - ProducerOutput { - payload: &nodes[index].payload, - output_schema: &nodes[index].output_schema, - output: output.output, - sources: output.sources.iter().map(String::as_str).collect(), - family: match &output.family { - OwnedStateFamily::None => StateFamily::None, - OwnedStateFamily::Sketch(parameters) => StateFamily::Sketch(parameters), - OwnedStateFamily::Exact(kind) => StateFamily::Exact(kind), - }, - unchanged_columns: output - .unchanged_columns - .iter() - .map(String::as_str) - .collect(), - } - }) - .collect(); - let candidate = &physical.physical_candidates[id]; - let group = candidate.logical_candidate_id.group; - let per_second = computations_per_second(memo, group, candidate, entries, cost_inputs)?; - let usage = node_usage( - &node, - &producer_outputs, - (1, per_second), - scan_window_of(memo, group), - asap_cost::tumbling_windows_of(&node), - (candidate.physical_properties.timing() == ExecutionTiming::IngestionTime) - .then(|| window_ms_below(memo, group)) - .flatten(), - data, - cost_inputs, - formula_scale, - ) - .map_err(|error| error.to_string())?; - let cost_per_second = cost_per_second(&usage, cost_inputs)?; - let output = UsageOutput { - output: usage.output, - sources: usage.sources.iter().map(|s| s.to_string()).collect(), - family: match usage.family { - StateFamily::None => OwnedStateFamily::None, - StateFamily::Sketch(parameters) => OwnedStateFamily::Sketch(parameters.clone()), - StateFamily::Exact(kind) => OwnedStateFamily::Exact(*kind), - }, - unchanged_columns: usage - .unchanged_columns - .iter() - .map(|s| s.to_string()) - .collect(), - cost_per_second, - }; - Ok((node, output)) -} - -/// [`StateFamily`] owned, so a costed candidate outlives its node's borrow. -enum OwnedStateFamily { - None, - Sketch(SketchParams), - Exact(ExactKind), -} - -/// Candidate `id` as one exported plan node over `producers` (its input -/// `i` read from the node of candidate `producers[i]`): its operator, or -/// for a tumbling-window layout the chain the realizer builds; its timing, -/// lifetime and θ; its group's schema. Err for a tumbling-window layout the -/// realizer would refuse: not on a build over a range whose window its width -/// divides. -fn candidate_node( - physical: &PhysicalASAPMemoCandidates, - id: usize, - producers: &[usize], -) -> Result { - let memo = &physical.logical_candidates.memo; - let candidate = &physical.physical_candidates[id]; - let logical = memo.candidate(candidate.logical_candidate_id); - let group = &memo.groups()[candidate.logical_candidate_id.group.0 as usize]; - let mut next = producers.iter(); - let operator = logical - .expression - .operator - .map_children(|_| *next.next().expect("one producer per input")); - let payload = match (logical.logical_properties.window_layout(), operator) { - ( - Some(&WindowLayout::Tumbling { width_ms }), - Operator::ASAP(ASAPOp::SummaryAgg { - child, - family, - input, - reduction, - grouping, - filter, - }), - ) => { - let range_input = match &logical.expression.operator { - Operator::ASAP(ASAPOp::SummaryAgg { child, .. }) => *child, - _ => unreachable!("matched a build"), - }; - let window_ms = match &memo.groups()[range_input.0 as usize].expression { - Operator::NonASAP(NonASAPOp::TimeRange { - range, - kind: TimeRangeKind::Range, - .. - }) => u64::try_from(range.as_millis()).unwrap_or(u64::MAX), - _ => { - return Err( - "a tumbling-window layout on a build that does not read a range".into(), - ) - } - }; - if width_ms == 0 || window_ms % width_ms != 0 { - return Err(format!( - "tumbling windows {width_ms} ms wide do not divide the range of {window_ms} ms" - )); - } - Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { - child, - family, - input, - reduction, - grouping, - filter, - tumbling_window_width_ms: width_ms, - tumbling_window_count: u32::try_from(window_ms / width_ms) - .map_err(|_| "too many tumbling windows".to_string())?, - }) - } - (Some(_), _) => { - return Err("a tumbling-window layout on an operator other than a summary build".into()) - } - (None, operator) => operator, - }; - // As the realizer's export sets it: a summary state or a maintained - // population is state, everything else raw values. - let primitive = match &payload { - Operator::ASAP( - ASAPOp::SummaryAgg { .. } - | ASAPOp::SummaryTumblingWindowChain { .. } - | ASAPOp::SummaryJoin { .. } - | ASAPOp::MaintainPopulation { .. }, - ) => DataPrimitive::SummaryState, - _ if group.group_properties.result_kind == OperatorResultKind::State => { - DataPrimitive::SummaryState - } - _ => DataPrimitive::Raw, - }; - Ok(PhysicalASAPDAGNode { - id, - payload, - output_state: ExecutionDataState { - timing: candidate.physical_properties.timing(), - primitive, - }, - lifetime: candidate.physical_properties.lifetime(), - parameters: logical.logical_properties.parameters().cloned(), - output_schema: group.group_properties.schema.clone(), - guarantee: None, - retention_ms: None, - summary_coverage: None, - }) -} - -/// How often `candidate` of `group` is computed per second (module doc). -fn computations_per_second( - memo: &Memo, - group: MemoGroupId, - candidate: &asap_types::stages::PhysicalCandidate, - entries: &[QueryWorkloadEntry], - cost_inputs: &CostInputs, -) -> Result { - let properties = candidate.physical_properties; - if properties.timing() == ExecutionTiming::IngestionTime { - return Ok(1.0); - } - let per_request = matches!(properties.lifetime(), Lifetime::AcrossEvaluations(_)); - evaluation_rate( - memo.groups()[group.0 as usize] - .queries - .iter() - .filter_map(|&query| entries.get(query)) - .map(|entry| { - ( - &entry.recurrence, - if per_request { - None - } else { - entry.range_evaluation - }, - ) - }), - cost_inputs.non_repeating_query_horizon_seconds, - ) - .map_err(|error| error.to_string()) -} - -/// $w_v$ from a candidate's usage: its cost per second under cost model 1 -/// plus the raw data a query-time scan retains on disk. -fn cost_per_second(usage: &NodeUsage<'_>, cost_inputs: &CostInputs) -> Result { - let CostModel::UsageOverTime(unit_cost) = &cost_inputs.cost_model else { - return Err("the selection MILP costs under cost model 1 only".into()); - }; - let own = usage - .cost - .cost_per_second - .ok_or("cost model 1 gives every node a cost per second")?; - let raw_data = usage.raw_scan.map_or(0.0, |(_, bytes)| { - unit_cost.cost_per_disk_byte_second * bytes as f64 - }); - let cost = own + raw_data; - if cost.is_finite() && cost >= 0.0 { - Ok(cost) - } else { - Err(format!( - "its cost per second {cost} is not a finite non-negative number" - )) - } -} - -/// The rows a query-time `Scan` of `group` reads, back from the evaluation -/// time: the union of the windows of the ranges over it, through time -/// shifts without an `@` anchor (`PhysicalASAPDAG::scan_window` on the -/// memo's groups instead of a plan's nodes). `Unbounded` when another -/// operator reads it or nothing does. -fn scan_window_of(memo: &Memo, group: MemoGroupId) -> ScanWindow { - fn windows( - memo: &Memo, - group: MemoGroupId, - end_offset_ms: i64, - seen: &mut BTreeSet, - ) -> ScanWindow { - if !seen.insert(group) { - return ScanWindow::Unbounded; - } - let parents = &memo.groups()[group.0 as usize].parents; - let mut union: Option = None; - for &parent in parents { - let window = match &memo.groups()[parent.0 as usize].expression { - Operator::NonASAP(NonASAPOp::TimeRange { range, .. }) => ScanWindow::Within { - end_offset_ms, - width_ms: i64::try_from(range.as_millis()).unwrap_or(i64::MAX), - }, - Operator::NonASAP(NonASAPOp::TimeShift { shift, .. }) if shift.at.is_none() => { - windows( - memo, - parent, - end_offset_ms.saturating_add(shift.offset_ms), - seen, - ) - } - _ => ScanWindow::Unbounded, - }; - union = Some(match union { - None => window, - Some(known) => known.union(window), - }); - } - union.unwrap_or(ScanWindow::Unbounded) - } - windows(memo, group, 0, &mut BTreeSet::new()) -} - -/// The window, in ms, of the first range below `group`'s operator, -/// through filters, projections and time shifts -/// (`asap_cost::maintained_window_ms` on the memo's groups): what an -/// ingestion-time grouping maintains, and a tumbling-window chain covers. -fn window_ms_below(memo: &Memo, group: MemoGroupId) -> Option { - let range_ms = |range: &std::time::Duration| u64::try_from(range.as_millis()).ok(); - let expression = &memo.groups()[group.0 as usize].expression; - if let Operator::NonASAP(NonASAPOp::TimeRange { range, .. }) = expression { - return range_ms(range); - } - let mut below = **expression.children().first()?; - for _ in 0..memo.groups().len() { - let expression = &memo.groups()[below.0 as usize].expression; - match expression { - Operator::NonASAP(NonASAPOp::TimeRange { range, .. }) => return range_ms(range), - Operator::NonASAP( - NonASAPOp::Filter { .. } | NonASAPOp::Project { .. } | NonASAPOp::TimeShift { .. }, - ) => below = **expression.children().first()?, - _ => return None, - } - } - None -} diff --git a/crates/stage3-plan-selection/src/execution_cost.rs b/crates/stage3-plan-selection/src/execution_cost.rs new file mode 100644 index 000000000..e8d4f2ad8 --- /dev/null +++ b/crates/stage3-plan-selection/src/execution_cost.rs @@ -0,0 +1,689 @@ +//! The costs of the selection MILP's objective under cost model 1, from +//! `asap-cost` (concepts doc, section 5, "Cost and retention"): +//! +//! $$\sum_v W^{\mathrm{mat}}_v x^{\mathrm{mat}}_v + \sum_{\kappa,v} +//! \rho_\kappa c_{\kappa,v} x_{\kappa,v} + \sum_v \pi_v h_v.$$ +//! +//! Each execution ([`Execution`]) is costed as one node of a plan +//! ([`asap_cost::node_usage`]) over one representative producer per input, +//! at its context's executions per second $\rho_\kappa$ +//! ([`executions_per_second`]): +//! +//! - $\rho_\kappa c_{\kappa,v}$: a candidate with $\ell$ = None or +//! `OneEvaluation` run in $\kappa$, including a `OneEvaluation` result's +//! storage within its evaluation and the raw data a query-time scan +//! retains on disk. A query-time scan reads the windows of the consumers +//! that may read it in $\kappa$. The range a tumbling-window chain is +//! built over costs nothing: the realizer builds the chain over the +//! range's input. +//! - $W^{\mathrm{mat}}_v$: an `AcrossEvaluations` candidate's own +//! computation, one build per tumbling window, retaining no more windows +//! than its range holds; $\pi_v$: the cost of retaining one more window. +//! So $W^{\mathrm{mat}}_v + \pi_v h_v$ is its cost retaining $h_v$ +//! windows. +//! +//! Query latency is ignored (decided 2026-11-06): a result counts as read +//! when its evaluation starts, so the memory a query-time result holds +//! while its evaluation runs, which `asap_cost::plan_cost` charges for the +//! latency of the queries reading it, is left out. + +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; + +use asap_cost::execution_demand::evaluation_rate; +use asap_cost::{node_usage, CostInputs, CostModel, NodeUsage, ProducerOutput, StateFamily}; +use asap_types::ir::physical_export::{PhysicalASAPDAGNode, ScanWindow}; +use asap_types::ir::properties::{DataPrimitive, ExecutionDataState}; +use asap_types::ir::schema::{ExactKind, SketchParams}; +use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorResultKind, TimeRangeKind}; +use asap_types::physical::{ExecutionTiming, Lifetime}; +use asap_types::stages::{ + ExecutionContext, ExecutionContextId, Memo, MemoGroupId, PhysicalASAPMemoCandidates, + PhysicalCandidateId, WindowLayout, +}; +use asap_types::workload::DataWorkload; + +use crate::milp::execution_contexts::{Execution, ExecutionContexts, TumblingWindowChainGeometry}; + +/// The objective's costs per second: of each execution that could be +/// costed, and why the others could not ([`execution_costs`]). +#[derive(Debug, Clone, Default)] +pub(crate) struct ExecutionCosts { + /// $\rho_\kappa c_{\kappa,v}$ of an [`Execution::InContext`]; + /// $W^{\mathrm{mat}}_v$ of an [`Execution::Materialized`]. + pub(crate) cost_per_second: HashMap, + /// $\pi_v$, per window retained, of each costed `AcrossEvaluations` + /// candidate. + pub(crate) retention_cost_per_window_per_second: HashMap, + /// Why `asap-cost` could not cost an execution (the MILP leaves it out). + pub(crate) uncostable: BTreeMap, + /// The producer execution each costed execution was costed over, per + /// input: its cost assumes that producer's output. + pub(crate) representative_producers: HashMap>, +} + +/// $\rho_\kappa$: the executions per second of `context`. An evaluation +/// runs at its queries' evaluation rate +/// (`asap_cost::execution_demand::evaluation_rate` of its recurrence and +/// range evaluation; a one-off query's invocations over the horizon); an +/// ingestion pipeline is costed over one second of arriving rows; an +/// `AcrossEvaluations` candidate's computation builds one tumbling window +/// per window width at query time, and is costed over one second of +/// arriving rows at ingestion time. +pub(crate) fn executions_per_second( + physical: &PhysicalASAPMemoCandidates, + context: &ExecutionContext, + cost_inputs: &CostInputs, +) -> Result { + match context { + ExecutionContext::Evaluation { + recurrence, + range_evaluation, + } => evaluation_rate( + [(recurrence, *range_evaluation)], + cost_inputs.non_repeating_query_horizon_seconds, + ) + .map_err(|error| error.to_string()), + ExecutionContext::IngestionPipeline(_) => Ok(1.0), + ExecutionContext::Materialized(v) => { + let candidate = &physical.physical_candidates[v.0]; + if candidate.physical_properties.timing() == ExecutionTiming::IngestionTime { + return Ok(1.0); + } + match physical + .logical_candidates + .memo + .candidate(candidate.logical_candidate_id) + .logical_properties + .window_layout() + { + Some(&WindowLayout::Tumbling { width_ms }) if width_ms > 0 => { + Ok(1_000.0 / width_ms as f64) + } + _ => Err("materialized across evaluations without tumbling windows".into()), + } + } + } +} + +/// The objective's costs of every execution of `contexts` ([`ExecutionCosts`]). +/// Executions are costed producers first (Kahn's order over the edges +/// $y_{\kappa,u,c,i}$ may select): an execution is costed once each of its +/// inputs has a costed producer, and its representative producer per input +/// is the first costed one, so the representatives never form a cycle. An +/// execution one of whose inputs never gets a costed producer is not +/// costable either. +pub(crate) fn execution_costs( + physical: &PhysicalASAPMemoCandidates, + contexts: &ExecutionContexts, + data: &DataWorkload, + cost_inputs: &CostInputs, + formula_scale: f64, +) -> ExecutionCosts { + let memo = &physical.logical_candidates.memo; + let executions = contexts.executions(); + let index: HashMap = executions + .iter() + .enumerate() + .map(|(index, &execution)| (execution, index)) + .collect(); + let input_count = |execution: Execution| { + memo.candidate(physical.physical_candidates[execution.candidate().0].logical_candidate_id) + .expression + .operator + .children() + .len() + }; + let edges: Vec<_> = physical.edges().map(|(_, edge)| edge).collect(); + // Per execution and input, the producer executions it may read; per + // execution, the (consumer, input) it may supply. + let mut producers_of: HashMap<(usize, usize), Vec> = HashMap::new(); + let mut supplies: Vec> = vec![Vec::new(); executions.len()]; + for &(context, edge) in contexts.edges() { + let edge = &edges[edge.0]; + let consumer = + index[&ExecutionContexts::execution_in(physical, context, edge.consumer_candidate_id)]; + let producer = + index[&ExecutionContexts::execution_in(physical, context, edge.producer_candidate_id)]; + producers_of + .entry((consumer, edge.input_index_of_consumer)) + .or_default() + .push(producer); + supplies[producer].push((consumer, edge.input_index_of_consumer)); + } + let rate_of_context: Vec> = contexts + .contexts() + .iter() + .map(|context| executions_per_second(physical, context, cost_inputs)) + .collect(); + // The groups with an execution in each context, and the range each + // tumbling-window chain reads in its computation context: the + // realizer builds the chain over the range's input, so the range is + // not run. + let groups_in_context: HashSet<(ExecutionContextId, MemoGroupId)> = executions + .iter() + .filter_map(|&execution| match execution { + Execution::InContext(context, v) => Some(( + context, + physical.physical_candidates[v.0].logical_candidate_id.group, + )), + Execution::Materialized(_) => None, + }) + .collect(); + let mut chain_range_of_context: HashMap> = + HashMap::new(); + for &execution in executions { + if let Execution::Materialized(v) = execution { + if let (Some(context), Some(chain)) = ( + contexts.computation_context(v), + TumblingWindowChainGeometry::of(memo, v, physical), + ) { + chain_range_of_context + .entry(context) + .or_default() + .push((chain.range_input, chain.width_ms)); + } + } + } + let mut costs = ExecutionCosts::default(); + let mut costed = vec![false; executions.len()]; + let mut input_without_producer: Vec> = executions + .iter() + .map(|&execution| vec![true; input_count(execution)]) + .collect(); + let mut missing: Vec = executions.iter().map(|&e| input_count(e)).collect(); + let mut ready: Vec = (0..executions.len()) + .filter(|&execution| missing[execution] == 0) + .collect(); + let mut representative: Vec> = vec![Vec::new(); executions.len()]; + let mut nodes: Vec> = + (0..executions.len()).map(|_| None).collect(); + let mut uncostable: Vec> = + vec![ + Some("one of its inputs has no producer that can be costed".to_string()); + executions.len() + ]; + while let Some(at) = ready.pop() { + let execution = executions[at]; + let producers: Vec = (0..input_count(execution)) + .map(|input| { + producers_of[&(at, input)] + .iter() + .copied() + .find(|&producer| costed[producer]) + .expect("ready: every input has a costed producer") + }) + .collect(); + let context = contexts + .input_context(execution) + .expect("a reached execution has a context"); + let costing = (|| { + let rate = rate_of_context[context.0].clone()?; + let candidate = execution.candidate(); + let group = physical.physical_candidates[candidate.0] + .logical_candidate_id + .group; + let chain_ranges = chain_range_of_context + .get(&context) + .map_or(&[][..], Vec::as_slice); + let scan_window = scan_window_in_context(memo, group, |parent| { + if let Some(&(_, width_ms)) = + chain_ranges.iter().find(|(range, _)| *range == parent) + { + return Some(Some(width_ms)); + } + groups_in_context + .contains(&(context, parent)) + .then_some(None) + }); + let bypassed = matches!(execution, Execution::InContext(..)) + && chain_ranges.iter().any(|(range, _)| *range == group); + cost_execution( + physical, + at, + candidate, + &producers, + &representative, + &nodes, + (rate, scan_window), + data, + cost_inputs, + formula_scale, + ) + .map(|(node, mut output, retention)| { + if bypassed { + output.cost_per_second = 0.0; + } + (node, output, retention) + }) + })(); + representative[at] = producers; + let (node, output, retention) = match costing { + Ok(costed_node) => costed_node, + Err(error) => { + uncostable[at] = Some(error); + continue; + } + }; + uncostable[at] = None; + costs + .cost_per_second + .insert(execution, output.cost_per_second); + costs.representative_producers.insert( + execution, + representative[at].iter().map(|&p| executions[p]).collect(), + ); + if let Some(retention) = retention { + costs + .retention_cost_per_window_per_second + .insert(execution.candidate(), retention); + } + nodes[at] = Some((node, output)); + costed[at] = true; + for &(consumer, input) in &supplies[at] { + if std::mem::take(&mut input_without_producer[consumer][input]) { + missing[consumer] -= 1; + if missing[consumer] == 0 { + ready.push(consumer); + } + } + } + } + costs.uncostable = executions + .iter() + .zip(uncostable) + .filter_map(|(&execution, reason)| reason.map(|reason| (execution, reason))) + .collect(); + costs +} + +/// What a costed execution's consumers read of it ([`ProducerOutput`] +/// without its borrowed node), and its cost per second at its context's +/// rate: $\rho_\kappa c_{\kappa,v}$, or $W^{\mathrm{mat}}_v$. +struct UsageOutput { + output: asap_cost::EdgeStatistics, + sources: BTreeSet, + family: OwnedStateFamily, + unchanged_columns: BTreeSet, + cost_per_second: f64, +} + +/// Cost execution `at` of `candidate` over `producers` (its input `i` read +/// from execution `producers[i]`, already costed: `nodes[producers[i]]`), +/// at `rate` executions per second, a query-time scan reading +/// `scan_window`: its node, what its consumers read of it, and for an +/// `AcrossEvaluations` candidate $\pi_v$ (its cost retaining one window +/// more than its range holds, less its cost retaining those). Its cost +/// retaining as many windows as its range holds is +/// $W^{\mathrm{mat}}_v + \pi_v \cdot$ that count: the objective costs +/// $h_v$ windows at $\pi_v$ each over a fixed cost +/// $W^{\mathrm{mat}}_v$ = that cost less $\pi_v$ times the count. +#[allow(clippy::too_many_arguments)] +fn cost_execution( + physical: &PhysicalASAPMemoCandidates, + at: usize, + candidate: PhysicalCandidateId, + producers: &[usize], + representative: &[Vec], + nodes: &[Option<(PhysicalASAPDAGNode, UsageOutput)>], + (rate, scan_window): (f64, ScanWindow), + data: &DataWorkload, + cost_inputs: &CostInputs, + formula_scale: f64, +) -> Result<(PhysicalASAPDAGNode, UsageOutput, Option), String> { + let memo = &physical.logical_candidates.memo; + let node = candidate_node(physical, candidate.0, at, producers)?; + // A tumbling-window chain reads the input below its range: the range's + // own representative producer, as the realizer builds it. + let producers: Vec = match &node.payload { + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) => producers + .iter() + .map(|&range| representative[range].first().copied().unwrap_or(range)) + .collect(), + _ => producers.to_vec(), + }; + let producer_outputs: Vec> = producers + .iter() + .map(|&producer| { + let (node, output) = nodes[producer].as_ref().expect("costed"); + ProducerOutput { + payload: &node.payload, + output_schema: &node.output_schema, + output: output.output, + sources: output.sources.iter().map(String::as_str).collect(), + family: match &output.family { + OwnedStateFamily::None => StateFamily::None, + OwnedStateFamily::Sketch(parameters) => StateFamily::Sketch(parameters), + OwnedStateFamily::Exact(kind) => StateFamily::Exact(kind), + }, + unchanged_columns: output + .unchanged_columns + .iter() + .map(String::as_str) + .collect(), + } + }) + .collect(); + let physical_candidate = &physical.physical_candidates[candidate.0]; + let group = physical_candidate.logical_candidate_id.group; + let ingestion_window_ms = (physical_candidate.physical_properties.timing() + == ExecutionTiming::IngestionTime) + .then(|| window_ms_below(memo, group)) + .flatten(); + let windows = asap_cost::tumbling_windows_of(&node); + let usage_retaining = |retained_windows: u64| { + node_usage( + &node, + &producer_outputs, + (1, rate), + scan_window, + retained_windows, + ingestion_window_ms, + data, + cost_inputs, + formula_scale, + ) + .map_err(|error| error.to_string()) + }; + let usage = usage_retaining(windows)?; + let cost = cost_per_second(&usage, cost_inputs)?; + let retention = match physical_candidate.physical_properties.lifetime() { + Lifetime::AcrossEvaluations(_) => { + let retaining_one_more = cost_per_second(&usage_retaining(windows + 1)?, cost_inputs)?; + Some(retaining_one_more - cost) + .filter(|cost| cost.is_finite() && *cost >= 0.0) + .map(Some) + .ok_or("its cost per retained window is not a finite non-negative number")? + } + Lifetime::None | Lifetime::OneEvaluation(_) => None, + }; + // W^mat_v: its cost retaining its range's windows, less those windows' + // retention, which pi_v h_v costs. + let cost = cost - retention.unwrap_or(0.0) * windows as f64; + let owned = |usage: &NodeUsage<'_>, cost_per_second| UsageOutput { + output: usage.output, + sources: usage.sources.iter().map(|s| s.to_string()).collect(), + family: match usage.family { + StateFamily::None => OwnedStateFamily::None, + StateFamily::Sketch(parameters) => OwnedStateFamily::Sketch(parameters.clone()), + StateFamily::Exact(kind) => OwnedStateFamily::Exact(*kind), + }, + unchanged_columns: usage + .unchanged_columns + .iter() + .map(|s| s.to_string()) + .collect(), + cost_per_second, + }; + if !matches!( + node.payload, + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) + ) { + let output = owned(&usage, cost); + return Ok((node, output, retention)); + } + // The realizer reads a chain through a merge of all its windows, at + // query time: its consumers read the merge. A chain run in its + // consumers' context (None or `OneEvaluation`) runs the merge there + // too; the merge over a chain materialized across evaluations runs + // once per context reading it, which no execution's cost can hold (an + // inexact term of the objective). + let merge = PhysicalASAPDAGNode { + id: at, + payload: Operator::ASAP(ASAPOp::SummaryMerge { + children: vec![at], + newest_tumbling_windows: None, + group_by: None, + }), + output_state: ExecutionDataState { + timing: ExecutionTiming::QueryTime, + primitive: DataPrimitive::SummaryState, + }, + lifetime: Lifetime::None, + parameters: None, + output_schema: node.output_schema.clone(), + guarantee: None, + retention_ms: None, + summary_coverage: None, + }; + let chain_output = owned(&usage, cost); + let chain = ProducerOutput { + payload: &node.payload, + output_schema: &node.output_schema, + output: usage.output, + sources: usage.sources.clone(), + family: usage.family, + unchanged_columns: usage.unchanged_columns.clone(), + }; + let merge_usage = node_usage( + &merge, + &[chain], + (1, rate), + ScanWindow::Unbounded, + 1, + None, + data, + cost_inputs, + formula_scale, + ) + .map_err(|error| error.to_string())?; + let merge_cost = match retention { + Some(_) => 0.0, + None => cost_per_second(&merge_usage, cost_inputs)?, + }; + let output = owned(&merge_usage, chain_output.cost_per_second + merge_cost); + Ok((merge, output, retention)) +} + +/// The rows a query-time `Scan` of `group` reads in a context, back from +/// the evaluation time: the union of the windows of the ranges over it +/// that run in the context, through time shifts without an `@` anchor +/// (`PhysicalASAPDAG::scan_window` on the memo's groups instead of a +/// plan's nodes). `in_context(parent)` is `None` for a parent group that +/// does not run in the context, `Some(Some(w))` for the range a +/// tumbling-window chain of width `w` is built over (it reads its newest +/// window), `Some(None)` otherwise. `Unbounded` when another operator in +/// the context reads it or none does. +fn scan_window_in_context( + memo: &Memo, + group: MemoGroupId, + in_context: impl Fn(MemoGroupId) -> Option>, +) -> ScanWindow { + fn windows( + memo: &Memo, + group: MemoGroupId, + end_offset_ms: i64, + in_context: &dyn Fn(MemoGroupId) -> Option>, + seen: &mut BTreeSet, + ) -> ScanWindow { + if !seen.insert(group) { + return ScanWindow::Unbounded; + } + let mut union: Option = None; + for &parent in &memo.groups()[group.0 as usize].parents { + let Some(chain_width_ms) = in_context(parent) else { + continue; + }; + let window = match &memo.groups()[parent.0 as usize].expression { + Operator::NonASAP(NonASAPOp::TimeRange { range, .. }) => ScanWindow::Within { + end_offset_ms, + width_ms: i64::try_from( + chain_width_ms + .unwrap_or(u64::try_from(range.as_millis()).unwrap_or(u64::MAX)), + ) + .unwrap_or(i64::MAX), + }, + Operator::NonASAP(NonASAPOp::TimeShift { shift, .. }) if shift.at.is_none() => { + windows( + memo, + parent, + end_offset_ms.saturating_add(shift.offset_ms), + in_context, + seen, + ) + } + _ => ScanWindow::Unbounded, + }; + union = Some(match union { + None => window, + Some(known) => known.union(window), + }); + } + union.unwrap_or(ScanWindow::Unbounded) + } + windows(memo, group, 0, &in_context, &mut BTreeSet::new()) +} + +/// [`StateFamily`] owned, so a costed candidate outlives its node's borrow. +enum OwnedStateFamily { + None, + Sketch(SketchParams), + Exact(ExactKind), +} + +/// Candidate `candidate_id` as plan node `node_id` over `producers` (its +/// input `i` read from node `producers[i]`): its operator, or +/// for a tumbling-window layout the chain the realizer builds; its timing, +/// lifetime and θ; its group's schema. Err for a tumbling-window layout the +/// realizer would refuse: not on a build over a range whose window its width +/// divides. +fn candidate_node( + physical: &PhysicalASAPMemoCandidates, + candidate_id: usize, + node_id: usize, + producers: &[usize], +) -> Result { + let memo = &physical.logical_candidates.memo; + let candidate = &physical.physical_candidates[candidate_id]; + let logical = memo.candidate(candidate.logical_candidate_id); + let group = &memo.groups()[candidate.logical_candidate_id.group.0 as usize]; + let mut next = producers.iter(); + let operator = logical + .expression + .operator + .map_children(|_| *next.next().expect("one producer per input")); + let payload = match (logical.logical_properties.window_layout(), operator) { + ( + Some(&WindowLayout::Tumbling { width_ms }), + Operator::ASAP(ASAPOp::SummaryAgg { + child, + family, + input, + reduction, + grouping, + filter, + }), + ) => { + let range_input = match &logical.expression.operator { + Operator::ASAP(ASAPOp::SummaryAgg { child, .. }) => *child, + _ => unreachable!("matched a build"), + }; + let window_ms = match &memo.groups()[range_input.0 as usize].expression { + Operator::NonASAP(NonASAPOp::TimeRange { + range, + kind: TimeRangeKind::Range, + .. + }) => u64::try_from(range.as_millis()).unwrap_or(u64::MAX), + _ => { + return Err( + "a tumbling-window layout on a build that does not read a range".into(), + ) + } + }; + if width_ms == 0 || window_ms % width_ms != 0 { + return Err(format!( + "tumbling windows {width_ms} ms wide do not divide the range of {window_ms} ms" + )); + } + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { + child, + family, + input, + reduction, + grouping, + filter, + tumbling_window_width_ms: width_ms, + tumbling_window_count: u32::try_from(window_ms / width_ms) + .map_err(|_| "too many tumbling windows".to_string())?, + }) + } + (Some(_), _) => { + return Err("a tumbling-window layout on an operator other than a summary build".into()) + } + (None, operator) => operator, + }; + // As the realizer's export sets it: a summary state or a maintained + // population is state, everything else raw values. + let primitive = match &payload { + Operator::ASAP( + ASAPOp::SummaryAgg { .. } + | ASAPOp::SummaryTumblingWindowChain { .. } + | ASAPOp::SummaryJoin { .. } + | ASAPOp::MaintainPopulation { .. }, + ) => DataPrimitive::SummaryState, + _ if group.group_properties.result_kind == OperatorResultKind::State => { + DataPrimitive::SummaryState + } + _ => DataPrimitive::Raw, + }; + Ok(PhysicalASAPDAGNode { + id: node_id, + payload, + output_state: ExecutionDataState { + timing: candidate.physical_properties.timing(), + primitive, + }, + lifetime: candidate.physical_properties.lifetime(), + parameters: logical.logical_properties.parameters().cloned(), + output_schema: group.group_properties.schema.clone(), + guarantee: None, + retention_ms: None, + summary_coverage: None, + }) +} + +/// An execution's cost per second from its usage under cost model 1, plus +/// the raw data a query-time scan retains on disk. +fn cost_per_second(usage: &NodeUsage<'_>, cost_inputs: &CostInputs) -> Result { + let CostModel::UsageOverTime(unit_cost) = &cost_inputs.cost_model else { + return Err("the selection MILP costs under cost model 1 only".into()); + }; + let own = usage + .cost + .cost_per_second + .ok_or("cost model 1 gives every node a cost per second")?; + let raw_data = usage.raw_scan.map_or(0.0, |(_, bytes)| { + unit_cost.cost_per_disk_byte_second * bytes as f64 + }); + let cost = own + raw_data; + if cost.is_finite() && cost >= 0.0 { + Ok(cost) + } else { + Err(format!( + "its cost per second {cost} is not a finite non-negative number" + )) + } +} + +/// The window, in ms, of the first range below `group`'s operator, +/// through filters, projections and time shifts +/// (`asap_cost::maintained_window_ms` on the memo's groups): what an +/// ingestion-time grouping maintains, and a tumbling-window chain covers. +fn window_ms_below(memo: &Memo, group: MemoGroupId) -> Option { + let range_ms = |range: &std::time::Duration| u64::try_from(range.as_millis()).ok(); + let expression = &memo.groups()[group.0 as usize].expression; + if let Operator::NonASAP(NonASAPOp::TimeRange { range, .. }) = expression { + return range_ms(range); + } + let mut below = **expression.children().first()?; + for _ in 0..memo.groups().len() { + let expression = &memo.groups()[below.0 as usize].expression; + match expression { + Operator::NonASAP(NonASAPOp::TimeRange { range, .. }) => return range_ms(range), + Operator::NonASAP( + NonASAPOp::Filter { .. } | NonASAPOp::Project { .. } | NonASAPOp::TimeShift { .. }, + ) => below = **expression.children().first()?, + _ => return None, + } + } + None +} diff --git a/crates/stage3-plan-selection/src/lib.rs b/crates/stage3-plan-selection/src/lib.rs index aef09d579..744552c43 100644 --- a/crates/stage3-plan-selection/src/lib.rs +++ b/crates/stage3-plan-selection/src/lib.rs @@ -3,18 +3,19 @@ //! selection and realization", and sections 4 and 5). //! //! [`select_plan`] reads Stage 2's physical candidates $\mathcal C$ and the -//! admissible part of the physical edges $\mathcal E$, selects with a MILP -//! over $x_v$, $y_{u,c,i}$, $z_{q,v}$ and $t_v$ (constraints (A)–(I); -//! (G) and (H) hold because $y$ exists only on admissible edges), and -//! decodes $\{x = 1\}$ and $\{y = 1\}$ into a -//! `DagOfSelectedCandidates`, which the injected realizer builds. The -//! extensions add execution contexts $\kappa$ with $x^{\mathrm{mat}}_v$, -//! $x_{\kappa,v}$, $y_{\kappa,u,c,i}$ and the retention amount $h_v$ -//! (section 5), and accuracy at each query root, constraint (K), with -//! per-candidate error bounds $E_c$ and $D_c$ (section 4). Costs come from -//! `asap-cost`. +//! admissible part of the physical edges $\mathcal E$, and selects with a +//! MILP over execution contexts $\kappa$ (section 5): $x^{\mathrm{mat}}_v$ +//! per `AcrossEvaluations` candidate, $x_{\kappa,v}$ per candidate with +//! $\ell$ = None or `OneEvaluation` and context, $y_{\kappa,u,c,i}$ per +//! edge and context of its consumer, $z_{q,v}$, $t$, and the retention +//! amount $h_v$; constraints (A)–(I) per execution ((G) and (H) hold because +//! $y$ exists only on admissible edges) and (R). It decodes $\{x = 1\}$ and +//! $\{y = 1\}$ into a `DagOfSelectedCandidates` (one node per selected +//! execution), which the injected realizer builds. Accuracy at each query +//! root, constraint (K), with per-candidate error bounds $E_c$ and $D_c$ +//! (section 4), is #1207 PR 8. Costs come from `asap-cost`. -mod candidate_cost; +mod execution_cost; mod milp; mod models; mod report; diff --git a/crates/stage3-plan-selection/src/milp/execution_contexts.rs b/crates/stage3-plan-selection/src/milp/execution_contexts.rs new file mode 100644 index 000000000..f74b68a8c --- /dev/null +++ b/crates/stage3-plan-selection/src/milp/execution_contexts.rs @@ -0,0 +1,393 @@ +//! Execution contexts $\kappa$ (concepts doc, section 5, "Execution +//! identity"): the distinct contexts of a workload, each candidate's +//! executions, and the edges $y_{\kappa,u,c,i}$ may select. +//! +//! - An `AcrossEvaluations` candidate $v$ has one execution, its own +//! computation ($x^{\mathrm{mat}}_v$). Its inputs run in its computation +//! context: the ingestion pipeline $I_r$ of its refresh window $r$ when it +//! is maintained at ingestion time with a bounded window, otherwise +//! $\kappa_v$ (`ExecutionContext::Materialized`). +//! - A candidate with $\ell$ = None or `OneEvaluation` has one execution per +//! context $\kappa$ that may read it ($x_{\kappa,v}$). A query's context is +//! the evaluation $\kappa_T$ of its recurrence and range evaluation, shared +//! by every query with equal ones. A chain of None and `OneEvaluation` +//! candidates inherits its consumer's context; an `AcrossEvaluations` +//! producer switches to its own computation context for its inputs. +//! A `OneEvaluation` result belongs to one evaluation, so it runs only in +//! evaluation contexts $\kappa_T$. +//! - An edge $(u, c, i)$ is selected per context of its consumer +//! ($y_{\kappa,u,c,i}$): within $\kappa$ when $u$ is None or +//! `OneEvaluation`; from $u$'s one execution when $u$ is +//! `AcrossEvaluations`, and then only if $u$ delivers on $\kappa$ +//! (`PhysicalCandidate::delivers_on`). +//! +//! Contexts are deduplicated by equality ([`ExecutionContext`] holds an +//! `f64` and cannot be hashed); [`ExecutionContextId`] indexes the list. + +use std::collections::{BTreeSet, HashMap}; + +use asap_types::ir::materialized_view::RefreshWindow; +use asap_types::ir::{ASAPOp, NonASAPOp, Operator, TimeRangeKind}; +use asap_types::physical::{ExecutionTiming, Lifetime}; +use asap_types::stages::{ + output_references, ExecutionContext, ExecutionContextId, Memo, MemoGroupId, + PhysicalASAPMemoCandidates, PhysicalCandidateId, PhysicalEdge, PhysicalEdgeId, RefreshWindowId, + WindowLayout, +}; +use asap_types::workload::QueryWorkloadEntry; + +/// One execution of a candidate: a node of the selected DAG. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub(crate) enum Execution { + /// An `AcrossEvaluations` candidate's own computation: $x^{\mathrm{mat}}_v$. + Materialized(PhysicalCandidateId), + /// A candidate with $\ell$ = None or `OneEvaluation` run in context + /// $\kappa$: $x_{\kappa,v}$. + InContext(ExecutionContextId, PhysicalCandidateId), +} + +impl Execution { + /// The candidate it runs. + pub(crate) fn candidate(self) -> PhysicalCandidateId { + match self { + Execution::Materialized(v) | Execution::InContext(_, v) => v, + } + } +} + +/// The distinct contexts of a workload and every execution and edge the +/// selection may choose ([`Self::new`]). +pub(crate) struct ExecutionContexts { + /// The distinct contexts, by [`ExecutionContextId`]. + contexts: Vec, + /// The distinct refresh windows of the ingestion pipelines, by + /// [`RefreshWindowId`]. + refresh_windows: Vec, + /// Each query's evaluation context $\kappa_T$, by query index. + context_of_query: Vec, + /// Each `AcrossEvaluations` candidate reached, with the context its + /// inputs run in. + computation_context: HashMap, + /// Every execution reached from a query output, ascending. + executions: Vec, + /// Every $(\kappa, (u, c, i))$ a consumer execution in $\kappa$ may + /// select, ascending. + edges: Vec<(ExecutionContextId, PhysicalEdgeId)>, +} + +impl ExecutionContexts { + /// The contexts, executions and edges of `physical` for the queries + /// `entries` (by index), reached from each query output: each candidate + /// of an output group runs in the query's context (an + /// `AcrossEvaluations` one if it delivers on it); each input of an + /// execution in $\kappa$ reads, over an admissible edge, a None or + /// `OneEvaluation` producer in $\kappa$ or an `AcrossEvaluations` + /// producer that delivers on $\kappa$; an `AcrossEvaluations` execution's + /// inputs run in its computation context. + pub(crate) fn new( + physical: &PhysicalASAPMemoCandidates, + entries: &[QueryWorkloadEntry], + ) -> Self { + let memo = &physical.logical_candidates.memo; + let mut contexts = Self { + contexts: Vec::new(), + refresh_windows: Vec::new(), + context_of_query: Vec::new(), + computation_context: HashMap::new(), + executions: Vec::new(), + edges: Vec::new(), + }; + for entry in entries { + let id = contexts.context_id(ExecutionContext::Evaluation { + recurrence: entry.recurrence.clone(), + range_evaluation: entry.range_evaluation, + }); + contexts.context_of_query.push(id); + } + let mut candidates_of_group: HashMap> = + HashMap::new(); + for (id, candidate) in physical.physical_candidates.iter().enumerate() { + candidates_of_group + .entry(candidate.logical_candidate_id.group) + .or_default() + .push(PhysicalCandidateId(id)); + } + let all_edges: Vec = physical.edges().map(|(_, edge)| edge).collect(); + let mut reached: BTreeSet = BTreeSet::new(); + let mut pending: Vec = Vec::new(); + for (query, group) in output_references(&physical.query_roots) { + for &v in candidates_of_group.get(&group).into_iter().flatten() { + if let Some(execution) = contexts.output_execution(physical, query, v) { + if reached.insert(execution) { + pending.push(execution); + } + } + } + } + let mut edges = BTreeSet::new(); + while let Some(consumer) = pending.pop() { + let c = consumer.candidate(); + let context = match consumer { + Execution::InContext(context, _) => context, + Execution::Materialized(_) => { + let context = contexts.computation_context_of(memo, physical, c); + contexts.computation_context.insert(c, context); + context + } + }; + let input_count = memo + .candidate(physical.physical_candidates[c.0].logical_candidate_id) + .expression + .operator + .children() + .len(); + for input in 0..input_count { + for &edge in physical.admissible_edges_into(c, input) { + let u = all_edges[edge.0].producer_candidate_id; + if let Some(producer) = contexts.execution_reading(physical, context, u) { + edges.insert((context, edge)); + if reached.insert(producer) { + pending.push(producer); + } + } + } + } + } + contexts.executions = reached.into_iter().collect(); + contexts.edges = edges.into_iter().collect(); + contexts + } + + /// The id of `context`, added to the list when new. + fn context_id(&mut self, context: ExecutionContext) -> ExecutionContextId { + match self.contexts.iter().position(|known| *known == context) { + Some(index) => ExecutionContextId(index), + None => { + self.contexts.push(context); + ExecutionContextId(self.contexts.len() - 1) + } + } + } + + /// The execution of candidate `v` of one of query `query`'s output + /// groups that gives the query's output, in the query's context: a + /// consumer there reading `v` ([`Self::execution_reading`]), but never + /// a result computed at ingestion time and not materialized, which no + /// query reads. + pub(crate) fn output_execution( + &self, + physical: &PhysicalASAPMemoCandidates, + query: usize, + v: PhysicalCandidateId, + ) -> Option { + let context = self.context_of_query(query)?; + let properties = physical.physical_candidates[v.0].physical_properties; + if properties.timing() == ExecutionTiming::IngestionTime + && properties.lifetime() == Lifetime::None + { + return None; + } + self.execution_reading(physical, context, v) + } + + /// The execution of producer `u` a consumer in `context` reads: $u$'s + /// own when it is `AcrossEvaluations` and delivers on `context`; its + /// run in `context` when it is None, or `OneEvaluation` and `context` is + /// an evaluation; `None` otherwise. + fn execution_reading( + &self, + physical: &PhysicalASAPMemoCandidates, + context: ExecutionContextId, + u: PhysicalCandidateId, + ) -> Option { + let producer = &physical.physical_candidates[u.0]; + let context_value = &self.contexts[context.0]; + match producer.physical_properties.lifetime() { + Lifetime::AcrossEvaluations(_) => producer + .delivers_on(&physical.logical_candidates.memo, context_value) + .then_some(Execution::Materialized(u)), + Lifetime::None => Some(Execution::InContext(context, u)), + Lifetime::OneEvaluation(_) => { + matches!(context_value, ExecutionContext::Evaluation { .. }) + .then_some(Execution::InContext(context, u)) + } + } + } + + /// The context the inputs of `AcrossEvaluations` candidate `v` run in: + /// the ingestion pipeline of its refresh window when it is maintained + /// at ingestion time with a bounded window (as `asap_cost` refreshes + /// the views of one refresh window in one ingestion run), otherwise its + /// own computation. + fn computation_context_of( + &mut self, + memo: &Memo, + physical: &PhysicalASAPMemoCandidates, + v: PhysicalCandidateId, + ) -> ExecutionContextId { + let candidate = &physical.physical_candidates[v.0]; + let refresh_window = (candidate.physical_properties.timing() + == ExecutionTiming::IngestionTime) + .then(|| TumblingWindowChainGeometry::of(memo, v, physical)) + .flatten() + .and_then(|chain| chain.refresh_window()); + let context = match refresh_window { + Some(window) => { + let index = match self.refresh_windows.iter().position(|w| *w == window) { + Some(index) => index, + None => { + self.refresh_windows.push(window); + self.refresh_windows.len() - 1 + } + }; + ExecutionContext::IngestionPipeline(RefreshWindowId(index)) + } + None => ExecutionContext::Materialized(v), + }; + self.context_id(context) + } + + /// The distinct contexts, by [`ExecutionContextId`]. + pub(crate) fn contexts(&self) -> &[ExecutionContext] { + &self.contexts + } + + /// Query `query`'s evaluation context. + pub(crate) fn context_of_query(&self, query: usize) -> Option { + self.context_of_query.get(query).copied() + } + + /// The context the inputs of `AcrossEvaluations` candidate `v` run in, + /// when `v` is reached. + pub(crate) fn computation_context(&self, v: PhysicalCandidateId) -> Option { + self.computation_context.get(&v).copied() + } + + /// The context consumer execution `execution` reads its inputs in. + pub(crate) fn input_context(&self, execution: Execution) -> Option { + match execution { + Execution::InContext(context, _) => Some(context), + Execution::Materialized(v) => self.computation_context(v), + } + } + + /// Every execution reached from a query output, ascending. + pub(crate) fn executions(&self) -> &[Execution] { + &self.executions + } + + /// Every $(\kappa, (u, c, i))$ a consumer execution in $\kappa$ may + /// select, ascending. + pub(crate) fn edges(&self) -> &[(ExecutionContextId, PhysicalEdgeId)] { + &self.edges + } + + /// The execution of candidate `v` that a consumer in `context` reads, + /// or a query in `context` outputs: `v`'s own when `AcrossEvaluations`, + /// otherwise its run in `context`. + pub(crate) fn execution_in( + physical: &PhysicalASAPMemoCandidates, + context: ExecutionContextId, + v: PhysicalCandidateId, + ) -> Execution { + match physical.physical_candidates[v.0] + .physical_properties + .lifetime() + { + Lifetime::AcrossEvaluations(_) => Execution::Materialized(v), + Lifetime::None | Lifetime::OneEvaluation(_) => Execution::InContext(context, v), + } + } +} + +/// Where a tumbling-window build's chain lies in time, as the realizer +/// builds it: windows `width_ms` wide, `window_count` of them over the +/// range its build reads (`range_input`), the newest ending +/// `end_offset_ms` before the evaluation: the offsets of the `TimeShift`s +/// below the range, added up as far as the first with an `@` anchor +/// (`anchored` when there is one). +#[derive(Debug, Clone, Copy)] +pub(crate) struct TumblingWindowChainGeometry { + pub(crate) width_ms: u64, + pub(crate) window_count: u64, + pub(crate) end_offset_ms: i64, + pub(crate) anchored: bool, + pub(crate) range_input: MemoGroupId, +} + +impl TumblingWindowChainGeometry { + /// The chain of candidate `v`, when it is a build with a tumbling-window + /// layout over a range its width divides. + pub(crate) fn of( + memo: &Memo, + v: PhysicalCandidateId, + physical: &PhysicalASAPMemoCandidates, + ) -> Option { + let logical = memo.candidate(physical.physical_candidates[v.0].logical_candidate_id); + let Some(&WindowLayout::Tumbling { width_ms }) = logical.logical_properties.window_layout() + else { + return None; + }; + let Operator::ASAP(ASAPOp::SummaryAgg { + child: range_input, .. + }) = &logical.expression.operator + else { + return None; + }; + let Operator::NonASAP(NonASAPOp::TimeRange { + range, + kind: TimeRangeKind::Range, + child, + }) = &memo.groups()[range_input.0 as usize].expression + else { + return None; + }; + let range_ms = u64::try_from(range.as_millis()).ok()?; + if width_ms == 0 || range_ms % width_ms != 0 { + return None; + } + let mut end_offset_ms = 0i64; + let mut anchored = false; + let mut below = *child; + while let Operator::NonASAP(NonASAPOp::TimeShift { shift, child }) = + &memo.groups()[below.0 as usize].expression + { + if shift.at.is_some() { + anchored = true; + break; + } + end_offset_ms = end_offset_ms.saturating_add(shift.offset_ms); + below = *child; + } + Some(Self { + width_ms, + window_count: range_ms / width_ms, + end_offset_ms, + anchored, + range_input: *range_input, + }) + } + + /// The refresh window of its newest tumbling window (`RefreshWindow::of_window`), + /// as `PhysicalASAPDAG::refresh_window` gives it for the chain; `None` + /// under an `@` anchor. + pub(crate) fn refresh_window(&self) -> Option { + if self.anchored { + return None; + } + RefreshWindow::of_window(self.end_offset_ms, i64::try_from(self.width_ms).ok()?) + } + + /// $\operatorname{need}_{\kappa,v,c,i}$ in windows: the tumbling windows + /// a consumer whose queries evaluate `span_ms` back from each request's + /// end reads, back from the evaluation: its offset, its window and + /// that span (`asap_cost::tumbling_window_retention_ms`), over the + /// width, at least the windows its range holds. + pub(crate) fn retention_need_windows(&self, span_ms: u64) -> u64 { + let offset_ms = u64::try_from(self.end_offset_ms).unwrap_or(0); + let reach_ms = offset_ms + .saturating_add(self.width_ms.saturating_mul(self.window_count)) + .saturating_add(span_ms); + (reach_ms / self.width_ms).max(self.window_count) + } +} diff --git a/crates/stage3-plan-selection/src/milp/mod.rs b/crates/stage3-plan-selection/src/milp/mod.rs index 9a2d7f7e3..8a535ee7b 100644 --- a/crates/stage3-plan-selection/src/milp/mod.rs +++ b/crates/stage3-plan-selection/src/milp/mod.rs @@ -1,3 +1,4 @@ //! Stage 3's selection MILP (concepts doc, sections 3–5): its variables, //! one function per constraint family, and the execution contexts. +pub(crate) mod execution_contexts; pub(crate) mod model; diff --git a/crates/stage3-plan-selection/src/milp/model.rs b/crates/stage3-plan-selection/src/milp/model.rs index 767aee908..20e8059df 100644 --- a/crates/stage3-plan-selection/src/milp/model.rs +++ b/crates/stage3-plan-selection/src/milp/model.rs @@ -3,19 +3,30 @@ //! //! | Family | Variables | Constraints | //! |---|---|---| -//! | baseline (section 3) | $x_v$, $y_{u,c,i}$, $z_{q,v}$, $t_v$ ([`SelectionVariables`]) | (A) [`input_satisfaction_constraints`], (B) [`producer_selection_constraints`], (C)/(D) [`query_coverage_constraints`], (E) [`acyclicity_constraints`], (F) [`utilization_constraints`]; (G), (H): $y$ exists only on admissible edges | +//! | baseline (section 3) | $z_{q,v}$, $t_v$ ([`SelectionVariables`]) | (A) [`input_satisfaction_constraints`], (B) [`producer_selection_constraints`], (C)/(D) [`query_coverage_constraints`], (E) [`acyclicity_constraints`], (F) [`utilization_constraints`]; (G), (H): $y$ exists only on admissible edges | //! | execution contexts (section 5) | $x^{\mathrm{mat}}_v$, $x_{\kappa,v}$, $y_{\kappa,u,c,i}$, $h_v$ ([`ExecutionContextVariables`]) | (R) [`retention_constraints`] | //! | accuracy (section 4) | $E_c$, $F_{c,i}$, $D_c$, $H_{c,i}$ ([`AccuracyVariables`]) | (K) [`accuracy_constraints`] | //! -//! The baseline (section 3) is built and solved by [`solve_selection`]: -//! $x_v$ and $y_{u,c,i}$ binary, $z_{q,v}$ binary per query output group, -//! $t_v$ continuous in $[0, |\mathcal C| - 1]$, the objective -//! $\min \sum_v w_v x_v$ with $w_v$ from [`crate::candidate_cost`]. The -//! execution-context and accuracy families are `todo!()` until #1207 PRs 7 -//! (contexts) and 8 (accuracy); nothing calls them yet, hence their -//! `dead_code` allowances. - -use std::collections::{BTreeSet, HashMap}; +//! The execution-context variables replace the baseline's $x_v$ and +//! unindexed $y_{u,c,i}$: a candidate with $\ell$ = None or `OneEvaluation` +//! runs once per context $\kappa$ that selects it ($x_{\kappa,v}$), an +//! `AcrossEvaluations` candidate once ($x^{\mathrm{mat}}_v$), and every edge +//! is selected per context of its consumer ($y_{\kappa,u,c,i}$), an +//! `AcrossEvaluations` consumer's being its computation context +//! ([`crate::milp::execution_contexts`]). Constraints (A)–(F) hold per +//! execution ([`Execution`]): $x$ below is $x_{\kappa,v}$ or +//! $x^{\mathrm{mat}}_v$, whichever runs $v$ for a consumer in $\kappa$. +//! +//! [`solve_selection`] builds and solves it: $x$, $y$ and $z$ binary, $t$ +//! continuous in $[0, |\text{executions}| - 1]$ on cyclic components only, +//! $h_v \ge 0$ continuous, and the objective (section 5) +//! $$\min \sum_v W^{\mathrm{mat}}_v x^{\mathrm{mat}}_v + \sum_{\kappa,v} +//! \rho_\kappa c_{\kappa,v} x_{\kappa,v} + \sum_v \pi_v h_v$$ +//! with the costs from [`crate::execution_cost`]. The accuracy family is +//! `todo!()` until #1207 PR 8; nothing calls it yet, hence its `dead_code` +//! allowances. + +use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::time::Instant; use good_lp::solvers::highs::highs; @@ -23,41 +34,62 @@ use good_lp::{ constraint, variable, Constraint, Expression, ProblemVariables, SolverModel, Variable, }; -use asap_types::physical::{ExecutionTiming, Lifetime}; use asap_types::stages::{ output_references, DagOfSelectedCandidates, ExecutionContext, ExecutionContextId, MemoGroupId, PhysicalASAPMemoCandidates, PhysicalCandidateId, PhysicalEdge, PhysicalEdgeId, SelectedEdge, SelectedNodeId, }; -use asap_types::workload::QueryWorkloadEntry; + +use crate::execution_cost::ExecutionCosts; +use crate::milp::execution_contexts::{Execution, ExecutionContexts, TumblingWindowChainGeometry}; /// A query, by index in the workload. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] pub struct QueryId(pub usize); -/// The baseline variables (section 3). +/// The baseline variables (section 3) the execution contexts keep. pub(crate) struct SelectionVariables { - /// $x_v$, by [`PhysicalCandidateId`]. - x: Vec, - /// $y_{u,c,i}$, on admissible edges only. - y: HashMap, - /// $z_{q,v}$, for $v \in G_q$. + /// $z_{q,v}$, for $v \in G_q$ whose execution in $q$'s context can + /// give its output. z: HashMap<(QueryId, PhysicalCandidateId), Variable>, - /// $t_v$, by [`PhysicalCandidateId`]. - t: Vec, + /// $t$, per execution on a cyclic component of the selectable edges. + t: HashMap, +} + +/// The execution-context variables (section 5), which replace $x$ and the +/// unindexed $y$ of the baseline. +pub(crate) struct ExecutionContextVariables { + /// $x^{\mathrm{mat}}_v$, for an `AcrossEvaluations` candidate. + x_materialized: HashMap, + /// $x_{\kappa,v}$, for a candidate with $\ell$ = None or `OneEvaluation`. + x_in_execution_context: HashMap<(ExecutionContextId, PhysicalCandidateId), Variable>, + /// $y_{\kappa,u,c,i}$, indexed by the consumer's context. + y_in_execution_context: HashMap<(ExecutionContextId, PhysicalEdgeId), Variable>, + /// $h_v$, for an `AcrossEvaluations` producer: the tumbling windows it + /// retains. + retention_amount: HashMap, +} + +impl ExecutionContextVariables { + /// $x$ of `execution`, when it can be costed. + fn x(&self, execution: Execution) -> Option { + match execution { + Execution::Materialized(v) => self.x_materialized.get(&v).copied(), + Execution::InContext(context, v) => { + self.x_in_execution_context.get(&(context, v)).copied() + } + } + } } -/// What the baseline MILP is built over: Stage 2's candidates and -/// admissible edges, each candidate's $w_v$ (`None`: not costable, so -/// $x_v = 0$), each query's entry, and indexes built once -/// ([`Self::new`]). +/// What the MILP is built over: Stage 2's candidates and admissible edges, +/// the workload's execution contexts with each execution and edge they +/// allow, the objective's costs (an execution without one is left out), +/// and indexes built once ([`Self::new`]). pub(crate) struct SelectionInputs<'a> { pub(crate) physical: &'a PhysicalASAPMemoCandidates, - /// $w_v$ per second, by [`PhysicalCandidateId`]. - pub(crate) cost_per_second: &'a [Option], - /// Each query's workload entry, whose recurrence and range evaluation - /// are its evaluation context. - pub(crate) entries: &'a [QueryWorkloadEntry], + pub(crate) contexts: &'a ExecutionContexts, + pub(crate) costs: &'a ExecutionCosts, /// $\mathcal E$, by [`PhysicalEdgeId`]. edges: Vec, /// The physical candidates of each memo group, ascending. @@ -70,8 +102,8 @@ pub(crate) struct SelectionInputs<'a> { impl<'a> SelectionInputs<'a> { pub(crate) fn new( physical: &'a PhysicalASAPMemoCandidates, - cost_per_second: &'a [Option], - entries: &'a [QueryWorkloadEntry], + contexts: &'a ExecutionContexts, + costs: &'a ExecutionCosts, ) -> Self { let mut candidates_of_group: HashMap> = HashMap::new(); @@ -89,8 +121,8 @@ impl<'a> SelectionInputs<'a> { .collect(); Self { physical, - cost_per_second, - entries, + contexts, + costs, edges: physical.edges().map(|(_, edge)| edge).collect(), candidates_of_group, query_output_groups, @@ -114,225 +146,324 @@ impl<'a> SelectionInputs<'a> { .len() } - /// Whether edge `edge`'s producer delivers on every evaluation context - /// its consumer serves: a producer materialized across evaluations read - /// by a query-time consumer must deliver on the evaluation of each - /// query reaching the consumer's group (`PhysicalCandidate::delivers_on`). - /// The baseline has no execution contexts, so it requires every one; - /// with contexts (#1207 PR 7) the check is per context $\kappa$ of - /// $y_{\kappa,u,c,i}$. A consumer at ingestion time is not checked - /// here (its pipeline context is #1207 PR 7's). - fn delivers_to_consumer(&self, edge: &PhysicalEdge) -> bool { - let producer = &self.physical.physical_candidates[edge.producer_candidate_id.0]; - let consumer = &self.physical.physical_candidates[edge.consumer_candidate_id.0]; - if !matches!( - producer.physical_properties.lifetime(), - Lifetime::AcrossEvaluations(_) - ) || consumer.physical_properties.timing() != ExecutionTiming::QueryTime - { - return true; - } - let memo = &self.physical.logical_candidates.memo; - memo.groups()[consumer.logical_candidate_id.group.0 as usize] - .queries - .iter() - .filter_map(|&query| self.entries.get(query)) - .all(|entry| { - producer.delivers_on( - memo, - &ExecutionContext::Evaluation { - recurrence: entry.recurrence.clone(), - range_evaluation: entry.range_evaluation, - }, - ) - }) + /// The producer and consumer executions of edge `edge` selected in the + /// consumer's context `context`. + fn executions_of_edge( + &self, + context: ExecutionContextId, + edge: PhysicalEdgeId, + ) -> (Execution, Execution) { + let edge = &self.edges[edge.0]; + ( + ExecutionContexts::execution_in(self.physical, context, edge.producer_candidate_id), + ExecutionContexts::execution_in(self.physical, context, edge.consumer_candidate_id), + ) + } + + /// The objective's cost per second of `execution` + /// ($\rho_\kappa c_{\kappa,v}$ or $W^{\mathrm{mat}}_v$), when it can be + /// costed. + fn cost_per_second(&self, execution: Execution) -> Option { + self.costs.cost_per_second.get(&execution).copied() + } + + /// $\operatorname{need}_{\kappa,v,c,i}$ of `AcrossEvaluations` producer + /// `v` for a consumer in `context`, in tumbling windows: what the + /// queries evaluated in `context` read of it, their range-evaluation + /// span included ([`TumblingWindowChainGeometry::retention_need_windows`]). + /// Query latency is ignored (decided 2026-11-06): a result counts as + /// read when its evaluation starts. + fn retention_need_windows(&self, context: ExecutionContextId, v: PhysicalCandidateId) -> u64 { + let span_ms = match &self.contexts.contexts()[context.0] { + ExecutionContext::Evaluation { + range_evaluation: Some(range), + .. + } => range.span_ms, + _ => 0, + }; + TumblingWindowChainGeometry::of(&self.physical.logical_candidates.memo, v, self.physical) + .map_or(0, |chain| chain.retention_need_windows(span_ms)) } /// The physical candidates of memo group `group`, ascending. - fn candidates_of_group(&self, group: MemoGroupId) -> Vec { + fn candidates_of_group(&self, group: MemoGroupId) -> &[PhysicalCandidateId] { self.candidates_of_group .get(&group) - .cloned() - .unwrap_or_default() + .map_or(&[], Vec::as_slice) } /// Each query's output groups $G_q$, each pair once, in output-reference /// order: a scalar query reads one per operator reference. - fn query_output_groups(&self) -> Vec<(QueryId, MemoGroupId)> { - self.query_output_groups.clone() + fn query_output_groups(&self) -> &[(QueryId, MemoGroupId)] { + &self.query_output_groups + } + + /// The execution of `v` that gives query `query`'s output, when it can + /// be costed. + fn output_execution(&self, query: QueryId, v: PhysicalCandidateId) -> Option { + self.contexts + .output_execution(self.physical, query.0, v) + .filter(|&execution| self.cost_per_second(execution).is_some()) } } -/// The baseline MILP's variables over `inputs` (constraint (I), domains): -/// $x_v$ binary, fixed at 0 for a candidate that cannot be costed; $y$ -/// binary on each admissible edge ((G) and (H)) whose producer and consumer -/// can be costed and whose producer delivers on its consumer's evaluations -/// ([`SelectionInputs::delivers_to_consumer`]); $z_{q,v}$ binary for each costable $v \in G_q$; -/// $t_v \in [0, |\mathcal C| - 1]$ continuous. +/// The MILP's variables over `inputs` (constraint (I), domains): $x$ +/// binary per costed execution; $y_{\kappa,u,c,i}$ binary per edge the +/// contexts allow ((G) and (H): admissible; an `AcrossEvaluations` +/// producer delivers on $\kappa$) whose producer and consumer executions +/// are costed; $z_{q,v}$ binary per costed output execution; $h_v \ge 0$ +/// per costed `AcrossEvaluations` candidate; $t \in [0, n - 1]$ continuous +/// per execution on a cyclic component of the $y$ edges, $n$ the executions. fn selection_variables( inputs: &SelectionInputs<'_>, problem: &mut ProblemVariables, -) -> SelectionVariables { - let count = inputs.physical.physical_candidates.len(); - let costable = |id: PhysicalCandidateId| inputs.cost_per_second[id.0].is_some(); - let x = (0..count) - .map(|id| match costable(PhysicalCandidateId(id)) { - true => problem.add(variable().binary()), - false => problem.add(variable().min(0).max(0)), - }) - .collect(); - let mut y = HashMap::new(); - for (id, edge) in inputs.physical.edges() { - if edge.satisfies_logical_requirement - && edge.satisfies_physical_requirement - && inputs.delivers_to_consumer(&edge) - && costable(edge.producer_candidate_id) - && costable(edge.consumer_candidate_id) - { - y.insert(id, problem.add(variable().binary())); +) -> (SelectionVariables, ExecutionContextVariables) { + let mut x_materialized = HashMap::new(); + let mut x_in_execution_context = HashMap::new(); + let mut retention_amount = HashMap::new(); + for &execution in inputs.contexts.executions() { + if inputs.cost_per_second(execution).is_none() { + continue; + } + match execution { + Execution::Materialized(v) => { + x_materialized.insert(v, problem.add(variable().binary())); + retention_amount.insert(v, problem.add(variable().min(0))); + } + Execution::InContext(context, v) => { + x_in_execution_context.insert((context, v), problem.add(variable().binary())); + } + } + } + let mut context_variables = ExecutionContextVariables { + x_materialized, + x_in_execution_context, + y_in_execution_context: HashMap::new(), + retention_amount, + }; + for &(context, edge) in inputs.contexts.edges() { + let (producer, consumer) = inputs.executions_of_edge(context, edge); + if context_variables.x(producer).is_some() && context_variables.x(consumer).is_some() { + context_variables + .y_in_execution_context + .insert((context, edge), problem.add(variable().binary())); } } let mut z = HashMap::new(); - for (query, group) in inputs.query_output_groups() { - for v in inputs.candidates_of_group(group) { - if costable(v) { + for &(query, group) in inputs.query_output_groups() { + for &v in inputs.candidates_of_group(group) { + if inputs.output_execution(query, v).is_some() { z.entry((query, v)) .or_insert_with(|| problem.add(variable().binary())); } } } - let rank_bound = count.saturating_sub(1) as f64; - let t = (0..count) - .map(|_| problem.add(variable().min(0).max(rank_bound))) + let executions: Vec = inputs.contexts.executions().to_vec(); + let position: HashMap = executions + .iter() + .enumerate() + .map(|(position, &execution)| (execution, position)) .collect(); - SelectionVariables { x, y, z, t } + let component = strongly_connected_components( + executions.len(), + sorted(&context_variables.y_in_execution_context) + .into_iter() + .map(|(&(context, edge), _)| { + let (producer, consumer) = inputs.executions_of_edge(context, edge); + (position[&producer], position[&consumer]) + }), + ); + let rank_bound = executions.len().saturating_sub(1) as f64; + let mut t = HashMap::new(); + for (&(context, edge), _) in sorted(&context_variables.y_in_execution_context) { + let (producer, consumer) = inputs.executions_of_edge(context, edge); + if component[position[&producer]] == component[position[&consumer]] { + for execution in [producer, consumer] { + t.entry(execution) + .or_insert_with(|| problem.add(variable().min(0).max(rank_bound))); + } + } + } + (SelectionVariables { z, t }, context_variables) +} + +/// `map`'s entries in key order, so constraints come out in the same order +/// every run. +fn sorted(map: &HashMap) -> Vec<(&K, &V)> { + let mut entries: Vec<(&K, &V)> = map.iter().collect(); + entries.sort_by(|a, b| a.0.cmp(b.0)); + entries } -/// (A) $\sum_{u \in P_{c,i}} y_{u,c,i} = x_c$: every input of a selected -/// consumer has exactly one selected producer, and an unselected consumer -/// none. +/// (A) $\sum_{u \in P_{c,i}} y_{\kappa,u,c,i} = x_c$: every input of a +/// selected consumer execution has exactly one selected producer, read in +/// the consumer's context, and an unselected one none. fn input_satisfaction_constraints( inputs: &SelectionInputs<'_>, - variables: &SelectionVariables, + variables: &ExecutionContextVariables, ) -> Vec { + let mut producers: BTreeMap<(Execution, usize), Expression> = BTreeMap::new(); + for (&(context, edge), &y) in sorted(&variables.y_in_execution_context) { + let (_, consumer) = inputs.executions_of_edge(context, edge); + *producers + .entry((consumer, inputs.edges()[edge.0].input_index_of_consumer)) + .or_default() += y; + } let mut constraints = Vec::new(); - for c in (0..inputs.physical.physical_candidates.len()).map(PhysicalCandidateId) { - for i in 0..inputs.input_count(c) { - let producers: Expression = inputs - .physical - .admissible_edges_into(c, i) - .iter() - .filter_map(|edge| variables.y.get(edge)) - .sum(); - constraints.push(constraint!(producers == variables.x[c.0])); + for &consumer in inputs.contexts.executions() { + let Some(x) = variables.x(consumer) else { + continue; + }; + for input in 0..inputs.input_count(consumer.candidate()) { + let chosen = producers.remove(&(consumer, input)).unwrap_or_default(); + constraints.push(constraint!(chosen == x)); } } constraints } -/// (B) $y_{u,c,i} \le x_u$: a chosen producer is selected. +/// (B) $y_{\kappa,u,c,i} \le x_{\kappa,u}$ for a None or `OneEvaluation` +/// producer (read only within $\kappa$), $y_{\kappa,u,c,i} \le +/// x^{\mathrm{mat}}_u$ for an `AcrossEvaluations` one: a chosen producer is +/// selected. fn producer_selection_constraints( inputs: &SelectionInputs<'_>, - variables: &SelectionVariables, + variables: &ExecutionContextVariables, ) -> Vec { - let edges = inputs.edges(); - let mut by_edge: Vec<(&PhysicalEdgeId, &Variable)> = variables.y.iter().collect(); - by_edge.sort_by_key(|(edge, _)| **edge); - by_edge + sorted(&variables.y_in_execution_context) .into_iter() - .map(|(edge, &y)| { - let producer = edges[edge.0].producer_candidate_id; - constraint!(y <= variables.x[producer.0]) + .map(|(&(context, edge), &y)| { + let (producer, _) = inputs.executions_of_edge(context, edge); + let x = variables + .x(producer) + .expect("y only between costed executions"); + constraint!(y <= x) }) .collect() } /// (C) $\sum_{v \in G_q} z_{q,v} = 1$, for each output group of each query; -/// (D) $z_{q,v} \le x_v$. +/// (D) $z_{q,v} \le x$ of $v$'s execution in $q$'s context. fn query_coverage_constraints( inputs: &SelectionInputs<'_>, - variables: &SelectionVariables, + selection: &SelectionVariables, + variables: &ExecutionContextVariables, ) -> Vec { let mut constraints = Vec::new(); - for (query, group) in inputs.query_output_groups() { + for &(query, group) in inputs.query_output_groups() { let outputs: Vec<(PhysicalCandidateId, Variable)> = inputs .candidates_of_group(group) - .into_iter() - .filter_map(|v| variables.z.get(&(query, v)).map(|&z| (v, z))) + .iter() + .filter_map(|&v| selection.z.get(&(query, v)).map(|&z| (v, z))) .collect(); let covered: Expression = outputs.iter().map(|&(_, z)| z).sum(); constraints.push(constraint!(covered == 1)); for (v, z) in outputs { - constraints.push(constraint!(z <= variables.x[v.0])); + let execution = inputs + .output_execution(query, v) + .expect("z only on costed output executions"); + let x = variables.x(execution).expect("costed"); + constraints.push(constraint!(z <= x)); } } constraints } -/// (E) $t_u + 1 \le t_c + |\mathcal C|(1 - y_{u,c,i})$: no selected cycle. -/// Only edges inside a strongly connected component of the admissible -/// edges can close a cycle, so (E) is written for those only (a producer -/// reading itself included, which (E) forbids). +/// (E) $t_u + 1 \le t_c + n(1 - y_{\kappa,u,c,i})$ over executions ($n$ +/// the executions): no selected cycle. Only edges inside a strongly +/// connected component of the selectable edges can close a cycle, so (E) +/// is written for those only (an execution reading itself included, which +/// (E) forbids). fn acyclicity_constraints( inputs: &SelectionInputs<'_>, - variables: &SelectionVariables, + selection: &SelectionVariables, + variables: &ExecutionContextVariables, ) -> Vec { - let edges = inputs.edges(); - let component = strongly_connected_components( - inputs.physical.physical_candidates.len(), - variables.y.keys().map(|edge| { - ( - edges[edge.0].producer_candidate_id.0, - edges[edge.0].consumer_candidate_id.0, - ) - }), - ); - let candidate_count = inputs.physical.physical_candidates.len() as f64; - let mut on_cycles: Vec<(&PhysicalEdgeId, &Variable)> = variables - .y - .iter() - .filter(|(edge, _)| { - let edge = &edges[edge.0]; - component[edge.producer_candidate_id.0] == component[edge.consumer_candidate_id.0] - }) - .collect(); - on_cycles.sort_by_key(|(edge, _)| **edge); - on_cycles + let execution_count = inputs.contexts.executions().len() as f64; + sorted(&variables.y_in_execution_context) .into_iter() - .map(|(edge, &y)| { - let edge = &edges[edge.0]; - let t_u = variables.t[edge.producer_candidate_id.0]; - let t_c = variables.t[edge.consumer_candidate_id.0]; - constraint!(t_u + 1 <= t_c + candidate_count * (1 - y)) + .filter_map(|(&(context, edge), &y)| { + let (producer, consumer) = inputs.executions_of_edge(context, edge); + let t_u = *selection.t.get(&producer)?; + let t_c = *selection.t.get(&consumer)?; + Some(constraint!(t_u + 1 <= t_c + execution_count * (1 - y))) }) .collect() } -/// (F) $x_v \le \sum_{(v,c,i)} y_{v,c,i} + \sum_q z_{q,v}$: a selected -/// candidate supplies a selected input or a query output. +/// (F) $x \le \sum y + \sum_q z_{q,v}$ per execution: a selected execution +/// supplies a selected input (in its context, or, for an +/// `AcrossEvaluations` candidate, in any context) or a query output. fn utilization_constraints( inputs: &SelectionInputs<'_>, - variables: &SelectionVariables, + selection: &SelectionVariables, + variables: &ExecutionContextVariables, ) -> Vec { - let mut outputs_of: HashMap = HashMap::new(); - for (&(_, v), &z) in &variables.z { - *outputs_of.entry(v).or_default() += z; + let mut used: BTreeMap = BTreeMap::new(); + for (&(context, edge), &y) in sorted(&variables.y_in_execution_context) { + let (producer, _) = inputs.executions_of_edge(context, edge); + *used.entry(producer).or_default() += y; } - (0..inputs.physical.physical_candidates.len()) - .map(PhysicalCandidateId) - .map(|v| { - let consumers: Expression = inputs - .physical - .admissible_edges_out_of(v) - .iter() - .filter_map(|edge| variables.y.get(edge)) - .sum(); - let used = consumers + outputs_of.remove(&v).unwrap_or_default(); - constraint!(variables.x[v.0] <= used) + for (&(query, v), &z) in sorted(&selection.z) { + let execution = inputs + .output_execution(query, v) + .expect("z only on costed output executions"); + *used.entry(execution).or_default() += z; + } + inputs + .contexts + .executions() + .iter() + .filter_map(|&execution| { + let x = variables.x(execution)?; + let used = used.remove(&execution).unwrap_or_default(); + Some(constraint!(x <= used)) }) .collect() } +/// (R) $h_v \ge \operatorname{need}_{\kappa,v,c,i}\, y_{\kappa,v,c,i}$, per +/// edge out of an `AcrossEvaluations` producer $v$, and +/// $h_v \ge \operatorname{need}_{\kappa_q,v}\, z_{q,v}$ for a query whose +/// output $v$ is: it retains as many tumbling windows as the most +/// demanding selected consumer reads +/// ([`SelectionInputs::retention_need_windows`]). +/// +/// Assumptions (decided 2026-11-06): an edge's need is the time horizon +/// the queries it serves read, a constant per edge (a 5m range over 10 s +/// windows needs 30 windows), so (R) is linear; query latency is ignored +/// until the latency design: a result counts as read when its evaluation +/// starts. +fn retention_constraints( + inputs: &SelectionInputs<'_>, + selection: &SelectionVariables, + variables: &ExecutionContextVariables, +) -> Vec { + let edges = sorted(&variables.y_in_execution_context) + .into_iter() + .filter_map(|(&(context, edge), &y)| { + let Execution::Materialized(v) = inputs.executions_of_edge(context, edge).0 else { + return None; + }; + let h = variables.retention_amount[&v]; + let need = inputs.retention_need_windows(context, v) as f64; + Some(constraint!(h >= need * y)) + }); + // A query reading `v` itself is its consumer too, in its own context. + let outputs = sorted(&selection.z) + .into_iter() + .filter_map(|(&(query, v), &z)| { + let Some(Execution::Materialized(_)) = inputs.output_execution(query, v) else { + return None; + }; + let context = inputs.contexts.context_of_query(query.0)?; + let h = variables.retention_amount[&v]; + let need = inputs.retention_need_windows(context, v) as f64; + Some(constraint!(h >= need * z)) + }); + edges.chain(outputs).collect() +} + /// The component of each of `node_count` nodes in the graph of `arcs` /// (`(from, to)`), numbered so equal numbers are one strongly connected /// component (Kosaraju's algorithm, iterative). @@ -386,11 +517,16 @@ fn strongly_connected_components( component } -/// The solved baseline: the selected DAG and what the solve took. +/// The solved selection: the selected DAG and what the solve took. #[derive(Debug, Clone)] pub(crate) struct SolvedSelection { pub(crate) selected_dag: DagOfSelectedCandidates, - /// $\sum_v w_v x_v$ at the optimum, per second. + /// The execution behind each node of `selected_dag`, by + /// [`SelectedNodeId`]. + pub(crate) execution_of_node: Vec, + /// The objective at the optimum, per second: the selected executions' + /// $W^{\mathrm{mat}}_v$ and $\rho_\kappa c_{\kappa,v}$, plus $\pi_v h_v$ + /// with each $h_v$ the largest need of its selected edges. pub(crate) objective_per_second: f64, /// The solver's lower bound on it; NaN when it reports none. pub(crate) lower_bound_per_second: f64, @@ -400,64 +536,76 @@ pub(crate) struct SolvedSelection { pub(crate) solver_ms: u64, } -/// Why the baseline MILP selected nothing. +/// Why the MILP selected nothing. #[derive(Debug, Clone, PartialEq)] pub(crate) enum SelectionMilpError { - /// No assignment meets (A)–(I): some query output has no candidate - /// whose inputs all have costable admissible producers. + /// No assignment meets the constraints: some query output has no + /// execution whose inputs all have costable producers it can read. Infeasible, /// HiGHS failed. Solver(String), } -/// Build the baseline MILP over `inputs` ((A)–(I), $\min \sum_v w_v x_v$), -/// solve it to proven optimality with HiGHS, and decode $\{x = 1\}$, -/// $\{y = 1\}$ and $\{z = 1\}$ into a [`DagOfSelectedCandidates`]: one node -/// per selected candidate, in candidate order; one edge per selected $y$; -/// each output reference's node the selected $z$ of its query and group. +/// Build the MILP over `inputs` ((A)–(F) and (R) per execution, the +/// objective of section 5), solve it to proven optimality with HiGHS, and +/// decode $\{x = 1\}$, $\{y = 1\}$ and $\{z = 1\}$ into a +/// [`DagOfSelectedCandidates`]: one node per selected execution, in +/// execution order, so a candidate run in two contexts is two nodes; one +/// edge per selected $y$; each output reference's node the execution of +/// the candidate its query's $z$ selects. pub(crate) fn solve_selection( inputs: &SelectionInputs<'_>, ) -> Result { - // (C) cannot hold for a query output group without a costable - // candidate. - if inputs.query_output_groups().into_iter().any(|(_, group)| { + // (C) cannot hold for a query output group without a costable output + // execution. + if inputs.query_output_groups().iter().any(|&(query, group)| { inputs .candidates_of_group(group) - .into_iter() - .all(|v| inputs.cost_per_second[v.0].is_none()) + .iter() + .all(|&v| inputs.output_execution(query, v).is_none()) }) { return Err(SelectionMilpError::Infeasible); } let started = Instant::now(); let mut problem = ProblemVariables::new(); - let variables = selection_variables(inputs, &mut problem); + let (selection, variables) = selection_variables(inputs, &mut problem); let constraints: Vec = [ input_satisfaction_constraints(inputs, &variables), producer_selection_constraints(inputs, &variables), - query_coverage_constraints(inputs, &variables), - acyclicity_constraints(inputs, &variables), - utilization_constraints(inputs, &variables), + query_coverage_constraints(inputs, &selection, &variables), + acyclicity_constraints(inputs, &selection, &variables), + utilization_constraints(inputs, &selection, &variables), + retention_constraints(inputs, &selection, &variables), ] .into_iter() .flatten() .collect(); - // Scaled so the largest cost is 1,000: HiGHS's tolerances are absolute. - let largest = inputs - .cost_per_second + // The objective's coefficients, per second, in execution order. + let mut terms: Vec<(f64, Variable)> = Vec::new(); + for &execution in inputs.contexts.executions() { + if let (Some(cost), Some(x)) = (inputs.cost_per_second(execution), variables.x(execution)) { + terms.push((cost, x)); + } + if let Execution::Materialized(v) = execution { + if let (Some(&cost), Some(&h)) = ( + inputs.costs.retention_cost_per_window_per_second.get(&v), + variables.retention_amount.get(&v), + ) { + terms.push((cost, h)); + } + } + } + // Scaled so the largest coefficient is 1,000: HiGHS's tolerances are + // absolute. + let largest = terms .iter() - .flatten() - .fold(0.0_f64, |largest, &cost| largest.max(cost)); + .fold(0.0_f64, |largest, &(cost, _)| largest.max(cost)); let scale = if largest > 0.0 && largest.is_finite() { 1_000.0 / largest } else { 1.0 }; - let objective: Expression = inputs - .cost_per_second - .iter() - .zip(&variables.x) - .filter_map(|(cost, &x)| cost.map(|cost| cost * scale * x)) - .sum(); + let objective: Expression = terms.iter().map(|&(cost, v)| cost * scale * v).sum(); let variable_count = problem.len(); // HiGHS's columns are the variables in the order they were added. let column_of: HashMap = problem @@ -467,7 +615,7 @@ pub(crate) fn solve_selection( .collect(); let constraint_count = constraints.len(); let mut model = problem - .minimise(objective.clone()) + .minimise(objective) .using(highs) .with_all(constraints) .try_into_inner() @@ -486,8 +634,10 @@ pub(crate) fn solve_selection( if solved.status() == HighsModelStatus::ModelEmpty { // No constraint: no query reads an operator (literal-only scalar // roots), so nothing is selected. + let (selected_dag, execution_of_node) = decode(inputs, &selection, &variables, |_| false); return Ok(SolvedSelection { - selected_dag: decode(inputs, &variables, |_| false), + selected_dag, + execution_of_node, objective_per_second: 0.0, lower_bound_per_second: 0.0, variable_count, @@ -514,16 +664,11 @@ pub(crate) fn solve_selection( / scale; let columns = solved.get_solution().columns().to_vec(); let chosen = |variable: Variable| columns[column_of[&variable]] > 0.5; - let objective_per_second: f64 = inputs - .cost_per_second - .iter() - .zip(&variables.x) - .filter(|(_, &x)| chosen(x)) - .filter_map(|(cost, _)| *cost) - .sum(); - let selected_dag = decode(inputs, &variables, chosen); + let (selected_dag, execution_of_node) = decode(inputs, &selection, &variables, chosen); + let objective_per_second = objective_of(inputs, &selected_dag, &execution_of_node); Ok(SolvedSelection { selected_dag, + execution_of_node, objective_per_second, lower_bound_per_second, variable_count, @@ -533,55 +678,116 @@ pub(crate) fn solve_selection( }) } +/// The objective of a decoded selection, per second: each selected +/// execution's cost ($W^{\mathrm{mat}}_v$ or $\rho_\kappa c_{\kappa,v}$) +/// and $\pi_v h_v$ of each selected `AcrossEvaluations` candidate, $h_v$ +/// the largest need of its selected edges and of the queries it gives the +/// output of, which minimizes $\pi_v h_v$ under (R). +pub(crate) fn objective_of( + inputs: &SelectionInputs<'_>, + selected_dag: &DagOfSelectedCandidates, + execution_of_node: &[Execution], +) -> f64 { + let mut retained: BTreeMap = BTreeMap::new(); + for edge in &selected_dag.edges { + if let (Execution::Materialized(v), Some(context)) = ( + execution_of_node[edge.producer.0], + inputs + .contexts + .input_context(execution_of_node[edge.consumer.0]), + ) { + let need = inputs.retention_need_windows(context, v); + let windows = retained.entry(v).or_default(); + *windows = (*windows).max(need); + } + } + for ((query, _), node) in output_references(&inputs.physical.query_roots) + .into_iter() + .zip(&selected_dag.node_of_output_reference) + { + if let (Some(Execution::Materialized(v)), Some(context)) = ( + execution_of_node.get(node.0).copied(), + inputs.contexts.context_of_query(query), + ) { + let need = inputs.retention_need_windows(context, v); + let windows = retained.entry(v).or_default(); + *windows = (*windows).max(need); + } + } + let executions: f64 = execution_of_node + .iter() + .filter_map(|&execution| inputs.cost_per_second(execution)) + .sum(); + let retention: f64 = retained + .into_iter() + .map(|(v, windows)| { + inputs + .costs + .retention_cost_per_window_per_second + .get(&v) + .map_or(0.0, |cost| cost * windows as f64) + }) + .sum(); + executions + retention +} + /// $\{x = 1\}$, $\{y = 1\}$ and $\{z = 1\}$ as a [`DagOfSelectedCandidates`] -/// (`decode.rs` in the formulation): one node per selected candidate, in -/// candidate order; each selected edge between their nodes; each output -/// reference's node the candidate its query's $z$ selects in its group. +/// (`decode.rs` in the formulation), with the execution behind each node: +/// one node per selected execution, in execution order; each selected edge +/// between the nodes of its producer and consumer executions; each output +/// reference's node the execution of the candidate its query's $z$ selects +/// in its group. fn decode( inputs: &SelectionInputs<'_>, - variables: &SelectionVariables, + selection: &SelectionVariables, + variables: &ExecutionContextVariables, chosen: impl Fn(Variable) -> bool, -) -> DagOfSelectedCandidates { - let candidate_of_node: Vec = (0..variables.x.len()) - .filter(|&v| chosen(variables.x[v])) - .map(PhysicalCandidateId) +) -> (DagOfSelectedCandidates, Vec) { + let execution_of_node: Vec = inputs + .contexts + .executions() + .iter() + .copied() + .filter(|&execution| variables.x(execution).is_some_and(&chosen)) .collect(); - let node_of: HashMap = candidate_of_node + let node_of: HashMap = execution_of_node .iter() .enumerate() - .map(|(node, &candidate)| (candidate, SelectedNodeId(node))) + .map(|(node, &execution)| (execution, SelectedNodeId(node))) .collect(); - let edges = inputs.edges(); let mut selected_edges: BTreeSet<(usize, usize, usize)> = BTreeSet::new(); - for (edge, &y) in &variables.y { + for (&(context, edge), &y) in &variables.y_in_execution_context { if chosen(y) { - let edge = &edges[edge.0]; - if let (Some(producer), Some(consumer)) = ( - node_of.get(&edge.producer_candidate_id), - node_of.get(&edge.consumer_candidate_id), - ) { - selected_edges.insert((consumer.0, edge.input_index_of_consumer, producer.0)); + let (producer, consumer) = inputs.executions_of_edge(context, edge); + if let (Some(producer), Some(consumer)) = + (node_of.get(&producer), node_of.get(&consumer)) + { + selected_edges.insert(( + consumer.0, + inputs.edges()[edge.0].input_index_of_consumer, + producer.0, + )); } } } let node_of_output_reference = output_references(&inputs.physical.query_roots) .into_iter() .map(|(query, group)| { + let query = QueryId(query); inputs .candidates_of_group(group) - .into_iter() - .find(|v| { - variables - .z - .get(&(QueryId(query), *v)) - .is_some_and(|&z| chosen(z)) - }) - .and_then(|v| node_of.get(&v).copied()) + .iter() + .find(|v| selection.z.get(&(query, **v)).is_some_and(|&z| chosen(z))) + .and_then(|&v| inputs.output_execution(query, v)) + .and_then(|execution| node_of.get(&execution).copied()) .unwrap_or(SelectedNodeId(usize::MAX)) }) .collect(); - DagOfSelectedCandidates { - candidate_of_node, + let selected_dag = DagOfSelectedCandidates { + candidate_of_node: execution_of_node + .iter() + .map(|execution| execution.candidate()) + .collect(), edges: selected_edges .into_iter() .map( @@ -593,21 +799,8 @@ fn decode( ) .collect(), node_of_output_reference, - } -} - -#[allow(dead_code)] // #1207 PR 7. -/// The execution-context variables (section 5), which replace $x$ and the -/// unindexed $y$ of the baseline. -struct ExecutionContextVariables { - /// $x^{\mathrm{mat}}_v$, for an `AcrossEvaluations` candidate. - x_materialized: HashMap, - /// $x_{\kappa,v}$, for a candidate with $\ell$ = None or `OneEvaluation`. - x_in_execution_context: HashMap<(ExecutionContextId, PhysicalCandidateId), Variable>, - /// $y_{\kappa,u,c,i}$, indexed by the consumer's context. - y_in_execution_context: HashMap<(ExecutionContextId, PhysicalEdgeId), Variable>, - /// $h_v$, for an `AcrossEvaluations` producer. - retention_amount: HashMap, + }; + (selected_dag, execution_of_node) } #[allow(dead_code)] // #1207 PR 8. @@ -623,22 +816,6 @@ struct AccuracyVariables { failure_probability_of_input: HashMap<(PhysicalCandidateId, usize), Variable>, } -#[allow(dead_code)] // #1207 PR 7. -/// (R) $h_v \ge \operatorname{need}_{\kappa,v,c,i}\, y_{\kappa,v,c,i}$. -/// -/// Assumptions (decided 2026-11-06): an edge's need is the time horizon -/// the queries it serves read, a constant per edge (a 5m range over 10 s -/// windows needs 30 windows), so (R) is linear; query latency is ignored -/// until the latency design: a result counts as read when its evaluation -/// starts. -fn retention_constraints( - physical: &PhysicalASAPMemoCandidates, - variables: &ExecutionContextVariables, -) -> Vec { - let _ = (physical, variables); - todo!("#1207 PR 7: constraint (R)") -} - #[allow(dead_code)] // #1207 PR 8. /// (K): $E_c$ and $D_c$ per candidate, big-M on selected edges, and the /// root checks $E_v \le \varepsilon_q + M_q(1 - z_{q,v})$, diff --git a/crates/stage3-plan-selection/src/milp/model/tests.rs b/crates/stage3-plan-selection/src/milp/model/tests.rs index cc5b4ee31..4421e41e4 100644 --- a/crates/stage3-plan-selection/src/milp/model/tests.rs +++ b/crates/stage3-plan-selection/src/milp/model/tests.rs @@ -1,8 +1,10 @@ -//! The baseline MILP's variables and constraints (A)–(I), each on a small -//! memo built through `Memo::find_or_add_group` and `Memo::add_candidate` -//! with hand-made physical candidates, edges and costs $w_v$; and the -//! reference enumeration of the concepts doc's Appendix A against the MILP. -use std::collections::BTreeMap; +//! The MILP's variables and constraints (A)–(I) and (R) over execution +//! contexts, each on a small memo built through `Memo::find_or_add_group` +//! and `Memo::add_candidate` with hand-made physical candidates, edges and +//! costs; the contexts and their executions; and the reference enumeration +//! of the concepts doc's Appendix A over executions against the MILP. +use std::collections::{BTreeMap, BTreeSet}; +use std::time::Duration; use asap_types::ir::operator::operator_properties::{Reduction, Source}; use asap_types::ir::scalar::ColumnRef; @@ -10,19 +12,21 @@ use asap_types::ir::schema::{ DataType, Field, FieldDataType, GroupingStrategy, Schema, SketchAlgorithm, SketchParams, SketchStatistic, SummaryUpdate, }; -use asap_types::ir::{ASAPOp, NonASAPOp, Operator, QueryRoot}; +use asap_types::ir::{ASAPOp, NonASAPOp, Operator, QueryRoot, TimeRangeKind}; use asap_types::physical::{ExecutionTiming, Lifetime, PhysicalProperties, StorageMedium}; use asap_types::stages::{ - parameter_list, CandidateExpression, CandidateForMemoGroup, CandidateForMemoGroupId, - LogicalASAPMemoCandidates, LogicalProperties, LogicalRequirements, Memo, MemoGroupId, - OriginOfCandidateForMemoGroup, PhysicalASAPMemoCandidates, PhysicalCandidate, - PhysicalCandidateId, PhysicalEdge, WindowLayout, + parameter_list, producer_satisfies_physical_requirements, CandidateExpression, + CandidateForMemoGroup, CandidateForMemoGroupId, LogicalASAPMemoCandidates, LogicalProperties, + LogicalRequirements, Memo, MemoGroupId, OriginOfCandidateForMemoGroup, + PhysicalASAPMemoCandidates, PhysicalCandidate, PhysicalCandidateId, PhysicalEdge, WindowLayout, }; use asap_types::workload::{ - QueryRecurrence, QueryRequirements, QueryWorkloadEntry, RepeatedDemand, RepetitionInterval, - TimestampMs, + QueryRecurrence, QueryRequirements, QueryWorkloadEntry, RangeEvaluation, RepeatedDemand, + RepetitionInterval, TimestampMs, }; +use crate::execution_cost::ExecutionCosts; + use super::*; const QT_NONE: (ExecutionTiming, Lifetime) = (ExecutionTiming::QueryTime, Lifetime::None); @@ -156,18 +160,67 @@ fn example(inadmissible: &[(usize, usize, usize)]) -> PhysicalASAPMemoCandidates ) } -/// One query every 10 s at `phase_ms`. -fn entry(phase_ms: u64) -> QueryWorkloadEntry { +/// One query every `interval_ms` at `phase_ms`, its requests evaluating +/// `span_ms` back in 10 s steps. +fn entry_every(interval_ms: u32, phase_ms: u64, span_ms: u64) -> QueryWorkloadEntry { QueryWorkloadEntry { query: Default::default(), requirements: QueryRequirements::default(), predictability: Default::default(), recurrence: QueryRecurrence::Repeated(RepeatedDemand::FixedIntervalAt { - interval: RepetitionInterval(10_000), + interval: RepetitionInterval(interval_ms), evaluation_phase: TimestampMs(phase_ms), }), - range_evaluation: None, + range_evaluation: (span_ms > 0).then_some(RangeEvaluation { + step_ms: 10_000, + span_ms, + }), + } +} + +/// One query every 10 s at `phase_ms`. +fn entry(phase_ms: u64) -> QueryWorkloadEntry { + entry_every(10_000, phase_ms, 0) +} + +/// Every execution of `contexts` costing its candidate's `costs` entry +/// (`None`: not costable), in every context; each `AcrossEvaluations` +/// candidate in `retention` costing that per retained window. +fn uniform_costs( + contexts: &ExecutionContexts, + costs: &[Option], + retention: &[(usize, f64)], +) -> ExecutionCosts { + let mut execution_costs = ExecutionCosts::default(); + for &execution in contexts.executions() { + match costs[execution.candidate().0] { + Some(cost) => { + execution_costs.cost_per_second.insert(execution, cost); + } + None => { + execution_costs + .uncostable + .insert(execution, "not costable".into()); + } + } } + for &(v, cost) in retention { + execution_costs + .retention_cost_per_window_per_second + .insert(PhysicalCandidateId(v), cost); + } + execution_costs +} + +fn solve_with_retention( + physical: &PhysicalASAPMemoCandidates, + costs: &[Option], + retention: &[(usize, f64)], + entries: &[QueryWorkloadEntry], +) -> Result { + let contexts = ExecutionContexts::new(physical, entries); + let costs = uniform_costs(&contexts, costs, retention); + solve_selection(&SelectionInputs::new(physical, &contexts, &costs)) } fn solve( @@ -175,17 +228,19 @@ fn solve( costs: &[Option], entries: &[QueryWorkloadEntry], ) -> Result { - solve_selection(&SelectionInputs::new(physical, costs, entries)) + solve_with_retention(physical, costs, &[], entries) } -/// The selected candidates, ascending. +/// The selected candidates, one per selected execution, ascending. fn selected(solved: &SolvedSelection) -> Vec { - solved + let mut candidates: Vec = solved .selected_dag .candidate_of_node .iter() .map(|id| id.0) - .collect() + .collect(); + candidates.sort_unstable(); + candidates } /// (A), (B), (C)/(D) and the objective: the readout (C) reads exactly one @@ -334,92 +389,178 @@ fn strongly_connected_components_group_cycles_only() { assert_ne!(component[0], component[4]); } -/// The concepts doc's Appendix A over physical candidates: every DAG with -/// one admissible producer per input of every reached candidate, from -/// each root candidate, without cycles; each DAG's cost the sum of its -/// distinct candidates' $w_v$. Returns the cheapest cost. +/// The concepts doc's Appendix A over executions: every DAG with one +/// output execution per query output and one producer execution per input +/// of every reached execution (the edges the contexts allow), without +/// cycles; each DAG's cost the sum of its distinct executions' costs plus, +/// per `AcrossEvaluations` candidate, its retention cost per window times +/// the largest need of its edges. Returns the cheapest cost. fn reference_minimum( physical: &PhysicalASAPMemoCandidates, costs: &[Option], + retention: &[(usize, f64)], entries: &[QueryWorkloadEntry], ) -> Option { - let inputs = SelectionInputs::new(physical, costs, entries); - let edges = inputs.edges(); - let producers = |c: PhysicalCandidateId, i: usize| -> Vec { - physical - .admissible_edges_into(c, i) - .iter() - .map(|edge| &edges[edge.0]) - .filter(|edge| { - inputs.delivers_to_consumer(edge) && costs[edge.producer_candidate_id.0].is_some() - }) - .map(|edge| edge.producer_candidate_id) - .collect() - }; - fn reaches( - dag: &BTreeMap>, - from: PhysicalCandidateId, - to: PhysicalCandidateId, - ) -> bool { + let contexts = ExecutionContexts::new(physical, entries); + let execution_costs = uniform_costs(&contexts, costs, retention); + let inputs = SelectionInputs::new(physical, &contexts, &execution_costs); + let costed = |execution: Execution| inputs.cost_per_second(execution).is_some(); + // Per (consumer execution, input), its producer executions, each with + // the edge's context. + let mut producers: BTreeMap<(Execution, usize), Vec<(Execution, ExecutionContextId)>> = + BTreeMap::new(); + for &(context, edge) in contexts.edges() { + let (producer, consumer) = inputs.executions_of_edge(context, edge); + if costed(producer) && costed(consumer) { + producers + .entry((consumer, inputs.edges()[edge.0].input_index_of_consumer)) + .or_default() + .push((producer, context)); + } + } + type Dag = BTreeMap>; + fn reaches(dag: &Dag, from: Execution, to: Execution) -> bool { from == to || dag .get(&from) .into_iter() .flatten() - .any(|&next| reaches(dag, next, to)) + .any(|&(next, _)| reaches(dag, next, to)) } - fn expand( - inputs: &SelectionInputs<'_>, - producers: &dyn Fn(PhysicalCandidateId, usize) -> Vec, - dag: BTreeMap>, - mut pending: Vec, - best: &mut Option, - ) { - let Some(consumer) = pending.pop() else { - let cost: f64 = dag - .keys() - .map(|v| inputs.cost_per_second[v.0].unwrap()) - .sum(); - *best = Some(best.map_or(cost, |known: f64| known.min(cost))); - return; - }; - let mut choices: Vec> = vec![Vec::new()]; - for i in 0..inputs.input_count(consumer) { - choices = choices - .into_iter() - .flat_map(|prefix| { - producers(consumer, i).into_iter().map(move |u| { - let mut next = prefix.clone(); - next.push(u); - next + struct Search<'s, 'a> { + inputs: &'s SelectionInputs<'a>, + producers: &'s BTreeMap<(Execution, usize), Vec<(Execution, ExecutionContextId)>>, + best: Option, + } + impl Search<'_, '_> { + fn expand(&mut self, dag: Dag, roots: &BTreeSet, mut pending: Vec) { + let Some(consumer) = pending.pop() else { + let mut retained: BTreeMap = BTreeMap::new(); + for choices in dag.values() { + for &(producer, context) in choices { + if let Execution::Materialized(v) = producer { + let need = self.inputs.retention_need_windows(context, v); + let windows = retained.entry(v).or_default(); + *windows = (*windows).max(need); + } + } + } + let cost: f64 = dag + .keys() + .map(|&execution| self.inputs.cost_per_second(execution).unwrap()) + .sum::() + + retained + .into_iter() + .map(|(v, windows)| { + self.inputs + .costs + .retention_cost_per_window_per_second + .get(&v) + .map_or(0.0, |cost| cost * windows as f64) + }) + .sum::(); + self.best = Some(self.best.map_or(cost, |known: f64| known.min(cost))); + return; + }; + let mut choices: Vec> = vec![Vec::new()]; + for i in 0..self.inputs.input_count(consumer.candidate()) { + let options = self + .producers + .get(&(consumer, i)) + .cloned() + .unwrap_or_default(); + choices = choices + .into_iter() + .flat_map(|prefix| { + options.iter().map(move |&option| { + let mut next = prefix.clone(); + next.push(option); + next + }) }) - }) - .collect(); - } - for choice in choices { - if choice.iter().any(|&u| reaches(&dag, u, consumer)) { - continue; + .collect(); } - let mut next = dag.clone(); - next.insert(consumer, choice.clone()); - let mut next_pending = pending.clone(); - for &u in &choice { - if !next.contains_key(&u) && !next_pending.contains(&u) { - next_pending.push(u); + for choice in choices { + if choice.iter().any(|&(u, _)| reaches(&dag, u, consumer)) { + continue; } + let mut next = dag.clone(); + next.insert(consumer, choice.clone()); + let mut next_pending = pending.clone(); + for &(u, _) in &choice { + if !next.contains_key(&u) && !next_pending.contains(&u) && !roots.contains(&u) { + next_pending.push(u); + } + } + self.expand(next, roots, next_pending); } - expand(inputs, producers, next, next_pending, best); } } - let mut best = None; - for (_, group) in inputs.query_output_groups() { - for root in inputs.candidates_of_group(group) { - if costs[root.0].is_some() { - expand(&inputs, &producers, BTreeMap::new(), vec![root], &mut best); - } + // Every choice of one output execution per query output. + let mut root_choices: Vec> = vec![BTreeSet::new()]; + for &(query, group) in inputs.query_output_groups() { + let options: Vec = inputs + .candidates_of_group(group) + .iter() + .filter_map(|&v| inputs.output_execution(query, v)) + .collect(); + root_choices = root_choices + .into_iter() + .flat_map(|prefix| { + options.iter().map(move |&option| { + let mut next = prefix.clone(); + next.insert(option); + next + }) + }) + .collect(); + } + let mut search = Search { + inputs: &inputs, + producers: &producers, + best: None, + }; + for roots in root_choices { + search.expand(BTreeMap::new(), &roots, roots.iter().copied().collect()); + } + search.best +} + +/// A deterministic pseudo-random sequence of costs in [0, 10). +fn pseudo_random_costs() -> impl FnMut() -> f64 { + let mut state = 0x2545_f491_4f6c_dd1d_u64; + move || { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + (state % 1000) as f64 / 100.0 + } +} + +/// Whether the MILP's optimum on `physical` equals the reference +/// enumeration's cheapest DAG, and its decoded DAG validates. +fn agrees_with_the_reference( + physical: &PhysicalASAPMemoCandidates, + costs: &[Option], + retention: &[(usize, f64)], + entries: &[QueryWorkloadEntry], +) { + let reference = reference_minimum(physical, costs, retention, entries); + match ( + reference, + solve_with_retention(physical, costs, retention, entries), + ) { + (Some(reference), Ok(solved)) => { + assert!( + (reference - solved.objective_per_second).abs() < 1e-6, + "costs {costs:?} retention {retention:?}: reference {reference}, MILP {}", + solved.objective_per_second + ); + solved.selected_dag.validate(physical).unwrap(); } + (None, Err(SelectionMilpError::Infeasible)) => {} + (reference, milp) => panic!("costs {costs:?}: reference {reference:?}, MILP {milp:?}"), } - best } /// On the small example, under many cost assignments and both query @@ -428,34 +569,467 @@ fn reference_minimum( #[test] fn the_milp_agrees_with_the_reference_enumeration() { let physical = example(&[(3, 5, 0)]); - // A deterministic pseudo-random sequence of costs in [0, 10). - let mut state = 0x2545_f491_4f6c_dd1d_u64; - let mut next = || { - state ^= state << 13; - state ^= state >> 7; - state ^= state << 17; - (state % 1000) as f64 / 100.0 - }; + let mut next = pseudo_random_costs(); for round in 0..50 { let costs: Vec> = (0..6).map(|id| (round % 7 != id).then(&mut next)).collect(); for phase_ms in [0, 5_000] { - let entries = [entry(phase_ms)]; - let reference = reference_minimum(&physical, &costs, &entries); - let milp = solve(&physical, &costs, &entries); - match (reference, milp) { - (Some(reference), Ok(solved)) => { - assert!( - (reference - solved.objective_per_second).abs() < 1e-9, - "costs {costs:?} phase {phase_ms}: reference {reference}, MILP {}", - solved.objective_per_second - ); - solved.selected_dag.validate(&physical).unwrap(); - } - (None, Err(SelectionMilpError::Infeasible)) => {} - (reference, milp) => { - panic!("costs {costs:?}: reference {reference:?}, MILP {milp:?}") - } - } + agrees_with_the_reference(&physical, &costs, &[], &[entry(phase_ms)]); } } } + +/// Over execution contexts and retention, the MILP agrees with the +/// reference enumeration on the windowed example with two queries: +/// in one context or two, with range evaluations of different spans, the +/// chain's retention cost per window random too. +#[test] +fn the_milp_agrees_with_the_reference_enumeration_over_contexts_and_retention() { + let physical = windowed_example(); + let mut next = pseudo_random_costs(); + for round in 0..30 { + let costs: Vec> = (0..WINDOWED_CANDIDATES) + .map(|id| (round % 9 != id).then(&mut next)) + .collect(); + let retention = [(CHAIN_10S, next() / 10.0), (CHAIN_5S, next() / 10.0)]; + for entries in [ + [entry(0), entry(0)], + [entry(0), entry(5_000)], + [entry(0), entry_every(10_000, 0, 50_000)], + [ + entry_every(20_000, 0, 0), + entry_every(10_000, 5_000, 30_000), + ], + ] { + agrees_with_the_reference(&physical, &costs, &retention, &entries); + } + } +} + +// Candidates of `windowed_example`. +const RAW: usize = 0; +const RANGE: usize = 1; +const BUILD: usize = 2; +const CHAIN_10S: usize = 3; +const CHAIN_5S: usize = 4; +const READOUT_A: usize = 5; +const READOUT_B: usize = 6; +const BUILD_ONE_EVALUATION: usize = 7; +const WINDOWED_CANDIDATES: usize = 8; + +/// Two queries over one KLL state of `latency[1m]`: a scan group (`RAW`), +/// the 1m range over it (`RANGE`), the state group with a build at query +/// time, not materialized (`BUILD`) or `OneEvaluation` +/// (`BUILD_ONE_EVALUATION`), and builds of 10 s and 5 s tumbling windows +/// materialized across evaluations at query time (`CHAIN_10S`, +/// `CHAIN_5S`); query A's p95 (`READOUT_A`) and query B's p50 +/// (`READOUT_B`) over it. Every other candidate runs at query time, not +/// materialized; every edge is admissible. +fn windowed_example() -> PhysicalASAPMemoCandidates { + windowed_example_with(ExecutionTiming::QueryTime, 5_000) +} + +/// [`windowed_example`] with the two chains at `chain_timing` (the scan +/// and the range too at ingestion time), the second `second_width_ms` +/// wide; edges admissible when $\alpha^{\mathrm{phys}}$ holds. +fn windowed_example_with( + chain_timing: ExecutionTiming, + second_width_ms: u64, +) -> PhysicalASAPMemoCandidates { + let mut memo = Memo::default(); + let scan_operator = Operator::NonASAP(NonASAPOp::Scan { + source: Source::TimeSeries { + metric: "latency".into(), + }, + predicates: vec![], + schema: Schema::with_time_index( + vec![ + Field::plain("ts", DataType::Timestamp, false), + Field::plain("value", DataType::Float64, false), + ], + 0, + vec![], + ), + }); + let scan_group = memo.find_or_add_group(scan_operator.clone()).unwrap(); + let mut logical = vec![add(&mut memo, scan_group, scan_operator, None, None)]; + let range_operator = Operator::NonASAP(NonASAPOp::TimeRange { + range: Duration::from_secs(60), + kind: TimeRangeKind::Range, + child: scan_group, + }); + let range_group = memo.find_or_add_group(range_operator.clone()).unwrap(); + logical.push(add(&mut memo, range_group, range_operator, None, None)); + let build = Operator::ASAP(ASAPOp::SummaryAgg { + child: range_group, + family: FieldDataType::Sketch(SketchAlgorithm::Kll, GroupingStrategy::default()), + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::PerEntity, + grouping: GroupingStrategy::default(), + filter: None, + }); + let states = memo.find_or_add_group(build.clone()).unwrap(); + let parameters = parameter_list(SketchAlgorithm::Kll)[0].clone(); + for window_layout in [ + None, + Some(WindowLayout::Tumbling { width_ms: 10_000 }), + Some(WindowLayout::Tumbling { + width_ms: second_width_ms, + }), + ] { + logical.push(add( + &mut memo, + states, + build.clone(), + Some(parameters.clone()), + window_layout, + )); + } + for q in [0.95, 0.5] { + let readout = Operator::ASAP(ASAPOp::SummaryEstimate { + summary_input: states, + query: SketchStatistic::Quantile { q }, + }); + let group = memo.find_or_add_group(readout.clone()).unwrap(); + logical.push(add(&mut memo, group, readout, None, None)); + memo.add_query(QueryRoot::Operator(group)); + } + // The unwindowed build again, `OneEvaluation`. + logical.push(logical[BUILD]); + let qt = |lifetime| PhysicalProperties::new(ExecutionTiming::QueryTime, lifetime).unwrap(); + let mut physical_candidates: Vec = logical + .iter() + .enumerate() + .map(|(id, &logical_candidate_id)| PhysicalCandidate { + logical_candidate_id, + physical_properties: match id { + CHAIN_10S | CHAIN_5S => PhysicalProperties::new( + chain_timing, + Lifetime::AcrossEvaluations(StorageMedium::Memory), + ) + .unwrap(), + BUILD_ONE_EVALUATION => qt(Lifetime::OneEvaluation(StorageMedium::Memory)), + _ => qt(Lifetime::None), + }, + }) + .collect(); + if chain_timing == ExecutionTiming::IngestionTime { + for id in [RAW, RANGE] { + physical_candidates[id].physical_properties = + PhysicalProperties::new(ExecutionTiming::IngestionTime, Lifetime::None).unwrap(); + } + } + // Grouped by (consumer, input). + let mut edges = vec![(RAW, RANGE, 0)]; + for state in [BUILD, CHAIN_10S, CHAIN_5S] { + edges.push((RANGE, state, 0)); + } + for readout in [READOUT_A, READOUT_B] { + for state in [BUILD, CHAIN_10S, CHAIN_5S, BUILD_ONE_EVALUATION] { + edges.push((state, readout, 0)); + } + } + edges.push((RANGE, BUILD_ONE_EVALUATION, 0)); + let edges = edges + .into_iter() + .map(|(producer, consumer, input)| PhysicalEdge { + producer_candidate_id: PhysicalCandidateId(producer), + consumer_candidate_id: PhysicalCandidateId(consumer), + input_index_of_consumer: input, + satisfies_logical_requirement: true, + satisfies_physical_requirement: producer_satisfies_physical_requirements( + &physical_candidates[producer], + &physical_candidates[consumer], + ), + }) + .collect(); + let query_roots = memo.query_roots.clone(); + PhysicalASAPMemoCandidates::new( + LogicalASAPMemoCandidates { + memo, + edges: Vec::new(), + }, + physical_candidates, + query_roots, + edges, + ) +} + +/// Costs of `windowed_example`'s candidates: `cheap` cost 1, the others +/// 100. +fn costs_where_cheap(cheap: &[usize]) -> Vec> { + (0..WINDOWED_CANDIDATES) + .map(|id| Some(if cheap.contains(&id) { 1.0 } else { 100.0 })) + .collect() +} + +/// Contexts are deduplicated by equality: queries with equal recurrence +/// and range evaluation share one evaluation context, different ones get +/// their own; a materialized chain's inputs run in its own computation +/// context. +#[test] +fn contexts_are_deduplicated_by_equality() { + let physical = windowed_example(); + let same = ExecutionContexts::new(&physical, &[entry(0), entry(0)]); + assert_eq!(same.context_of_query(0), same.context_of_query(1)); + let different = ExecutionContexts::new(&physical, &[entry(0), entry(5_000)]); + assert_ne!(different.context_of_query(0), different.context_of_query(1)); + let spans = ExecutionContexts::new(&physical, &[entry(0), entry_every(10_000, 0, 50_000)]); + assert_ne!(spans.context_of_query(0), spans.context_of_query(1)); + // Two evaluations and each chain's computation. + assert_eq!(different.contexts().len(), 4); + for chain in [CHAIN_10S, CHAIN_5S] { + let context = different + .computation_context(PhysicalCandidateId(chain)) + .unwrap(); + assert_eq!( + different.contexts()[context.0], + ExecutionContext::Materialized(PhysicalCandidateId(chain)) + ); + } +} + +/// $x_{\kappa,v}$: a not-materialized candidate read by two queries of one +/// context runs once there, and its cost is counted once; with the queries +/// in two contexts it runs in each, as two nodes of the selected DAG, and +/// its cost is counted twice. +#[test] +fn a_not_materialized_candidate_runs_once_per_context_that_selects_it() { + let physical = windowed_example(); + let costs = costs_where_cheap(&[RAW, RANGE, BUILD, READOUT_A, READOUT_B]); + let shared = solve(&physical, &costs, &[entry(0), entry(0)]).unwrap(); + assert_eq!(selected(&shared), [RAW, RANGE, BUILD, READOUT_A, READOUT_B]); + assert_eq!(shared.objective_per_second, 5.0); + shared.selected_dag.validate(&physical).unwrap(); + let apart = solve(&physical, &costs, &[entry(0), entry(5_000)]).unwrap(); + assert_eq!( + selected(&apart), + [RAW, RAW, RANGE, RANGE, BUILD, BUILD, READOUT_A, READOUT_B] + ); + assert_eq!(apart.objective_per_second, 8.0); + apart.selected_dag.validate(&physical).unwrap(); +} + +/// A `OneEvaluation` result belongs to one evaluation context: read by +/// two queries of one context, it runs once there. +#[test] +fn a_one_evaluation_result_is_shared_within_its_context() { + let physical = windowed_example(); + let costs = costs_where_cheap(&[RAW, RANGE, BUILD_ONE_EVALUATION, READOUT_A, READOUT_B]); + let solved = solve(&physical, &costs, &[entry(0), entry(0)]).unwrap(); + assert_eq!( + selected(&solved), + [RAW, RANGE, READOUT_A, READOUT_B, BUILD_ONE_EVALUATION] + ); + let contexts = ExecutionContexts::new(&physical, &[entry(0), entry(0)]); + // Never inside a chain's computation. + assert!(contexts + .executions() + .iter() + .all(|execution| match execution { + Execution::InContext(context, v) if v.0 == BUILD_ONE_EVALUATION => matches!( + contexts.contexts()[context.0], + ExecutionContext::Evaluation { .. } + ), + _ => true, + })); +} + +/// #1204 regression: two 10 s queries at phases 0 s and 5 s both read one +/// chain of 5 s tumbling windows materialized across evaluations: one +/// $x^{\mathrm{mat}}$, an edge $y_{\kappa,u,c,i}$ in each query's context, +/// its cost counted once. The 10 s chain delivers only on phase 0 +/// (`delivers_on` per context), so it has no edge in phase 5 s's context. +#[test] +fn two_queries_at_phases_0_and_5_s_read_one_5_s_chain() { + let physical = windowed_example(); + let contexts = ExecutionContexts::new(&physical, &[entry(0), entry(5_000)]); + let edges_out_of = |chain: usize| -> Vec { + contexts + .edges() + .iter() + .filter(|(_, edge)| { + physical + .edges() + .nth(edge.0) + .unwrap() + .1 + .producer_candidate_id + .0 + == chain + }) + .map(|&(context, _)| context) + .collect() + }; + assert_eq!(edges_out_of(CHAIN_5S).len(), 2); + assert_eq!( + edges_out_of(CHAIN_10S), + [contexts.context_of_query(0).unwrap()] + ); + let costs = costs_where_cheap(&[RAW, RANGE, CHAIN_5S, READOUT_A, READOUT_B]); + let solved = solve(&physical, &costs, &[entry(0), entry(5_000)]).unwrap(); + assert_eq!( + selected(&solved), + [RAW, RANGE, CHAIN_5S, READOUT_A, READOUT_B] + ); + assert_eq!( + solved + .execution_of_node + .iter() + .filter(|execution| matches!(execution, Execution::Materialized(_))) + .count(), + 1 + ); + assert_eq!(solved.objective_per_second, 5.0); + solved.selected_dag.validate(&physical).unwrap(); +} + +/// (R): a chain of 10 s windows over a 1m range retains as many windows as +/// its most demanding selected consumer reads: 6 for queries evaluated +/// once per request, 11 when one evaluates 50 s back from each request's +/// end; the objective costs them at $\pi_v$ each. +#[test] +fn a_chain_retains_what_its_most_demanding_consumer_reads() { + let physical = windowed_example(); + let costs = costs_where_cheap(&[RAW, RANGE, CHAIN_10S, READOUT_A, READOUT_B]); + let retention = [(CHAIN_10S, 0.5)]; + let once_per_request = + solve_with_retention(&physical, &costs, &retention, &[entry(0), entry(0)]).unwrap(); + assert_eq!( + selected(&once_per_request), + [RAW, RANGE, CHAIN_10S, READOUT_A, READOUT_B] + ); + assert_eq!(once_per_request.objective_per_second, 5.0 + 0.5 * 6.0); + let with_span = solve_with_retention( + &physical, + &costs, + &retention, + &[entry(0), entry_every(10_000, 0, 50_000)], + ) + .unwrap(); + assert_eq!( + selected(&with_span), + [RAW, RANGE, CHAIN_10S, READOUT_A, READOUT_B] + ); + assert_eq!(with_span.objective_per_second, 5.0 + 0.5 * 11.0); + // The MILP's own optimum, through h_v, agrees: its bound is the + // objective. + assert!((with_span.lower_bound_per_second - with_span.objective_per_second).abs() < 1e-6); +} + +/// The range a chain is built over, and the scan below it, run in the +/// chain's own computation context, not in the queries' evaluation +/// context. +#[test] +fn a_chain_reads_its_inputs_in_its_own_computation_context() { + let physical = windowed_example(); + let costs = costs_where_cheap(&[RAW, RANGE, CHAIN_10S, READOUT_A, READOUT_B]); + let solved = solve(&physical, &costs, &[entry(0), entry(0)]).unwrap(); + let chain_context = ExecutionContextId( + solved + .execution_of_node + .iter() + .find_map(|execution| match execution { + Execution::InContext(context, v) if v.0 == RANGE => Some(context.0), + _ => None, + }) + .unwrap(), + ); + let contexts = ExecutionContexts::new(&physical, &[entry(0), entry(0)]); + assert_eq!( + contexts.computation_context(PhysicalCandidateId(CHAIN_10S)), + Some(chain_context) + ); +} + +/// Chains maintained at ingestion time read their inputs in the +/// ingestion pipeline of their refresh window, shared by the chains with +/// equal ones: two chains 10 s wide share one pipeline, where the scan +/// and the range below them run once; chains 10 s and 5 s wide get one +/// pipeline each. +#[test] +fn chains_at_ingestion_share_the_pipeline_of_their_refresh_window() { + let pipeline_of = |contexts: &ExecutionContexts| { + [CHAIN_10S, CHAIN_5S].map(|chain| { + let context = contexts + .computation_context(PhysicalCandidateId(chain)) + .unwrap(); + assert!(matches!( + contexts.contexts()[context.0], + ExecutionContext::IngestionPipeline(_) + )); + context + }) + }; + let shared = windowed_example_with(ExecutionTiming::IngestionTime, 10_000); + let contexts = ExecutionContexts::new(&shared, &[entry(0), entry(0)]); + let [first, second] = pipeline_of(&contexts); + assert_eq!(first, second); + let scans: Vec<&Execution> = contexts + .executions() + .iter() + .filter(|execution| execution.candidate().0 == RAW) + .collect(); + assert_eq!( + scans, + [&Execution::InContext(first, PhysicalCandidateId(RAW))] + ); + let apart = windowed_example_with(ExecutionTiming::IngestionTime, 5_000); + let contexts = ExecutionContexts::new(&apart, &[entry(0), entry(0)]); + let [first, second] = pipeline_of(&contexts); + assert_ne!(first, second); +} + +/// The objective's costs from `asap-cost` on the windowed example: $\rho_\kappa$ +/// per context (an evaluation every 10 s: 0.1/s; a query-time chain one +/// build per window: 0.1/s for 10 s windows, 0.2/s for 5 s); the range a +/// chain is built over costs nothing in the chain's computation, but does +/// in the queries' evaluation; each chain has a positive cost $\pi_v$ per +/// retained window; every execution is costed. +#[test] +fn execution_costs_follow_the_context_of_each_execution() { + use crate::execution_cost::{execution_costs, executions_per_second}; + use asap_cost::CostInputs; + use asap_types::workload::{DataArrival, DataWorkload, DurationMs, SourceFacts}; + let physical = windowed_example(); + let entries = [entry(0), entry(0)]; + let contexts = ExecutionContexts::new(&physical, &entries); + let cost_inputs = CostInputs::default(); + let rate = |context: ExecutionContextId| { + executions_per_second(&physical, &contexts.contexts()[context.0], &cost_inputs).unwrap() + }; + let evaluation = contexts.context_of_query(0).unwrap(); + let chain_10s = contexts + .computation_context(PhysicalCandidateId(CHAIN_10S)) + .unwrap(); + let chain_5s = contexts + .computation_context(PhysicalCandidateId(CHAIN_5S)) + .unwrap(); + assert!((rate(evaluation) - 0.1).abs() < 1e-12); + assert!((rate(chain_10s) - 0.1).abs() < 1e-12); + assert!((rate(chain_5s) - 0.2).abs() < 1e-12); + let data = DataWorkload { + arrival: DataArrival::Ingesting { + rows_per_second: 100.0, + }, + sources: [( + "latency".to_string(), + SourceFacts { + sample_interval: Some(DurationMs(10_000)), + series_count: Some(1_000), + ..SourceFacts::default() + }, + )] + .into(), + }; + let costs = execution_costs(&physical, &contexts, &data, &cost_inputs, 1.0); + assert_eq!(costs.uncostable, BTreeMap::new()); + let range = + |context| costs.cost_per_second[&Execution::InContext(context, PhysicalCandidateId(RANGE))]; + assert_eq!(range(chain_10s), 0.0); + assert_eq!(range(chain_5s), 0.0); + assert!(range(evaluation) > 0.0); + for chain in [CHAIN_10S, CHAIN_5S] { + let retention = costs.retention_cost_per_window_per_second[&PhysicalCandidateId(chain)]; + assert!(retention > 0.0 && retention.is_finite()); + } +} diff --git a/crates/stage3-plan-selection/src/report.rs b/crates/stage3-plan-selection/src/report.rs index 27366573d..01f308b5c 100644 --- a/crates/stage3-plan-selection/src/report.rs +++ b/crates/stage3-plan-selection/src/report.rs @@ -211,19 +211,22 @@ pub struct SearchReport { pub solver_ms: u64, pub validation_ms: u64, /// Set when `asap_cost::plan_cost` of the selected plan differs from - /// the selection MILP's objective and the model encoded some cost terms - /// inexactly (open questions on #1095, e.g. the working-memory term), so - /// the plan is returned with its cost equality unproven. `None` when - /// the two are equal (within a relative 10⁻⁶). A - /// difference with no inexact term is an error + /// the selection MILP's objective (by more than a relative 10⁻⁶), with + /// the terms of `plan_cost` the formulation's objective does not encode + /// that the selection may meet (e.g. the memory a query-time result + /// holds while its evaluation runs, which needs query latency); the + /// plan is returned with its cost equality unproven. `None` when the + /// two are equal. A difference with no such term is an error /// ([`ErrorSelectingPlan::ObjectiveDiffersFromPlanCost`]). #[serde(default, skip_serializing_if = "Option::is_none")] pub inexact_objective: Option, } /// The selected plan's cost per second as the selection MILP's objective -/// and as `asap_cost::plan_cost` computes it (both without request fees), -/// and the cost terms the model encoded inexactly, with how many times. +/// and as `asap_cost::plan_cost` computes it (both with the runtime's +/// request fees), and the terms of `plan_cost` the objective does not +/// encode that the selection may meet, each with how many selected +/// executions it may concern. #[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] pub struct InexactObjective { pub objective_per_second: f64, @@ -288,11 +291,12 @@ pub enum ErrorSelectingPlan { #[error("the selection MILP does not support {0}")] UnsupportedCostModel(String), /// `asap_cost::plan_cost` of the selected plan differs from the - /// selection MILP's objective (both without request fees) and the model - /// encoded every cost term exactly: a bug in a cost term. + /// selection MILP's objective and the selection meets none of the terms + /// the formulation's objective leaves out + /// ([`InexactObjective::inexact_terms`]): a bug in a cost coefficient. #[error( "the selection MILP's objective {objective_per_second:e} differs from plan_cost \ - {plan_cost_per_second:e} (per second, without request fees)" + {plan_cost_per_second:e} (per second) with no term the objective leaves out" )] ObjectiveDiffersFromPlanCost { objective_per_second: f64, diff --git a/crates/stage3-plan-selection/src/select.rs b/crates/stage3-plan-selection/src/select.rs index caab227f4..50df9037f 100644 --- a/crates/stage3-plan-selection/src/select.rs +++ b/crates/stage3-plan-selection/src/select.rs @@ -1,7 +1,8 @@ -//! Stage 3, plan selection: [`select_plan`] costs every physical candidate -//! ($w_v$, [`crate::candidate_cost`]), builds and solves the baseline -//! selection MILP over Stage 2's candidates and admissible edges -//! ([`crate::milp::model`]), decodes $\{x = 1\}$, $\{y = 1\}$ and +//! Stage 3, plan selection: [`select_plan`] lists the workload's execution +//! contexts with each execution and edge they allow +//! ([`crate::milp::execution_contexts`]), costs every execution +//! ([`crate::execution_cost`]), builds and solves the selection MILP over +//! them ([`crate::milp::model`]), decodes $\{x = 1\}$, $\{y = 1\}$ and //! $\{z = 1\}$ into a //! [`DagOfSelectedCandidates`](asap_types::stages::DagOfSelectedCandidates), //! and realizes it once through the injected realizer. @@ -9,17 +10,21 @@ use std::collections::BTreeMap; use asap_accuracy::AccuracyModel; +use asap_cost::analytical::AnalyticalCostError; +use asap_cost::execution_demand::evaluation_rate; use asap_cost::{materialized_tumbling_window_chains, node_map, plan_cost, CostInputs, CostModel}; use asap_types::ir::schema::FieldDataType; -use asap_types::ir::{ASAPOp, OperatorNode, QueryRoot}; +use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode, QueryRoot}; +use asap_types::physical::{ExecutionTiming, Lifetime, StorageMedium}; use asap_types::stages::{ - AssignmentRealizer, PhysicalASAPMemoCandidates, PhysicalCandidateError, PhysicalCandidateId, - PruneReason, PrunedCandidate, RealizedPlan, + AssignmentRealizer, ExecutionContext, ExecutionContextId, PhysicalASAPMemoCandidates, + PhysicalCandidateError, PhysicalCandidateId, PruneReason, PrunedCandidate, RealizedPlan, }; use asap_types::workload::{DataWorkload, QueryWorkloadEntry}; -use crate::candidate_cost::candidate_costs; -use crate::milp::model::{solve_selection, SelectionInputs, SelectionMilpError}; +use crate::execution_cost::{execution_costs, ExecutionCosts}; +use crate::milp::execution_contexts::{Execution, ExecutionContexts}; +use crate::milp::model::{solve_selection, SelectionInputs, SelectionMilpError, SolvedSelection}; use crate::models::AccuracyCheckInputs; use crate::report::{ AcceptedAccuracy, ErrorSelectingPlan, InexactObjective, Rejection, RejectionCause, @@ -30,10 +35,27 @@ use crate::report::{ /// of the selected plan count as equal. const COST_EQUALITY_TOLERANCE: f64 = 1e-6; -/// The name of the inexact term of the baseline objective, with the -/// number of selected candidates it covers in [`InexactObjective`]. -const FIXED_COST_PER_CANDIDATE: &str = - "fixed cost per candidate, whichever consumers select it (execution contexts: #1207 PR 7)"; +/// The terms of `asap_cost::plan_cost` the formulation's objective (concepts +/// doc, section 5) does not encode, named in [`InexactObjective`] with the +/// number of selected executions each may concern. +const HELD_MEMORY: &str = "memory a query-time result holds while its evaluation runs: \ + plan_cost holds it for the latency of the queries reading it; the objective ignores query \ + latency (out of scope)"; +const RUNS_PER_QUERY: &str = "a not-materialized execution in a context of several queries: \ + plan_cost computes it once per query reaching it, the objective once per context"; +const CHAIN_BUILDS_PER_REQUEST: &str = "a tumbling-window chain materialized at query time: \ + plan_cost builds it once per request of the queries reaching it, the objective once per \ + window"; +const CHAIN_MERGE: &str = "the merge the realizer reads a tumbling-window chain materialized \ + across evaluations through: plan_cost runs it per query reading it; it is not a candidate"; +const REPRESENTATIVE_PRODUCER: &str = "a consumer costed over the output of another logical \ + candidate of its input than the selected one: the objective costs each execution over one \ + representative producer per input"; +const RANGE_UNDER_CHAIN: &str = "a range read only by tumbling-window chains run in its \ + context: the realizer builds each chain over the range's input, so plan_cost has no range"; +const QUERY_TIME_SCAN: &str = "a query-time scan: plan_cost reads the windows of its selected \ + consumers and retains the raw data of the largest scan of each source; the objective reads \ + the windows of every consumer that may read it in its context and retains each scan's"; /// Stage 3, plan selection: select one plan among `physical`'s candidates /// (module documentation). The selection is realized through `realizer` @@ -82,25 +104,17 @@ pub fn select_plan( }); } let formula_scale = cost_inputs.frozen_calibration_factor.unwrap_or(1.0); - let costs = candidate_costs(physical, entries, data, cost_inputs, formula_scale); - let pruned_candidates: Vec> = costs - .iter() - .enumerate() - .filter(|(_, cost)| cost.is_err()) - .map(|(id, _)| PrunedCandidate { - group: physical.physical_candidates[id].logical_candidate_id.group, - candidate: PhysicalCandidateId(id), - reason: PruneReason::UsageNotCostable, - }) - .collect(); - let cost_per_second: Vec> = costs.iter().map(|cost| cost.clone().ok()).collect(); - - let solved = solve_selection(&SelectionInputs::new(physical, &cost_per_second, entries)) - .map_err(|error| match error { - SelectionMilpError::Infeasible => { - ErrorSelectingPlan::NoValidCandidate(vec![infeasible(&costs)]) + let contexts = ExecutionContexts::new(physical, entries); + let costs = execution_costs(physical, &contexts, data, cost_inputs, formula_scale); + let pruned_candidates = uncostable_candidates(physical, &contexts, &costs); + let solved = + solve_selection(&SelectionInputs::new(physical, &contexts, &costs)).map_err(|error| { + match error { + SelectionMilpError::Infeasible => { + ErrorSelectingPlan::NoValidCandidate(vec![infeasible(&costs)]) + } + SelectionMilpError::Solver(reason) => ErrorSelectingPlan::Solver(reason), } - SelectionMilpError::Solver(reason) => ErrorSelectingPlan::Solver(reason), })?; let validation_started = std::time::Instant::now(); solved.selected_dag.validate(physical).map_err(|reason| { @@ -136,17 +150,37 @@ pub fn select_plan( let plan_cost_per_second = cost.cost_per_second.ok_or_else(|| { ErrorSelectingPlan::UnsupportedCostModel("a cost model without a cost per second".into()) })?; - let objective_per_second = solved.objective_per_second; + // The runtime's fee per query request: the same for every plan, so a + // constant of the objective. + let objective_per_second = solved.objective_per_second + + request_cost_per_second(entries, cost_inputs).map_err(|error| { + ErrorSelectingPlan::Cost { + candidate: plan.id.clone(), + node: plan.dag.nodes.first().map_or(0, |node| node.id), + error, + } + })?; let difference = (objective_per_second - plan_cost_per_second).abs(); let tolerance = COST_EQUALITY_TOLERANCE * plan_cost_per_second.abs().max(f64::MIN_POSITIVE); - let inexact_objective = (difference > tolerance).then(|| InexactObjective { - objective_per_second, - plan_cost_per_second, - inexact_terms: BTreeMap::from([( - FIXED_COST_PER_CANDIDATE.to_string(), - solved.selected_dag.candidate_of_node.len(), - )]), - }); + let inexact_objective = match difference > tolerance { + false => None, + true => { + let inexact_terms = inexact_terms(physical, &contexts, &costs, cost_inputs, &solved); + // A difference none of the formulation's known gaps explains is + // a bug in a cost coefficient. + if inexact_terms.is_empty() { + return Err(ErrorSelectingPlan::ObjectiveDiffersFromPlanCost { + objective_per_second, + plan_cost_per_second, + }); + } + Some(InexactObjective { + objective_per_second, + plan_cost_per_second, + inexact_terms, + }) + } + }; let accepted_accuracy_per_query = accepted_accuracy(&realized, entries, accuracy_checks.model); let validation_ms = validation_started.elapsed().as_millis() as u64; Ok(Selection { @@ -160,7 +194,10 @@ pub fn select_plan( // plan's cost only when the objective equals it. cost_lower_bound_per_second: (inexact_objective.is_none() && solved.lower_bound_per_second.is_finite()) - .then_some(solved.lower_bound_per_second), + .then_some( + solved.lower_bound_per_second + + (objective_per_second - solved.objective_per_second), + ), milp_variables: solved.variable_count, milp_constraints: solved.constraint_count, model_construction_ms: solved.construction_ms, @@ -171,13 +208,13 @@ pub fn select_plan( }) } -/// The rejection of an infeasible selection: no plan meets (A)–(I), with -/// the reasons `asap-cost` could not cost the candidates the model left -/// out, each once in candidate order: those candidates may be why no plan -/// is left. -fn infeasible(costs: &[Result]) -> Rejection { +/// The rejection of an infeasible selection: no plan meets (A)–(I) and +/// (R), with the reasons `asap-cost` could not cost the executions the +/// model left out, each once in execution order: those executions may be +/// why no plan is left. +fn infeasible(costs: &ExecutionCosts) -> Rejection { let mut reasons: Vec<&str> = Vec::new(); - for reason in costs.iter().filter_map(|cost| cost.as_ref().err()) { + for reason in costs.uncostable.values() { if !reasons.contains(&reason.as_str()) { reasons.push(reason); } @@ -187,7 +224,7 @@ fn infeasible(costs: &[Result]) -> Rejection { .to_string(); if !reasons.is_empty() { reason += &format!( - "; candidates left out because they could not be costed: {}", + "; executions left out because they could not be costed: {}", reasons.join("; ") ); } @@ -198,6 +235,174 @@ fn infeasible(costs: &[Result]) -> Rejection { } } +/// The candidates Stage 3 rules out before the search: those with an +/// execution none of which `asap-cost` could cost, in candidate order. +fn uncostable_candidates( + physical: &PhysicalASAPMemoCandidates, + contexts: &ExecutionContexts, + costs: &ExecutionCosts, +) -> Vec> { + let mut costed: BTreeMap = BTreeMap::new(); + for &execution in contexts.executions() { + *costed.entry(execution.candidate()).or_default() |= + costs.cost_per_second.contains_key(&execution); + } + costed + .into_iter() + .filter(|&(_, costed)| !costed) + .map(|(candidate, _)| PrunedCandidate { + group: physical.physical_candidates[candidate.0] + .logical_candidate_id + .group, + candidate, + reason: PruneReason::UsageNotCostable, + }) + .collect() +} + +/// The runtime's fee per query request under cost model 1, per second, as +/// `asap_cost::plan_cost` charges it: `ns_per_evaluation` of the measured +/// table, once per request of each query. +fn request_cost_per_second( + entries: &[QueryWorkloadEntry], + cost_inputs: &CostInputs, +) -> Result { + let evaluation_ms = cost_inputs.measured_operation_costs.ns_per_evaluation * 1e-6; + let CostModel::UsageOverTime(unit_cost) = &cost_inputs.cost_model else { + return Ok(0.0); + }; + if evaluation_ms <= 0.0 { + return Ok(0.0); + } + let mut cost = 0.0; + for entry in entries { + let requests_per_second = evaluation_rate( + [(&entry.recurrence, None)], + cost_inputs.non_repeating_query_horizon_seconds, + )?; + cost += unit_cost.cost_per_cpu_op * evaluation_ms + / cost_inputs.resource_speed.ms_per_cpu_op + * requests_per_second; + } + Ok(cost) +} + +/// The terms of `plan_cost` the objective does not encode that the +/// selected executions may meet, each with how many of them: query-time +/// results holding memory (when memory has a cost), not-materialized executions in a context of +/// several queries, query-time materialized chains, chains materialized +/// across evaluations read through the realizer's merge, consumers costed +/// over another producer than the selected one, ranges read only by +/// chains, query-time scans. +fn inexact_terms( + physical: &PhysicalASAPMemoCandidates, + contexts: &ExecutionContexts, + costs: &ExecutionCosts, + cost_inputs: &CostInputs, + solved: &SolvedSelection, +) -> BTreeMap { + let memo = &physical.logical_candidates.memo; + let memory_costs = matches!( + &cost_inputs.cost_model, + CostModel::UsageOverTime(unit_cost) if unit_cost.cost_per_memory_byte_second > 0.0 + ); + let execution_of_node = &solved.execution_of_node; + let windowed = |node: usize| { + memo.candidate( + physical.physical_candidates[execution_of_node[node].candidate().0] + .logical_candidate_id, + ) + .logical_properties + .window_layout() + .is_some() + }; + let mut terms: BTreeMap = BTreeMap::new(); + // Per node, whether every consumer selected for it is a chain. + let mut read_only_by_chains: BTreeMap = BTreeMap::new(); + for edge in &solved.selected_dag.edges { + let (producer, consumer) = (edge.producer.0, edge.consumer.0); + *read_only_by_chains.entry(producer).or_insert(true) &= windowed(consumer); + let representative = costs + .representative_producers + .get(&execution_of_node[consumer]) + .and_then(|producers| producers.get(edge.input_index_of_consumer)); + // Another run of the same logical candidate at the same timing has + // the same output. + let same_output = |a: Execution, b: Execution| { + let (a, b) = ( + &physical.physical_candidates[a.candidate().0], + &physical.physical_candidates[b.candidate().0], + ); + a.logical_candidate_id == b.logical_candidate_id + && a.physical_properties.timing() == b.physical_properties.timing() + }; + if !representative.is_some_and(|&r| same_output(r, execution_of_node[producer])) { + *terms + .entry(REPRESENTATIVE_PRODUCER.to_string()) + .or_default() += 1; + } + } + for (node, only_chains) in read_only_by_chains { + let candidate = &physical.physical_candidates[execution_of_node[node].candidate().0]; + if only_chains + && matches!( + memo.candidate(candidate.logical_candidate_id) + .expression + .operator, + Operator::NonASAP(NonASAPOp::TimeRange { .. }) + ) + && matches!( + execution_of_node[node], + Execution::InContext(context, _) + if matches!(contexts.contexts()[context.0], ExecutionContext::Evaluation { .. }) + ) + { + *terms.entry(RANGE_UNDER_CHAIN.to_string()).or_default() += 1; + } + } + let queries_of_context = |context: ExecutionContextId| { + (0..physical.query_roots.len()) + .filter(|&query| contexts.context_of_query(query) == Some(context)) + .count() + }; + for &execution in execution_of_node { + let candidate = &physical.physical_candidates[execution.candidate().0]; + let properties = candidate.physical_properties; + let logical = memo.candidate(candidate.logical_candidate_id); + let query_time = properties.timing() == ExecutionTiming::QueryTime; + let mut add = |term: &str| *terms.entry(term.to_string()).or_default() += 1; + if memory_costs + && query_time + && matches!( + properties.lifetime(), + Lifetime::None | Lifetime::OneEvaluation(StorageMedium::Memory) + ) + { + add(HELD_MEMORY); + } + if let Execution::InContext(context, _) = execution { + if properties.lifetime() == Lifetime::None && queries_of_context(context) > 1 { + add(RUNS_PER_QUERY); + } + } + if query_time && matches!(properties.lifetime(), Lifetime::AcrossEvaluations(_)) { + add(CHAIN_BUILDS_PER_REQUEST); + } + if matches!(properties.lifetime(), Lifetime::AcrossEvaluations(_)) { + add(CHAIN_MERGE); + } + if query_time + && matches!( + logical.expression.operator, + Operator::NonASAP(NonASAPOp::Scan { .. }) + ) + { + add(QUERY_TIME_SCAN); + } + } + terms +} + /// Per query, the accuracy accepted for the selected plan. An output that /// reads no sketch is exact. Otherwise, until constraint (K) (#1207 PR 8) /// checks the composed bound at the root, the output is accepted in its diff --git a/crates/types/src/stages/physical/memo_candidates.rs b/crates/types/src/stages/physical/memo_candidates.rs index 047ed068c..4dd3a5da3 100644 --- a/crates/types/src/stages/physical/memo_candidates.rs +++ b/crates/types/src/stages/physical/memo_candidates.rs @@ -44,6 +44,13 @@ impl PhysicalCandidate { /// context ($x_{\kappa,v}$, Stage 3), so its result belongs to that /// context. A non-windowed `AcrossEvaluations` producer is not offered /// yet (it needs its own delivery rule), so this returns false for it. + /// So does a windowed one read inside an ingestion pipeline or another + /// `AcrossEvaluations` candidate's computation: when those run is not + /// known from the memo (a refresh window is an index into Stage 3's + /// list; a candidate's tumbling width is in its own window layout), so, + /// as for an unknown recurrence, its windows are never known to be + /// complete. Stage 2 offers `AcrossEvaluations` only to tumbling-window + /// builds, whose consumers read them at query time. pub fn delivers_on(&self, memo: &Memo, context: &ExecutionContext) -> bool { match self.physical_properties.lifetime() { Lifetime::None | Lifetime::OneEvaluation(_) => true, @@ -67,7 +74,7 @@ impl PhysicalCandidate { .evaluation_grid_ms(*range_evaluation) .is_some_and(|grid_ms| grid_ms.is_multiple_of(*width_ms)), ExecutionContext::IngestionPipeline(_) | ExecutionContext::Materialized(_) => { - todo!("#1207 PR 7: delivery into an ingestion or materialized context") + false } } } @@ -435,6 +442,24 @@ mod tests { true, ), (one_evaluation, evaluation(estimated.clone(), None), true), + // Inside an ingestion pipeline or another candidate's + // computation the times are not in the memo: never, for a + // windowed producer; always for one that runs there. + ( + ingestion_10s, + ExecutionContext::IngestionPipeline(crate::stages::RefreshWindowId(0)), + false, + ), + ( + across_5s, + ExecutionContext::Materialized(PhysicalCandidateId(0)), + false, + ), + ( + unmaterialized, + ExecutionContext::Materialized(PhysicalCandidateId(0)), + true, + ), ] { assert_eq!( producer.delivers_on(&memo, &context), diff --git a/docs/design_docs/proposals/planner-layering.md b/docs/design_docs/proposals/planner-layering.md index b80798ae5..5c4a55a89 100644 --- a/docs/design_docs/proposals/planner-layering.md +++ b/docs/design_docs/proposals/planner-layering.md @@ -503,15 +503,25 @@ then the Pass 2 transformations), and expand shared sizes, tumbling-window horiz (planned) raw windows into concrete groups, so that choosing a threshold is choosing a group; choose one candidate per needed group; compose the choice into one plan and check and cost it with Stage 2 and Stage 3. The search is -the group selection MILP (mixed-integer linear program, #1095): it chooses one -candidate per memo group and execution context, with each shareable candidate -charged once, and is solved once; it prices cost model 1 only (cost models 2 -and 3 are an `UnsupportedCostModel` error, #888). Stage 3 realizes the chosen -plan through Stage 2's `AssignmentRealizer`, which the facade builds and passes in, -so Stage 3 does not depend on the Stage 2 crate. The facade runs Pass 2, -Stage 2 and Stage 3 once, over the full tumbling-window-width domain, with no -time limit. Stage 3 checks that `plan_cost` of the realized plan -equals the MILP's objective, and that every query's accuracy and latency hold. +the selection MILP (mixed-integer linear program, #1204, concepts doc +sections 3 and 5) over execution contexts: an `AcrossEvaluations` candidate +runs once ($x^{\mathrm{mat}}_v$), a candidate with lifetime None or +`OneEvaluation` once per context that selects it ($x_{\kappa,v}$; a +context is the evaluation shared by queries with equal recurrence and range +evaluation, an ingestion pipeline's refresh window, or a materialized +candidate's computation), every edge per context of its consumer +($y_{\kappa,u,c,i}$, only where `delivers_on` holds), and a tumbling-window +chain retains the windows its most demanding selected consumer reads ($h_v$, +constraint (R)). It is solved once and costs under cost model 1 only (cost +models 2 and 3 are an `UnsupportedCostModel` error, #888). Stage 3 realizes +the chosen plan through Stage 2's `AssignmentRealizer`, which the facade +builds and passes in, so Stage 3 does not depend on the Stage 2 crate. The +facade runs Pass 2, Stage 2 and Stage 3 once, over the full +tumbling-window-width domain, with no time limit. Stage 3 compares +`plan_cost` of the realized plan with the MILP's objective and reports the +terms of `plan_cost` the objective does not encode +(`SearchReport::inexact_objective`; e.g. memory held for the latency of the +queries reading it, which is out of scope). #### Memo code interface @@ -519,7 +529,7 @@ The memo rules are in `crates/stage1-pass2-reuse-candidates/src/` (module `memo`, crate-private: the crate root re-exports each entry point); the memo types Stage 1 hands Stage 2 are in `crates/types/src/stages/` (`asap_types::stages`); the facade that runs the stages is in `crates/planner/src/run.rs`; the search over Stage 2's -candidates is in `crates/stage3-plan-selection/src/` (`select.rs`, `groups/`, `milp/`). +candidates is in `crates/stage3-plan-selection/src/` (`select.rs`, `execution_cost.rs`, `milp/`). | Step | Entry point | Path | |---|---|---| @@ -528,7 +538,7 @@ candidates is in `crates/stage3-plan-selection/src/` (`select.rs`, `groups/`, `m | Memo types | `DagOfLogicalMemoGroups`, `PerComputationLogicalMemoCandidates`, `HowToComputeAggregateMeasure`, Pass 1's working types (`ComputationTarget`, `FusedComputation`, `HavingEntry`, `HavingPlacement`), `Memo`, `CandidateForMemoGroup`, `OriginOfCandidateForMemoGroup`, `LogicalASAPMemoCandidates`, `CandidateGenerationTimeout`, `root_groups`, `queries_rooted_at` | `crates/types/src/stages/` (`asap_types::stages`) | | Pass 2 | `generate_logical_asap_memo_candidates` → `LogicalASAPMemoCandidates`, from Pass 1's output. It calls in this order: 1. window composition for every legal tumbling-window width (`add_local_window_composition`); 2–9. the rules in `pass2/memo/rules/`: scan widening, summary inputs, top-k reuse, finer aggregate, rollup, window rollup, summary results, state configurations; 10. `add_shared_summary_state_groups`, the shared builds | `pass2/logical_memo_candidates.rs` (`LogicalASAPMemoCandidates` itself: `asap_types::stages`); `pass2/memo/window_candidates.rs`, `pass2/memo/rules/`, `pass2/memo/state_groups.rs`, `pass2/memo/shared_configurations.rs` | | Memo to candidate DAGs | `enumerate`, `build_logical_asap_dags_from_chosen_candidates`, `ChosenCandidatePerMemoGroup`, `AllChosenCandidatesPerMemoGroup`, `ErrorBuildingLogicalASAPDagsFromChosenCandidates` | `pass2/memo/enumeration.rs` (re-exported at the crate root) | -| Production memo build and search | The facade's `run_planning` builds the DAG of logical memo groups and runs Pass 1 (`pass1_memo`) once; then `generate_logical_asap_memo_candidates`, `generate_physical_asap_memo_candidates` and Stage 3's `select_plan`, once each, and builds the selected plan with Stage 2's `AssignmentRealizer::realize`. Behind `select_plan`: the group selection MILP (#1095), `groups/` and `milp/` | `crates/planner/src/run.rs`; `crates/stage3-plan-selection/src/select.rs` | +| Production memo build and search | The facade's `run_planning` builds the DAG of logical memo groups and runs Pass 1 (`pass1_memo`) once; then `generate_logical_asap_memo_candidates`, `generate_physical_asap_memo_candidates` and Stage 3's `select_plan`, once each, and builds the selected plan with Stage 2's `AssignmentRealizer::realize`. Behind `select_plan`: the selection MILP over execution contexts (#1204), `execution_cost.rs` and `milp/` | `crates/planner/src/run.rs`; `crates/stage3-plan-selection/src/select.rs` | ## ASAPPlanner Detailed Design: Planning Stages and their decisions/Strategies @@ -1299,10 +1309,9 @@ input no buildable producer can supply over admissible edges (`InputHasNoBuildableProducer`) and those no query output reads (`ReachesNoQueryOutput`). Stage 2 makes no execution-scope copies: how many times a candidate runs is Stage 3's choice, and a -selection names each run as a node of its `DagOfSelectedCandidates`. -Today's Stage 3 engine, whose MILP runs each candidate at most once, makes -the copies it needs itself before solving and decodes each selected copy -into a node. +selection names each run as a node of its `DagOfSelectedCandidates`: +Stage 3 runs a candidate with lifetime None or `OneEvaluation` once per +execution context that selects it, and decodes each run into a node. *Example for Algorithm 3* ([Example 4, Pattern B](#example-4-materialization-of-window-summaries-in-physical-planning)). The logical candidate is the `Tumbling(1m)` KLL option from Algorithm 2. The