diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/logical_plan/mod.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/logical_plan/mod.rs index a79239d6fca75..0bb17589b39f0 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/logical_plan/mod.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/logical_plan/mod.rs @@ -26,6 +26,7 @@ mod query; mod query_source; mod root_query; mod schema; +pub mod transforms; pub mod visitor; pub use aggregate_multiplied_subquery::*; diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/logical_plan/multistage/leaf_measure.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/logical_plan/multistage/leaf_measure.rs index 70f36cfa9fa8c..1aaa53b0b71ff 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/logical_plan/multistage/leaf_measure.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/logical_plan/multistage/leaf_measure.rs @@ -5,8 +5,9 @@ use cubenativeutils::CubeError; use std::rc::Rc; /// Leaf CTE of a multi-stage chain — a base query that produces the -/// raw aggregated values feeding the rest of the chain. Optional -/// state rendering and time shifts come from `evaluation_context`. +/// raw aggregated values feeding the rest of the chain. The measures +/// carry their own render form; time shifts and row-grain evaluation +/// come from `evaluation_context`. pub struct MultiStageLeafMeasure { pub measures: Vec>, pub evaluation_context: EvaluationContext, @@ -20,9 +21,6 @@ impl PrettyPrint for MultiStageLeafMeasure { for measure in self.measures.iter() { result.println(&format!("measure: {}", measure.full_name()), &state); } - if self.evaluation_context.measure_as_state { - result.println("render_measure_as_state: true", &state); - } if self.evaluation_context.measure_for_ungrouped { result.println("render_measure_for_ungrouped: true", &state); } diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/logical_plan/transforms/mod.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/logical_plan/transforms/mod.rs new file mode 100644 index 0000000000000..cd8e1a98b91e3 --- /dev/null +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/logical_plan/transforms/mod.rs @@ -0,0 +1,14 @@ +//! Transformations that derive new logical-plan pieces (schemas, +//! filters) from existing ones by lifting symbol-level transforms +//! over their members. +//! +//! The level rule: a transform lives at the level of what it takes as +//! input. Symbol-to-symbol transforms belong to +//! `planner::symbols::transforms`; anything that rewrites a plan-level +//! container of symbols belongs here. + +mod render_modifier; +mod tz_converted_at_source; + +pub use render_modifier::*; +pub use tz_converted_at_source::*; diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/logical_plan/transforms/render_modifier.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/logical_plan/transforms/render_modifier.rs new file mode 100644 index 0000000000000..928a2e0ef7b7c --- /dev/null +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/logical_plan/transforms/render_modifier.rs @@ -0,0 +1,27 @@ +use super::super::LogicalSchema; +use crate::planner::symbols::transforms; +use crate::planner::{MeasureRenderModifier, MemberSymbol}; +use cubenativeutils::CubeError; +use std::rc::Rc; + +/// Copy of the schema with `modifier` set on every measure that has +/// no render modifier yet. Every schema member is rewritten, so +/// measures embedded in a dimension's expression tree get the form +/// too — the form belongs to the measure as rendered in this select, +/// not to its position in the schema. +pub fn measures_render_modifier_in_schema( + schema: &LogicalSchema, + modifier: &MeasureRenderModifier, +) -> Result, CubeError> { + let stamp = |members: &Vec>| { + members + .iter() + .map(|m| transforms::measures_render_modifier(m, modifier)) + .collect::, _>>() + }; + let mut new = schema.clone(); + new.time_dimensions = stamp(&new.time_dimensions)?; + new.dimensions = stamp(&new.dimensions)?; + new.measures = stamp(&new.measures)?; + Ok(Rc::new(new)) +} diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/logical_plan/transforms/tz_converted_at_source.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/logical_plan/transforms/tz_converted_at_source.rs new file mode 100644 index 0000000000000..97c0c73bd4e50 --- /dev/null +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/logical_plan/transforms/tz_converted_at_source.rs @@ -0,0 +1,32 @@ +use super::super::LogicalSchema; +use crate::planner::symbols::transforms; +use crate::planner::MemberSymbol; +use cubenativeutils::CubeError; +use std::collections::HashSet; +use std::rc::Rc; + +/// Copy of the schema with every time dimension marked as +/// timezone-converted at the source (a pre-aggregation rollup or an +/// input CTE). Every schema member is rewritten, so occurrences of +/// the marked time dimensions embedded in other members' expression +/// trees carry the mark too. +pub fn mark_tz_converted_at_source_in_schema( + schema: &LogicalSchema, +) -> Result, CubeError> { + let names = schema + .time_dimensions + .iter() + .map(|d| d.full_name()) + .collect::>(); + let mark = |members: &Vec>| { + members + .iter() + .map(|m| transforms::mark_tz_converted_at_source(m, &names)) + .collect::, _>>() + }; + let mut new = schema.clone(); + new.time_dimensions = mark(&new.time_dimensions)?; + new.dimensions = mark(&new.dimensions)?; + new.measures = mark(&new.measures)?; + Ok(Rc::new(new)) +} diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan/sql_nodes/cube_calc_groups.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan/sql_nodes/cube_calc_groups.rs deleted file mode 100644 index 27b5bfe4d55c4..0000000000000 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan/sql_nodes/cube_calc_groups.rs +++ /dev/null @@ -1,114 +0,0 @@ -use super::SqlNode; -use crate::planner::query_tools::QueryTools; -use crate::planner::MemberSymbol; -use crate::physical_plan::SqlEvaluatorVisitor; -use crate::planner::sql_templates::structs::{TemplateCalcGroup, TemplateCalcSingleValue}; -use crate::planner::sql_templates::PlanSqlTemplates; -use cubenativeutils::CubeError; -use std::any::Any; -use std::collections::HashMap; -use std::rc::Rc; - -#[derive(Clone, Debug)] -pub struct CalcGroupItem { - pub name: String, - pub values: Vec, -} - -#[derive(Default, Clone, Debug)] -pub struct CalcGroupsItems { - items: HashMap>, -} - -impl CalcGroupsItems { - pub fn add(&mut self, cube_name: String, dimension_name: String, values: Vec) { - let items = self.items.entry(cube_name).or_default(); - if !items.iter().any(|itm| itm.name == dimension_name) { - items.push(CalcGroupItem { - name: dimension_name, - values, - }) - } - } - - pub fn get(&self, cube_name: &str) -> Option<&Vec> { - self.items.get(cube_name) - } - - pub fn is_empty(&self) -> bool { - self.items.is_empty() - } -} - -pub struct CubeCalcGroupsSqlNode { - input: Rc, - items: CalcGroupsItems, -} - -impl CubeCalcGroupsSqlNode { - pub fn new(input: Rc, items: CalcGroupsItems) -> Rc { - Rc::new(Self { input, items }) - } -} - -impl SqlNode for CubeCalcGroupsSqlNode { - fn to_sql( - &self, - visitor: &SqlEvaluatorVisitor, - node: &Rc, - query_tools: Rc, - node_processor: Rc, - templates: &PlanSqlTemplates, - ) -> Result { - let input = self.input.to_sql( - visitor, - node, - query_tools.clone(), - node_processor.clone(), - templates, - )?; - let res = match node.as_ref() { - MemberSymbol::CubeTable(ev) => { - let res = if let Some(calc_groups) = self.items.get(ev.cube_name()) { - let mut single_values = vec![]; - let mut template_groups = vec![]; - for calc_group in calc_groups { - if calc_group.values.len() == 1 { - single_values.push(TemplateCalcSingleValue { - name: calc_group.name.clone(), - value: calc_group.values[0].clone(), - }) - } else { - template_groups.push(TemplateCalcGroup { - name: calc_group.name.clone(), - alias: format!("{}_values", calc_group.name), - values: calc_group.values.clone(), - }) - } - } - let res = templates.calc_groups_join( - &ev.cube_name(), - &input, - single_values, - template_groups, - )?; - format!("({})", res) - } else { - input - }; - - res - } - _ => input, - }; - Ok(res) - } - - fn as_any(self: Rc) -> Rc { - self.clone() - } - - fn childs(&self) -> Vec> { - vec![] - } -} diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan/sql_nodes/factory.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan/sql_nodes/factory.rs index 1f6b80a243645..e6a28fe17e3f6 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan/sql_nodes/factory.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan/sql_nodes/factory.rs @@ -1,9 +1,9 @@ use super::{ AutoPrefixSqlNode, CaseSqlNode, EvaluateSqlNode, FinalMeasureSqlNode, FinalPreAggregationMeasureSqlNode, GeoDimensionSqlNode, MaskedSqlNode, MeasureFilterSqlNode, - MultiStageRankNode, MultiStageWindowNode, ParenthesizeSqlNode, RenderReferencesSqlNode, - RenderReferencesType, RollingWindowNode, RootSqlNode, SegmentDimensionSqlNode, SqlNode, - TimeDimensionNode, TimeShiftSqlNode, UngroupedMeasureSqlNode, + MeasureRenderModifierSqlNode, MultiStageRankNode, MultiStageWindowNode, ParenthesizeSqlNode, + RenderReferencesSqlNode, RenderReferencesType, RollingWindowNode, RootSqlNode, + SegmentDimensionSqlNode, SqlNode, TimeDimensionNode, TimeShiftSqlNode, UngroupedMeasureSqlNode, UngroupedQueryFinalMeasureSqlNode, }; use crate::physical_plan::cube_ref_evaluator::CubeRefEvaluator; @@ -15,26 +15,19 @@ use crate::planner::symbols::CalendarDimensionTimeShift; use std::collections::{HashMap, HashSet}; use std::rc::Rc; -/// Builds the SQL-node chain for a query. Carries all the flags and -/// reference maps the query needs (time shifts, render references, -/// pre-aggregation refs, multi-stage partitions, etc.) and assembles -/// them into a layered `SqlNode` via `default_node_processor`. +/// Builds the SQL-node chain for a query. Carries the reference maps +/// the query needs (time shifts, render references, pre-aggregation +/// refs, cube aliases) and assembles them into a layered `SqlNode` +/// via `default_node_processor`. #[derive(Clone, Default)] pub struct SqlNodesFactory { time_shifts: TimeShiftState, calendar_time_shifts: HashMap, - ungrouped: bool, - ungrouped_measure: bool, - count_approx_as_state: bool, render_references: RenderReferences, pre_aggregation_dimensions_references: RenderReferences, pre_aggregation_measures_references: RenderReferences, ungrouped_measure_references: RenderReferences, cube_name_references: HashMap, - multi_stage_rank: Option>, //partition_by - multi_stage_window: Option>, //partition_by - rolling_window: bool, - dimensions_with_ignored_timezone: HashSet, use_local_tz_in_date_range: bool, original_sql_pre_aggregations: HashMap, // Full names of the members present in the query GROUP BY. Used by @@ -63,10 +56,6 @@ impl SqlNodesFactory { self.calendar_time_shifts = calendar_time_shifts; } - pub fn set_ungrouped(&mut self, value: bool) { - self.ungrouped = value; - } - pub fn set_use_local_tz_in_date_range(&mut self, value: bool) { self.use_local_tz_in_date_range = value; } @@ -83,10 +72,6 @@ impl SqlNodesFactory { !self.pre_aggregation_dimensions_references.is_empty() } - pub fn set_ungrouped_measure(&mut self, value: bool) { - self.ungrouped_measure = value; - } - pub fn add_render_reference>(&mut self, name: String, value: T) { self.render_references.insert(name, value); } @@ -116,18 +101,6 @@ impl SqlNodesFactory { self.original_sql_pre_aggregations = value; } - pub fn add_dimensions_with_ignored_timezone(&mut self, value: String) { - self.dimensions_with_ignored_timezone.insert(value); - } - - pub fn set_multi_stage_rank(&mut self, partition_by: Vec) { - self.multi_stage_rank = Some(partition_by); - } - - pub fn set_multi_stage_window(&mut self, partition_by: Vec) { - self.multi_stage_window = Some(partition_by); - } - pub fn add_pre_aggregation_measure_reference>( &mut self, name: String, @@ -136,14 +109,6 @@ impl SqlNodesFactory { self.pre_aggregation_measures_references.insert(name, value); } - pub fn set_rolling_window(&mut self, value: bool) { - self.rolling_window = value; - } - - pub fn set_count_approx_as_state(&mut self, value: bool) { - self.count_approx_as_state = value; - } - pub fn add_ungrouped_measure_reference>( &mut self, name: String, @@ -171,10 +136,11 @@ impl SqlNodesFactory { /// Three sub-chains hang off a `RootSqlNode` keyed by member /// kind: a dimension chain (geo / case / time-shift / calendar /// time-shift wraps), a time-dimension chain, and a measure - /// chain (case → measure filter → final-measure / ungrouped / - /// multi-stage wraps → mask). The whole tree is then wrapped in - /// a top-level `RenderReferencesSqlNode` for query-wide reference - /// substitution. + /// chain (case → measure filter → render-modifier dispatch over + /// the final-measure / rolling-merge / ungrouped chains → mask → + /// multi-stage window and rank wraps). The whole tree is then + /// wrapped in a top-level `RenderReferencesSqlNode` for + /// query-wide reference substitution. pub fn default_node_processor(&self, query_tools: &QueryTools) -> Rc { // Build an "unmasked" copy of the tree (masking disabled, but still // dispatching by member kind) only when the query has masked members. It @@ -199,6 +165,7 @@ impl SqlNodesFactory { ) -> Rc { let evaluate_sql_processor = MaskedSqlNode::new( EvaluateSqlNode::new(), + false, self.group_by_members.clone(), skip_masking, unmasked_root.clone(), @@ -217,24 +184,16 @@ impl SqlNodesFactory { let measure_processor = self.final_measure_node_processor(measure_processor); // Wrap the entire measure chain with MaskedSqlNode so masked measures // are intercepted before aggregation/ungrouped wrapping. - let measure_processor = if self.ungrouped || self.ungrouped_measure { - MaskedSqlNode::new_ungrouped( - measure_processor, - self.group_by_members.clone(), - skip_masking, - unmasked_root.clone(), - ) - } else { - MaskedSqlNode::new( - measure_processor, - self.group_by_members.clone(), - skip_masking, - unmasked_root.clone(), - ) - }; - let measure_processor = self - .add_multi_stage_window_if_needed(measure_processor, measure_filter_processor.clone()); - let measure_processor = self.add_multi_stage_rank_if_needed(measure_processor); + let measure_processor = MaskedSqlNode::new( + measure_processor, + true, + self.group_by_members.clone(), + skip_masking, + unmasked_root.clone(), + ); + let measure_processor: Rc = + MultiStageWindowNode::new(measure_filter_processor.clone(), measure_processor); + let measure_processor: Rc = MultiStageRankNode::new(measure_processor); let default_processor: Rc = if !self.pre_aggregation_dimensions_references.is_empty() { @@ -285,51 +244,24 @@ impl SqlNodesFactory { } } - fn add_multi_stage_rank_if_needed(&self, default: Rc) -> Rc { - if let Some(partition_by) = &self.multi_stage_rank { - MultiStageRankNode::new(default, partition_by.clone()) - } else { - default - } - } - - fn add_multi_stage_window_if_needed( - &self, - default: Rc, - multi_stage_input: Rc, - ) -> Rc { - if let Some(partition_by) = &self.multi_stage_window { - MultiStageWindowNode::new(multi_stage_input, default, partition_by.clone()) - } else { - default - } - } - fn final_measure_node_processor(&self, input: Rc) -> Rc { - if self.ungrouped_measure { - self.wrap_ungrouped_pre_aggregation_measure(UngroupedMeasureSqlNode::new(input)) - } else if self.ungrouped { - self.wrap_ungrouped_pre_aggregation_measure(UngroupedQueryFinalMeasureSqlNode::new( - input, - )) - } else { - let final_processor: Rc = - FinalMeasureSqlNode::new(input.clone(), self.count_approx_as_state); - let final_processor = if !self.pre_aggregation_measures_references.is_empty() { + let aggregated: Rc = { + let final_processor: Rc = FinalMeasureSqlNode::new(input.clone()); + if !self.pre_aggregation_measures_references.is_empty() { FinalPreAggregationMeasureSqlNode::new( final_processor, self.pre_aggregation_measures_references.clone(), - self.count_approx_as_state, ) } else { final_processor - }; - if self.rolling_window { - RollingWindowNode::new(input, final_processor) - } else { - final_processor } - } + }; + let rolling_merge = RollingWindowNode::new(input.clone(), aggregated.clone()); + let raw_value = self + .wrap_ungrouped_pre_aggregation_measure(UngroupedMeasureSqlNode::new(input.clone())); + let ungrouped_final = self + .wrap_ungrouped_pre_aggregation_measure(UngroupedQueryFinalMeasureSqlNode::new(input)); + MeasureRenderModifierSqlNode::new(aggregated, rolling_merge, raw_value, ungrouped_final) } fn dimension_processor(&self, input: Rc) -> Rc { @@ -346,8 +278,7 @@ impl SqlNodesFactory { let input: Rc = ParenthesizeSqlNode::new(input); - let input: Rc = - TimeDimensionNode::new(self.dimensions_with_ignored_timezone.clone(), input); + let input: Rc = TimeDimensionNode::new(input); let input = if !self.calendar_time_shifts.is_empty() { CalendarTimeShiftSqlNode::new(self.calendar_time_shifts.clone(), input) @@ -365,8 +296,7 @@ impl SqlNodesFactory { } fn time_dimension_processor(&self, input: Rc) -> Rc { - let input: Rc = - TimeDimensionNode::new(self.dimensions_with_ignored_timezone.clone(), input); + let input: Rc = TimeDimensionNode::new(input); input } diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan/sql_nodes/final_measure.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan/sql_nodes/final_measure.rs index 425ebb639b90f..71b33d203e0eb 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan/sql_nodes/final_measure.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan/sql_nodes/final_measure.rs @@ -10,18 +10,13 @@ use std::rc::Rc; /// Applies the final aggregation wrap to a measure (sum / avg / /// count_distinct / pass-through, etc.) using `MeasureKind::aggregate_wrap`. -/// Routes `count_distinct_approx` through an HLL state when requested. pub struct FinalMeasureSqlNode { input: Rc, - count_approx_as_state: bool, } impl FinalMeasureSqlNode { - pub fn new(input: Rc, count_approx_as_state: bool) -> Rc { - Rc::new(Self { - input, - count_approx_as_state, - }) + pub fn new(input: Rc) -> Rc { + Rc::new(Self { input }) } pub fn input(&self) -> &Rc { @@ -38,13 +33,8 @@ impl FinalMeasureSqlNode { AggregateWrap::PassThrough => Ok(input), AggregateWrap::Function(name) => Ok(format!("{}({})", name, input)), AggregateWrap::CountDistinct => templates.count_distinct(&input), - AggregateWrap::CountDistinctApprox => { - if self.count_approx_as_state { - templates.hll_init(input) - } else { - templates.count_distinct_approx(input) - } - } + AggregateWrap::CountDistinctApprox => templates.count_distinct_approx(input), + AggregateWrap::CountDistinctApproxState => templates.hll_init(input), } } } @@ -63,7 +53,12 @@ impl SqlNode for FinalMeasureSqlNode { let wrap = ev.kind().aggregate_wrap(); let child_visitor = match wrap { AggregateWrap::PassThrough => visitor.clone(), - _ => visitor.with_arg_needs_paren_safe(false), + AggregateWrap::Function(_) + | AggregateWrap::CountDistinct + | AggregateWrap::CountDistinctApprox + | AggregateWrap::CountDistinctApproxState => { + visitor.with_arg_needs_paren_safe(false) + } }; let input = self.input.to_sql( &child_visitor, diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan/sql_nodes/final_pre_aggregation_measure.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan/sql_nodes/final_pre_aggregation_measure.rs index 100557d071b37..d92b39f5d657c 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan/sql_nodes/final_pre_aggregation_measure.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan/sql_nodes/final_pre_aggregation_measure.rs @@ -17,20 +17,11 @@ use std::rc::Rc; pub struct FinalPreAggregationMeasureSqlNode { input: Rc, references: RenderReferences, - count_approx_as_state: bool, } impl FinalPreAggregationMeasureSqlNode { - pub fn new( - input: Rc, - references: RenderReferences, - count_approx_as_state: bool, - ) -> Rc { - Rc::new(Self { - input, - references, - count_approx_as_state, - }) + pub fn new(input: Rc, references: RenderReferences) -> Rc { + Rc::new(Self { input, references }) } pub fn input(&self) -> &Rc { @@ -63,22 +54,23 @@ impl SqlNode for FinalPreAggregationMeasureSqlNode { templates.quote_identifier(&column_name.name())? ); match ev.kind().pre_aggregate_wrap() { + // The rollup column holds an HLL state, so it + // must be merged, not recomputed. Keep the + // merged state when this query itself feeds a + // further aggregation; otherwise take its + // cardinality. + AggregateWrap::CountDistinctApproxState => { + templates.hll_merge(pre_aggregation_measure)? + } AggregateWrap::CountDistinctApprox => { - // The rollup column holds an HLL state, so it - // must be merged, not recomputed. Keep the - // merged state when this query itself feeds a - // further aggregation; otherwise take its - // cardinality. - if self.count_approx_as_state { - templates.hll_merge(pre_aggregation_measure)? - } else { - templates.hll_cardinality_merge(pre_aggregation_measure)? - } + templates.hll_cardinality_merge(pre_aggregation_measure)? } AggregateWrap::Function(name) => { format!("{}({})", name, pre_aggregation_measure) } - _ => format!("sum({})", pre_aggregation_measure), + AggregateWrap::PassThrough | AggregateWrap::CountDistinct => { + format!("sum({})", pre_aggregation_measure) + } } } RenderReferencesType::LiteralValue(value) => { diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan/sql_nodes/masked.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan/sql_nodes/masked.rs index be3b2ae7a117e..0d3b075db903d 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan/sql_nodes/masked.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan/sql_nodes/masked.rs @@ -4,7 +4,7 @@ use crate::physical_plan::SqlEvaluatorVisitor; use crate::planner::query_tools::QueryTools; use crate::planner::sql_templates::PlanSqlTemplates; use crate::planner::FiltersContext; -use crate::planner::MemberSymbol; +use crate::planner::{MeasureRenderModifier, MemberSymbol}; use cubenativeutils::CubeError; use std::any::Any; use std::collections::HashSet; @@ -16,7 +16,12 @@ use std::rc::Rc; /// original ELSE mask END`. Pass-through for non-masked members. pub struct MaskedSqlNode { input: Rc, - ungrouped: bool, + // Only the node wrapping the final measure chain applies row-level + // mask semantics for measures with a render modifier; the node at + // the evaluate position always masks with grouped semantics, so a + // row-level measure whose mask has dependencies — deferred by the + // final-chain node — is still masked when its raw SQL is rendered. + row_level_semantics: bool, // Full names of the members present in the query GROUP BY. Used to decide // whether conditional masking can be applied to an aggregate measure. group_by_members: HashSet, @@ -36,28 +41,14 @@ pub struct MaskedSqlNode { impl MaskedSqlNode { pub fn new( input: Rc, + row_level_semantics: bool, group_by_members: HashSet, skip_masking: bool, unmasked_root: Option>, ) -> Rc { Rc::new(Self { input, - ungrouped: false, - group_by_members, - skip_masking, - unmasked_root, - }) - } - - pub fn new_ungrouped( - input: Rc, - group_by_members: HashSet, - skip_masking: bool, - unmasked_root: Option>, - ) -> Rc { - Rc::new(Self { - input, - ungrouped: true, + row_level_semantics, group_by_members, skip_masking, unmasked_root, @@ -79,8 +70,19 @@ impl MaskedSqlNode { let mask_filter = query_tools.member_mask_filter(&full_name); + // A measure with an ungrouped render modifier is emitted at row + // grain, which changes both mask decisions below. + let ungrouped = self.row_level_semantics + && match node.as_ref() { + MemberSymbol::Measure(m) => matches!( + m.render_modifier(), + Some(MeasureRenderModifier::RawValue | MeasureRenderModifier::UngroupedFinal) + ), + _ => false, + }; + let masked_sql = if let Some(mask_call) = node.mask_sql() { - if self.ungrouped { + if ungrouped { if let MemberSymbol::Measure(_) = node.as_ref() { if mask_call.dependencies_count() > 0 { return Ok(None); @@ -109,7 +111,7 @@ impl MaskedSqlNode { // WHERE clause, so we render the mask value directly for such measures. In // ungrouped queries the measure is rendered at row grain, so the CASE WHEN // is valid and is kept. - if !self.ungrouped { + if !ungrouped { if let MemberSymbol::Measure(_) = node.as_ref() { let filter_members = filter_item.all_member_evaluators(); let all_in_group_by = !filter_members.is_empty() diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan/sql_nodes/measure_render_modifier.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan/sql_nodes/measure_render_modifier.rs new file mode 100644 index 0000000000000..b9bbb382b3704 --- /dev/null +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan/sql_nodes/measure_render_modifier.rs @@ -0,0 +1,85 @@ +use super::SqlNode; +use crate::physical_plan::SqlEvaluatorVisitor; +use crate::planner::query_tools::QueryTools; +use crate::planner::sql_templates::PlanSqlTemplates; +use crate::planner::{MeasureRenderModifier, MemberSymbol}; +use cubenativeutils::CubeError; +use std::any::Any; +use std::rc::Rc; + +/// Routes a measure to the chain matching its render modifier: the +/// final aggregation by default, the window-partial merge for +/// `RollingMerge`, or one of the row-level forms. Rank and window +/// measures are intercepted by their dedicated nodes higher in the +/// chain and must not reach this dispatcher. +pub struct MeasureRenderModifierSqlNode { + aggregated: Rc, + rolling_merge: Rc, + raw_value: Rc, + ungrouped_final: Rc, +} + +impl MeasureRenderModifierSqlNode { + pub fn new( + aggregated: Rc, + rolling_merge: Rc, + raw_value: Rc, + ungrouped_final: Rc, + ) -> Rc { + Rc::new(Self { + aggregated, + rolling_merge, + raw_value, + ungrouped_final, + }) + } +} + +impl SqlNode for MeasureRenderModifierSqlNode { + fn to_sql( + &self, + visitor: &SqlEvaluatorVisitor, + node: &Rc, + query_tools: Rc, + node_processor: Rc, + templates: &PlanSqlTemplates, + ) -> Result { + let chain = match node.as_ref() { + MemberSymbol::Measure(m) => match m.render_modifier() { + None => &self.aggregated, + Some(modifier @ MeasureRenderModifier::RollingMerge) => { + modifier.ensure_applies_to(m)?; + &self.rolling_merge + } + Some(MeasureRenderModifier::RawValue) => &self.raw_value, + Some(MeasureRenderModifier::UngroupedFinal) => &self.ungrouped_final, + Some(MeasureRenderModifier::MultiStageRank { .. }) + | Some(MeasureRenderModifier::MultiStageWindow { .. }) => { + return Err(CubeError::internal(format!( + "Multi-stage window measure {} reached the render-modifier dispatcher instead of its dedicated node", + m.full_name() + ))); + } + }, + _ => { + return Err(CubeError::internal(format!( + "Measure render modifier node processor called for wrong node", + ))); + } + }; + chain.to_sql(visitor, node, query_tools, node_processor, templates) + } + + fn as_any(self: Rc) -> Rc { + self.clone() + } + + fn childs(&self) -> Vec> { + vec![ + self.aggregated.clone(), + self.rolling_merge.clone(), + self.raw_value.clone(), + self.ungrouped_final.clone(), + ] + } +} diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan/sql_nodes/mod.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan/sql_nodes/mod.rs index 4620ad8c66ba5..1214a340ea5d3 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan/sql_nodes/mod.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan/sql_nodes/mod.rs @@ -9,7 +9,6 @@ pub mod auto_prefix; pub mod calendar_time_shift; pub mod case; -//pub mod cube_calc_groups; pub mod evaluate_sql; pub mod factory; pub mod final_measure; @@ -17,6 +16,7 @@ pub mod final_pre_aggregation_measure; pub mod geo_dimension; pub mod masked; pub mod measure_filter; +pub mod measure_render_modifier; pub mod multi_stage_rank; pub mod multi_stage_window; pub mod parenthesize; @@ -29,10 +29,10 @@ pub mod time_dimension; pub mod time_shift; pub mod ungroupped_measure; pub mod ungroupped_query_final_measure; +pub mod window_partition; pub use auto_prefix::AutoPrefixSqlNode; pub use case::CaseSqlNode; -//pub use cube_calc_groups::CubeCalcGroupsSqlNode; pub use evaluate_sql::EvaluateSqlNode; pub use factory::SqlNodesFactory; pub use final_measure::FinalMeasureSqlNode; @@ -40,6 +40,7 @@ pub use final_pre_aggregation_measure::FinalPreAggregationMeasureSqlNode; pub use geo_dimension::GeoDimensionSqlNode; pub use masked::MaskedSqlNode; pub use measure_filter::MeasureFilterSqlNode; +pub use measure_render_modifier::MeasureRenderModifierSqlNode; pub use multi_stage_rank::MultiStageRankNode; pub use multi_stage_window::MultiStageWindowNode; pub use parenthesize::ParenthesizeSqlNode; diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan/sql_nodes/multi_stage_rank.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan/sql_nodes/multi_stage_rank.rs index 46404fdd0f199..19b067d9c43ea 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan/sql_nodes/multi_stage_rank.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan/sql_nodes/multi_stage_rank.rs @@ -1,35 +1,28 @@ +use super::window_partition::render_partition_by; use super::SqlNode; use crate::physical_plan::SqlEvaluatorVisitor; use crate::planner::query_tools::QueryTools; use crate::planner::sql_templates::PlanSqlTemplates; -use crate::planner::symbols::MeasureKind; -use crate::planner::MemberSymbol; +use crate::planner::{MeasureRenderModifier, MemberSymbol}; use cubenativeutils::CubeError; use std::any::Any; use std::rc::Rc; -/// Renders a `Rank` measure as a SQL window function partitioned -/// by `partition`. Non-rank measures go through `else_processor`. +/// Renders a `Rank` measure carrying the `MultiStageRank` render +/// modifier as a SQL window function partitioned by the modifier's +/// members. Everything else goes through `else_processor`. pub struct MultiStageRankNode { else_processor: Rc, - partition: Vec, } impl MultiStageRankNode { - pub fn new(else_processor: Rc, partition: Vec) -> Rc { - Rc::new(Self { - else_processor, - partition, - }) + pub fn new(else_processor: Rc) -> Rc { + Rc::new(Self { else_processor }) } pub fn else_processor(&self) -> &Rc { &self.else_processor } - - pub fn partition(&self) -> &Vec { - &self.partition - } } impl SqlNode for MultiStageRankNode { @@ -43,7 +36,10 @@ impl SqlNode for MultiStageRankNode { ) -> Result { let res = match node.as_ref() { MemberSymbol::Measure(m) => { - if m.is_multi_stage() && matches!(m.kind(), MeasureKind::Rank) { + if let Some(modifier @ MeasureRenderModifier::MultiStageRank { partition }) = + m.render_modifier() + { + modifier.ensure_applies_to(m)?; let inner_visitor = visitor.with_arg_needs_paren_safe(false); let order_by = if !m.measure_order_by().is_empty() { let sql = m @@ -64,11 +60,12 @@ impl SqlNode for MultiStageRankNode { } else { "".to_string() }; - let partition_by = if self.partition.is_empty() { - "".to_string() - } else { - format!("PARTITION BY {} ", self.partition.join(", ")) - }; + let partition_by = render_partition_by( + partition, + &inner_visitor, + node_processor.clone(), + templates, + )?; format!("rank() OVER ({partition_by}{order_by})") } else { self.else_processor.to_sql( diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan/sql_nodes/multi_stage_window.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan/sql_nodes/multi_stage_window.rs index 66806445ca297..adcafea2cf5ec 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan/sql_nodes/multi_stage_window.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan/sql_nodes/multi_stage_window.rs @@ -1,32 +1,26 @@ +use super::window_partition::render_partition_by; use super::SqlNode; use crate::physical_plan::SqlEvaluatorVisitor; use crate::planner::query_tools::QueryTools; use crate::planner::sql_templates::PlanSqlTemplates; -use crate::planner::MemberSymbol; +use crate::planner::{MeasureRenderModifier, MemberSymbol}; use cubenativeutils::CubeError; use std::any::Any; use std::rc::Rc; -/// Wraps a measure as a SQL window function partitioned by -/// `partition`. Used for multi-stage measures whose partition is -/// narrower than the full dimension set. Non-window measures go -/// through `else_processor`. +/// Wraps a measure carrying the `MultiStageWindow` render modifier +/// as a SQL window function partitioned by the modifier's members. +/// Everything else goes through `else_processor`. pub struct MultiStageWindowNode { input: Rc, else_processor: Rc, - partition: Vec, } impl MultiStageWindowNode { - pub fn new( - input: Rc, - else_processor: Rc, - partition: Vec, - ) -> Rc { + pub fn new(input: Rc, else_processor: Rc) -> Rc { Rc::new(Self { input, else_processor, - partition, }) } @@ -37,10 +31,6 @@ impl MultiStageWindowNode { pub fn else_processor(&self) -> &Rc { &self.else_processor } - - pub fn partition(&self) -> &Vec { - &self.partition - } } impl SqlNode for MultiStageWindowNode { @@ -54,7 +44,10 @@ impl SqlNode for MultiStageWindowNode { ) -> Result { let res = match node.as_ref() { MemberSymbol::Measure(m) => { - if m.is_multi_stage() && !m.is_calculated() { + if let Some(modifier @ MeasureRenderModifier::MultiStageWindow { partition }) = + m.render_modifier() + { + modifier.ensure_applies_to(m)?; let inner_visitor = visitor.with_arg_needs_paren_safe(false); let input_sql = self.input.to_sql( &inner_visitor, @@ -64,11 +57,12 @@ impl SqlNode for MultiStageWindowNode { templates, )?; - let partition_by = if self.partition.is_empty() { - "".to_string() - } else { - format!("PARTITION BY {} ", self.partition.join(", ")) - }; + let partition_by = render_partition_by( + partition, + &inner_visitor, + node_processor.clone(), + templates, + )?; let measure_type = m.measure_type(); format!("{measure_type}({measure_type}({input_sql})) OVER ({partition_by})") } else { diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan/sql_nodes/original_sql_pre_aggregation.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan/sql_nodes/original_sql_pre_aggregation.rs deleted file mode 100644 index ca472e524e691..0000000000000 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan/sql_nodes/original_sql_pre_aggregation.rs +++ /dev/null @@ -1,73 +0,0 @@ -use super::SqlNode; -use crate::planner::query_tools::QueryTools; -use crate::planner::MemberSymbol; -use crate::physical_plan::SqlEvaluatorVisitor; -use crate::planner::sql_templates::PlanSqlTemplates; -use cubenativeutils::CubeError; -use std::any::Any; -use std::collections::HashMap; -use std::rc::Rc; - -pub struct OriginalSqlPreAggregationSqlNode { - input: Rc, - original_sql_pre_aggregations: HashMap, -} - -impl OriginalSqlPreAggregationSqlNode { - pub fn new( - input: Rc, - original_pre_aggregations: HashMap, - ) -> Rc { - Rc::new(Self { - input, - original_sql_pre_aggregations: original_pre_aggregations, - }) - } - - pub fn input(&self) -> &Rc { - &self.input - } -} - -impl SqlNode for OriginalSqlPreAggregationSqlNode { - fn to_sql( - &self, - visitor: &SqlEvaluatorVisitor, - node: &Rc, - query_tools: Rc, - node_processor: Rc, - templates: &PlanSqlTemplates, - ) -> Result { - let res = match node.as_ref() { - MemberSymbol::CubeTable(ev) => { - if let Some(original_sql_table_name) = - self.original_sql_pre_aggregations.get(ev.cube_name()) - { - format!("{}", original_sql_table_name) - } else { - self.input.to_sql( - visitor, - node, - query_tools.clone(), - node_processor.clone(), - templates, - )? - } - } - _ => { - return Err(CubeError::internal(format!( - "OriginalSqlPreAggregationSqlNode node processor called for wrong node", - ))); - } - }; - Ok(res) - } - - fn as_any(self: Rc) -> Rc { - self.clone() - } - - fn childs(&self) -> Vec> { - vec![self.input.clone()] - } -} diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan/sql_nodes/rolling_window.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan/sql_nodes/rolling_window.rs index 7ad5e6ed2173d..06a8104f392e8 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan/sql_nodes/rolling_window.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan/sql_nodes/rolling_window.rs @@ -8,9 +8,9 @@ use cubenativeutils::CubeError; use std::any::Any; use std::rc::Rc; -/// Renders cumulative measures (rolling window / running total) -/// through `input`, delegating non-cumulative measures to -/// `default_processor`. +/// Merges the partial values a rolling window produces over `input`. +/// Aggregation types that have no merge form are delegated to +/// `default_processor`, which renders them as a plain aggregation. pub struct RollingWindowNode { input: Rc, default_processor: Rc, @@ -39,7 +39,7 @@ impl SqlNode for RollingWindowNode { templates: &PlanSqlTemplates, ) -> Result { let res = match node.as_ref() { - MemberSymbol::Measure(m) if m.is_cumulative() => { + MemberSymbol::Measure(m) => { let delegate = || { self.default_processor.to_sql( visitor, @@ -61,30 +61,30 @@ impl SqlNode for RollingWindowNode { }; match m.kind() { MeasureKind::Count(_) => format!("sum({})", render_input()?), - MeasureKind::Aggregated(a) => match a.agg_type() { - AggregationType::CountDistinctApprox => { - templates.hll_cardinality_merge(render_input()?)? + // A state form holds the same aggregation as its + // plain counterpart, stored unmerged, so it merges + // the same way. + MeasureKind::Aggregated(a) | MeasureKind::AggregatedState(a) => { + match a.agg_type() { + AggregationType::CountDistinctApprox => { + templates.hll_cardinality_merge(render_input()?)? + } + AggregationType::Sum => { + format!("sum({})", render_input()?) + } + AggregationType::Min | AggregationType::Max => { + format!("{}({})", a.agg_type().as_str(), render_input()?) + } + AggregationType::Avg + | AggregationType::CountDistinct + | AggregationType::NumberAgg => delegate()?, } - AggregationType::Sum => { - format!("sum({})", render_input()?) - } - AggregationType::Min | AggregationType::Max => { - format!("{}({})", a.agg_type().as_str(), render_input()?) - } - AggregationType::Avg - | AggregationType::CountDistinct - | AggregationType::NumberAgg => delegate()?, - }, - _ => delegate()?, + } + MeasureKind::MultipliedCount(_) + | MeasureKind::Calculated(_) + | MeasureKind::Rank => delegate()?, } } - MemberSymbol::Measure(_) => self.default_processor.to_sql( - visitor, - node, - query_tools.clone(), - node_processor, - templates, - )?, _ => { return Err(CubeError::internal(format!( "Unexpected evaluation node type for RollingWindowNode" diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan/sql_nodes/time_dimension.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan/sql_nodes/time_dimension.rs index e22b4408b5bfe..9631e7c7c827c 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan/sql_nodes/time_dimension.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan/sql_nodes/time_dimension.rs @@ -5,27 +5,18 @@ use crate::planner::sql_templates::PlanSqlTemplates; use crate::planner::MemberSymbol; use cubenativeutils::CubeError; use std::any::Any; -use std::collections::HashSet; use std::rc::Rc; /// Renders a time dimension: applies the granularity (predefined -/// or calendar SQL) and timezone conversion, unless the dimension's -/// full name is listed in `dimensions_with_ignored_timezone` (used -/// for pre-aggregation column references). +/// or calendar SQL) and timezone conversion, unless the symbol is +/// marked as already timezone-converted. pub struct TimeDimensionNode { - dimensions_with_ignored_timezone: HashSet, input: Rc, } impl TimeDimensionNode { - pub fn new( - dimensions_with_ignored_timezone: HashSet, - input: Rc, - ) -> Rc { - Rc::new(Self { - dimensions_with_ignored_timezone, - input, - }) + pub fn new(input: Rc) -> Rc { + Rc::new(Self { input }) } } @@ -62,11 +53,7 @@ impl SqlNode for TimeDimensionNode { node_processor.clone(), templates, )?; - let skip_convert_tz = self - .dimensions_with_ignored_timezone - .contains(&ev.full_name()); - - let converted_tz = if skip_convert_tz { + let converted_tz = if ev.tz_converted_at_source() { input_sql } else { templates.convert_tz(input_sql)? diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan/sql_nodes/ungroupped_query_final_measure.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan/sql_nodes/ungroupped_query_final_measure.rs index 94f720e312a36..cdd7073253f49 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan/sql_nodes/ungroupped_query_final_measure.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan/sql_nodes/ungroupped_query_final_measure.rs @@ -38,11 +38,11 @@ impl SqlNode for UngroupedQueryFinalMeasureSqlNode { MemberSymbol::Measure(ev) => { let is_count_like = match ev.kind() { MeasureKind::Count(_) | MeasureKind::MultipliedCount(_) => true, - MeasureKind::Aggregated(a) => matches!( + MeasureKind::Aggregated(a) | MeasureKind::AggregatedState(a) => matches!( a.agg_type(), AggregationType::CountDistinct | AggregationType::CountDistinctApprox ), - _ => false, + MeasureKind::Calculated(_) | MeasureKind::Rank => false, }; // Count-likes wrap the child in `CASE WHEN … IS NOT NULL THEN 1 END` // (safe), other kinds pass through and must propagate the flag. diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan/sql_nodes/window_partition.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan/sql_nodes/window_partition.rs new file mode 100644 index 0000000000000..36b24a57212d6 --- /dev/null +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan/sql_nodes/window_partition.rs @@ -0,0 +1,26 @@ +use super::SqlNode; +use crate::physical_plan::SqlEvaluatorVisitor; +use crate::planner::sql_templates::PlanSqlTemplates; +use crate::planner::MemberSymbol; +use cubenativeutils::CubeError; +use std::rc::Rc; + +/// `PARTITION BY` clause of a multi-stage window function, empty for an +/// unpartitioned window. Trailing space included so it concatenates +/// with the following clause. +pub fn render_partition_by( + partition: &[Rc], + visitor: &SqlEvaluatorVisitor, + node_processor: Rc, + templates: &PlanSqlTemplates, +) -> Result { + if partition.is_empty() { + return Ok("".to_string()); + } + let columns = partition + .iter() + .map(|dim| visitor.apply(dim, node_processor.clone(), templates)) + .collect::, _>>()? + .join(", "); + Ok(format!("PARTITION BY {} ", columns)) +} diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan/symbols/measure_kinds/mod.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan/symbols/measure_kinds/mod.rs index 419362ec7e2e8..d4004ca4bf3e2 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan/symbols/measure_kinds/mod.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan/symbols/measure_kinds/mod.rs @@ -10,7 +10,7 @@ impl ToSql for MeasureKind { fn to_sql(&self, ctx: &MemberSqlContext) -> Result { match self { Self::Count(c) | Self::MultipliedCount(c) => c.to_sql(ctx), - Self::Aggregated(a) => a.to_sql(ctx), + Self::Aggregated(a) | Self::AggregatedState(a) => a.to_sql(ctx), Self::Calculated(c) => c.to_sql(ctx), Self::Rank => Err(CubeError::internal(format!( "Rank measure doesn't support direct evaluation for {}", diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan_builder/builder.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan_builder/builder.rs index 6cd6819b01c19..490aa419dcbd3 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan_builder/builder.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan_builder/builder.rs @@ -60,11 +60,9 @@ impl PhysicalPlanBuilder { logical_plan: Rc, original_sql_pre_aggregations: HashMap, total_query: bool, - pre_aggregation_query: bool, ) -> Result, CubeError> { let mut context = PushDownBuilderContext::default(); context.original_sql_pre_aggregations = original_sql_pre_aggregations; - context.render_measure_as_state = pre_aggregation_query; let query = self.build_impl(logical_plan, &context)?; let query = if total_query { self.build_total_count(query, &context)? diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan_builder/context.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan_builder/context.rs index 146a5c0f3fad2..a9dff4ee578f8 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan_builder/context.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan_builder/context.rs @@ -17,7 +17,6 @@ pub struct MultiStageDimensionContext { #[derive(Clone, Debug, Default)] pub(super) struct PushDownBuilderContext { pub alias_prefix: Option, - pub render_measure_as_state: bool, //Render measure as state, for example hll state for count_approx pub render_measure_for_ungrouped: bool, pub time_shifts: TimeShiftState, pub original_sql_pre_aggregations: HashMap, @@ -35,7 +34,6 @@ impl PushDownBuilderContext { /// multiplied-subquery processors go through here, so the two /// stay in sync field-for-field. pub fn apply_evaluation_context(&mut self, evaluation_context: &EvaluationContext) { - self.render_measure_as_state = evaluation_context.measure_as_state; self.render_measure_for_ungrouped = evaluation_context.measure_for_ungrouped; self.time_shifts = evaluation_context.time_shifts.clone(); } @@ -50,8 +48,6 @@ impl PushDownBuilderContext { factory.set_time_shifts(common_time_shifts); factory.set_calendar_time_shifts(calendar_time_shifts); - factory.set_count_approx_as_state(self.render_measure_as_state); - factory.set_ungrouped_measure(self.render_measure_for_ungrouped); factory.set_original_sql_pre_aggregations(self.original_sql_pre_aggregations.clone()); Ok(factory) } diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan_builder/processors/aggregate_multiplied_subquery.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan_builder/processors/aggregate_multiplied_subquery.rs index e185d7fb0e8e0..96a198f867b38 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan_builder/processors/aggregate_multiplied_subquery.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan_builder/processors/aggregate_multiplied_subquery.rs @@ -1,4 +1,5 @@ use super::super::{LogicalNodeProcessor, ProcessableNode, PushDownBuilderContext}; +use crate::logical_plan::transforms as logical_transforms; use crate::logical_plan::{AggregateMultipliedSubquery, AggregateMultipliedSubquerySource}; use crate::physical_plan::ReferencesBuilder; use crate::physical_plan::VisitorContext; @@ -7,6 +8,7 @@ use crate::physical_plan::{ SelectBuilder, }; use crate::physical_plan_builder::PhysicalPlanBuilder; +use crate::planner::MeasureRenderModifier; use cubenativeutils::CubeError; use std::rc::Rc; @@ -166,7 +168,18 @@ impl<'a> LogicalNodeProcessor<'a, AggregateMultipliedSubquery> &mut context_factory, )?; - for member in aggregate_multiplied_subquery.schema.all_dimensions() { + // Under a measure-rendering context (a CTE hoisted out of an + // ungrouped multi-stage leaf) measures emit raw row-level values. + let schema = if context.render_measure_for_ungrouped { + logical_transforms::measures_render_modifier_in_schema( + &aggregate_multiplied_subquery.schema, + &MeasureRenderModifier::RawValue, + )? + } else { + aggregate_multiplied_subquery.schema.clone() + }; + + for member in schema.all_dimensions() { references_builder.resolve_references_for_member( member.clone(), &None, @@ -176,10 +189,7 @@ impl<'a> LogicalNodeProcessor<'a, AggregateMultipliedSubquery> group_by.push(Expr::Member(MemberExpression::new(member.clone()))); select_builder.add_projection_member(&member, alias); } - for (measure, exists) in self - .builder - .measures_for_query(&aggregate_multiplied_subquery.schema.measures, &context) - { + for (measure, exists) in self.builder.measures_for_query(&schema.measures, &context) { if exists { if matches!( &aggregate_multiplied_subquery.source, diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan_builder/processors/keys_sub_query.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan_builder/processors/keys_sub_query.rs index 684592d766490..86a28dbea19f8 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan_builder/processors/keys_sub_query.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan_builder/processors/keys_sub_query.rs @@ -5,7 +5,7 @@ use crate::physical_plan::{ }; use crate::physical_plan_builder::PhysicalPlanBuilder; use crate::planner::collectors::collect_calc_group_dims_from_nodes; -use crate::planner::get_filtered_values; +use crate::planner::symbols::transforms::get_filtered_values; use cubenativeutils::CubeError; use itertools::Itertools as _; use std::rc::Rc; diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan_builder/processors/measure_subquery.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan_builder/processors/measure_subquery.rs index 1278a0f4d7e43..33a612b85a1d4 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan_builder/processors/measure_subquery.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan_builder/processors/measure_subquery.rs @@ -3,6 +3,8 @@ use crate::logical_plan::MeasureSubquery; use crate::physical_plan::ReferencesBuilder; use crate::physical_plan::{Select, SelectBuilder}; use crate::physical_plan_builder::PhysicalPlanBuilder; +use crate::planner::symbols::transforms; +use crate::planner::MeasureRenderModifier; use cubenativeutils::CubeError; use std::rc::Rc; @@ -38,12 +40,14 @@ impl<'a> LogicalNodeProcessor<'a, MeasureSubquery> for MeasureSubqueryProcessor< for dim in measure_subquery.schema.dimensions.iter() { select_builder.add_projection_member(dim, None); } + // The subquery emits raw row-level measure values; the enclosing + // aggregate select applies the actual aggregation. for meas in measure_subquery.schema.measures.iter() { - select_builder.add_projection_member(meas, None); + let meas = + transforms::measures_render_modifier(meas, &MeasureRenderModifier::RawValue)?; + select_builder.add_projection_member(&meas, None); } - context_factory.set_ungrouped_measure(true); - let select = Rc::new(select_builder.build(query_tools.clone(), context_factory)); Ok(select) } diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan_builder/processors/multi_stage_measure_calculation.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan_builder/processors/multi_stage_measure_calculation.rs index 7a4f2a2ed4c11..8a2fd5254033b 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan_builder/processors/multi_stage_measure_calculation.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan_builder/processors/multi_stage_measure_calculation.rs @@ -1,9 +1,11 @@ use super::super::context::PushDownBuilderContext; use super::super::{LogicalNodeProcessor, ProcessableNode}; +use crate::logical_plan::transforms as logical_transforms; use crate::logical_plan::{MultiStageCalculationWindowFunction, MultiStageMeasureCalculation}; use crate::physical_plan::ReferencesBuilder; use crate::physical_plan::{Expr, MemberExpression, QueryPlan, SelectBuilder}; use crate::physical_plan_builder::PhysicalPlanBuilder; +use crate::planner::MeasureRenderModifier; use cubenativeutils::CubeError; use itertools::Itertools; use std::rc::Rc; @@ -25,7 +27,7 @@ impl<'a> LogicalNodeProcessor<'a, MultiStageMeasureCalculation> measure_calculation: &MultiStageMeasureCalculation, context: &PushDownBuilderContext, ) -> Result { - let (query_tools, templates) = self.builder.qtools_and_templates(); + let query_tools = self.builder.query_tools(); let mut context_factory = context.make_sql_nodes_factory()?; let from = self .builder @@ -48,7 +50,45 @@ impl<'a> LogicalNodeProcessor<'a, MultiStageMeasureCalculation> select_builder.add_projection_member(&member, None); } - for measure in measure_calculation.schema().measures.iter() { + for dim in measure_calculation.partition_by().iter() { + references_builder.resolve_references_for_member( + dim.clone(), + &None, + context_factory.render_references_mut(), + )?; + if references_builder + .find_reference_for_member(&dim, &None) + .is_none() + { + return Err(CubeError::internal(format!( + "Alias not found for partition_by dimension {}", + dim.full_name() + ))); + } + } + let measure_modifier = match measure_calculation.window_function_to_use() { + MultiStageCalculationWindowFunction::Rank => { + Some(MeasureRenderModifier::MultiStageRank { + partition: measure_calculation.partition_by().clone(), + }) + } + MultiStageCalculationWindowFunction::Window => { + Some(MeasureRenderModifier::MultiStageWindow { + partition: measure_calculation.partition_by().clone(), + }) + } + MultiStageCalculationWindowFunction::None => None, + }; + let schema = if let Some(modifier) = &measure_modifier { + logical_transforms::measures_render_modifier_in_schema( + measure_calculation.schema(), + modifier, + )? + } else { + measure_calculation.schema().clone() + }; + + for measure in schema.measures.iter() { references_builder.resolve_references_for_member( measure.clone(), &None, @@ -68,42 +108,10 @@ impl<'a> LogicalNodeProcessor<'a, MultiStageMeasureCalculation> select_builder.set_group_by(group_by); select_builder.set_order_by( self.builder - .make_order_by(measure_calculation.schema(), measure_calculation.order_by())?, + .make_order_by(&schema, measure_calculation.order_by())?, ); } - let partition_by = measure_calculation - .partition_by() - .iter() - .map(|dim| -> Result<_, CubeError> { - if let Some(reference) = references_builder.find_reference_for_member(&dim, &None) { - let table_ref = if let Some(table_name) = reference.source() { - format!("{}.", templates.quote_identifier(table_name)?) - } else { - format!("") - }; - Ok(format!( - "{}{}", - table_ref, - templates.quote_identifier(&reference.name())? - )) - } else { - Err(CubeError::internal(format!( - "Alias not found for partition_by dimension {}", - dim.full_name() - ))) - } - }) - .collect::, _>>()?; - match measure_calculation.window_function_to_use() { - MultiStageCalculationWindowFunction::Rank => { - context_factory.set_multi_stage_rank(partition_by) - } - MultiStageCalculationWindowFunction::Window => { - context_factory.set_multi_stage_window(partition_by) - } - MultiStageCalculationWindowFunction::None => {} - } let select = Rc::new(select_builder.build(query_tools.clone(), context_factory)); Ok(QueryPlan::Select(select)) } diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan_builder/processors/multi_stage_rolling_window.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan_builder/processors/multi_stage_rolling_window.rs index d34a89a2246e0..c0186cc85bcfb 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan_builder/processors/multi_stage_rolling_window.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan_builder/processors/multi_stage_rolling_window.rs @@ -1,5 +1,6 @@ use super::super::context::PushDownBuilderContext; use super::super::{LogicalNodeProcessor, ProcessableNode}; +use crate::logical_plan::transforms; use crate::logical_plan::{MultiStageRollingWindow, MultiStageRollingWindowType}; use crate::physical_plan::ReferencesBuilder; use crate::physical_plan::{ @@ -7,6 +8,7 @@ use crate::physical_plan::{ SelectBuilder, }; use crate::physical_plan_builder::PhysicalPlanBuilder; +use crate::planner::MeasureRenderModifier; use cubenativeutils::CubeError; use std::rc::Rc; @@ -82,7 +84,6 @@ impl<'a> LogicalNodeProcessor<'a, MultiStageRollingWindow> ); let mut context_factory = context.make_sql_nodes_factory()?; - context_factory.set_rolling_window(true); let from = From::new_from_join(join_builder.build()); let references_builder = ReferencesBuilder::new(from.clone()); let mut select_builder = SelectBuilder::new(from.clone()); @@ -102,14 +103,32 @@ impl<'a> LogicalNodeProcessor<'a, MultiStageRollingWindow> QualifiedColumnName::new(Some(root_alias.clone()), format!("date_from")), ); - for dim in rolling_window.schema.time_dimensions.iter() { - context_factory.add_dimensions_with_ignored_timezone(dim.full_name()); + // Time dimensions are read from the rolling source input, where they + // are already timezone-converted and truncated, so they must render + // without the conversion. + let schema = transforms::mark_tz_converted_at_source_in_schema(&rolling_window.schema)?; + // An ungrouped rolling select emits row-level values: count-like + // measures render a not-null indicator over the input column; + // otherwise the select merges the window's partial values. + let schema = if rolling_window.is_ungrouped { + transforms::measures_render_modifier_in_schema( + &schema, + &MeasureRenderModifier::UngroupedFinal, + )? + } else { + transforms::measures_render_modifier_in_schema( + &schema, + &MeasureRenderModifier::RollingMerge, + )? + }; + + for dim in schema.time_dimensions.iter() { let alias = references_builder .resolve_alias_for_member(&dim, &Some(measure_input_alias.clone())); select_builder.add_projection_member(dim, alias); } - for dim in rolling_window.schema.dimensions.iter() { + for dim in schema.dimensions.iter() { if dim.clone().resolve_reference_chain() != time_dimension.clone().resolve_reference_chain() { @@ -124,7 +143,7 @@ impl<'a> LogicalNodeProcessor<'a, MultiStageRollingWindow> select_builder.add_projection_member(dim, alias); } - for measure in rolling_window.schema.measures.iter() { + for measure in schema.measures.iter() { let name_in_base_query = measure_input_schema.resolve_member_alias(measure); context_factory.add_ungrouped_measure_reference( measure.full_name(), @@ -135,8 +154,7 @@ impl<'a> LogicalNodeProcessor<'a, MultiStageRollingWindow> } if !rolling_window.is_ungrouped { - let group_by = rolling_window - .schema + let group_by = schema .all_dimensions() .map(|dim| -> Result<_, CubeError> { Ok(Expr::Member(MemberExpression::new(dim.clone()))) @@ -145,10 +163,8 @@ impl<'a> LogicalNodeProcessor<'a, MultiStageRollingWindow> select_builder.set_group_by(group_by); select_builder.set_order_by( self.builder - .make_order_by(&rolling_window.schema, &rolling_window.order_by)?, + .make_order_by(&schema, &rolling_window.order_by)?, ); - } else { - context_factory.set_ungrouped(true); } let select = Rc::new(select_builder.build(query_tools.clone(), context_factory)); diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan_builder/processors/query.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan_builder/processors/query.rs index 3276753c09d47..c53fe7bf4fa7b 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan_builder/processors/query.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan_builder/processors/query.rs @@ -1,4 +1,5 @@ use super::super::{LogicalNodeProcessor, ProcessableNode, PushDownBuilderContext}; +use crate::logical_plan::transforms as logical_transforms; use crate::logical_plan::{all_symbols, Query, QuerySource}; use crate::physical_plan::{ CalcGroupItem, CalcGroupsJoin, Expr, From, MemberExpression, ReferencesBuilder, Select, @@ -6,7 +7,9 @@ use crate::physical_plan::{ }; use crate::physical_plan_builder::PhysicalPlanBuilder; use crate::planner::collectors::collect_calc_group_dims_from_nodes; -use crate::planner::get_filtered_values; +use crate::planner::symbols::transforms; +use crate::planner::symbols::transforms::get_filtered_values; +use crate::planner::{MeasureRenderModifier, MemberSymbol, OrderByItem}; use cubenativeutils::CubeError; use itertools::Itertools; use std::collections::HashSet; @@ -64,8 +67,8 @@ impl<'a> LogicalNodeProcessor<'a, Query> for QueryProcessor<'a> { } let from = self.builder.process_node(logical_plan.source(), &context)?; - let filter = logical_plan.filter().all_filters(); - let having = logical_plan.filter().measures_filter(); + let mut filter = logical_plan.filter().all_filters(); + let mut having = logical_plan.filter().measures_filter(); // Calc-group dimensions are resolved at query time: a value pinned by // a filter renders as a literal, otherwise the enumeration is @@ -118,6 +121,8 @@ impl<'a> LogicalNodeProcessor<'a, Query> for QueryProcessor<'a> { from }; + let mut schema = logical_plan.schema().clone(); + match logical_plan.source() { QuerySource::LogicalJoin(join) => { let references_builder = ReferencesBuilder::new(from.clone()); @@ -128,9 +133,21 @@ impl<'a> LogicalNodeProcessor<'a, Query> for QueryProcessor<'a> { )?; } QuerySource::PreAggregation(pre_aggregation) => { - for member in logical_plan.schema().time_dimensions.iter() { - context_factory.add_dimensions_with_ignored_timezone(member.full_name()); - } + // A rollup stores time dimensions already timezone-converted, + // so every occurrence of them in this select must render + // without the conversion. + let time_dimension_names = schema + .time_dimensions + .iter() + .map(|d| d.full_name()) + .collect::>(); + let mark_tz_converted = + |symbol: &Rc| -> Result, CubeError> { + transforms::mark_tz_converted_at_source(symbol, &time_dimension_names) + }; + schema = logical_transforms::mark_tz_converted_at_source_in_schema(&schema)?; + filter = transforms::map_filter_symbols(filter, &mark_tz_converted)?; + having = transforms::map_filter_symbols(having, &mark_tz_converted)?; context_factory.set_use_local_tz_in_date_range(true); for (name, column) in pre_aggregation.all_dimensions_refererences().into_iter() { @@ -143,24 +160,41 @@ impl<'a> LogicalNodeProcessor<'a, Query> for QueryProcessor<'a> { QuerySource::FullKeyAggregate(_) => {} } + // An ungrouped select emits row-level measure values: raw ones + // under a measure-rendering context (multi-stage leaves), a + // not-null indicator form for count-likes otherwise. + let measure_modifier = if context.render_measure_for_ungrouped { + Some(MeasureRenderModifier::RawValue) + } else if logical_plan.modifers().ungrouped { + Some(MeasureRenderModifier::UngroupedFinal) + } else { + None + }; + if let Some(modifier) = &measure_modifier { + let stamp = |symbol: &Rc| -> Result, CubeError> { + transforms::measures_render_modifier(symbol, modifier) + }; + schema = logical_transforms::measures_render_modifier_in_schema(&schema, modifier)?; + filter = transforms::map_filter_symbols(filter, &stamp)?; + having = transforms::map_filter_symbols(having, &stamp)?; + } + let is_pre_aggregation = matches!(logical_plan.source(), QuerySource::PreAggregation(_)); let references_builder = ReferencesBuilder::new(from.clone()); let mut select_builder = SelectBuilder::new(from); - context_factory.set_ungrouped(logical_plan.modifers().ungrouped); if !logical_plan.modifers().ungrouped { context_factory.set_group_by_members( - logical_plan - .schema() + schema .all_dimensions() .map(|symbol| symbol.full_name()) .collect(), ); } - for dimension in logical_plan.schema().all_dimensions() { + for dimension in schema.all_dimensions() { self.builder.process_query_dimension( dimension, &references_builder, @@ -170,10 +204,7 @@ impl<'a> LogicalNodeProcessor<'a, Query> for QueryProcessor<'a> { )?; } - for (measure, exists) in self - .builder - .measures_for_query(&logical_plan.schema().measures, &context) - { + for (measure, exists) in self.builder.measures_for_query(&schema.measures, &context) { if exists { references_builder.resolve_references_for_member( measure.clone(), @@ -192,8 +223,7 @@ impl<'a> LogicalNodeProcessor<'a, Query> for QueryProcessor<'a> { select_builder.set_filter(having); } else { if !logical_plan.modifers().ungrouped { - let group_by = logical_plan - .schema() + let group_by = schema .all_dimensions() .map(|symbol| -> Result<_, CubeError> { Ok(Expr::Member(MemberExpression::new(symbol.clone()))) @@ -216,9 +246,6 @@ impl<'a> LogicalNodeProcessor<'a, Query> for QueryProcessor<'a> { context_factory.add_render_reference(name, value); } } - if logical_plan.modifers().ungrouped { - context_factory.set_ungrouped(true); - } // When reading from a pre-aggregation, drop ORDER BY keys on measures that // are not part of the selection. CubeStore cannot ORDER BY an aggregate of a @@ -230,20 +257,33 @@ impl<'a> LogicalNodeProcessor<'a, Query> for QueryProcessor<'a> { .iter() .filter(|o| { !(o.member_symbol().is_measure() - && logical_plan - .schema() - .find_member_positions(&o.name()) - .is_empty()) + && schema.find_member_positions(&o.name()).is_empty()) }) .cloned() .collect() } else { logical_plan.modifers().order_by.clone() }; - select_builder.set_order_by( - self.builder - .make_order_by(logical_plan.schema(), &order_by)?, - ); + // Items present in the schema are sorted by their stamped schema + // symbol; only a measure absent from the projection carries its + // own symbol into the ORDER BY and needs the form here. + let order_by = if let Some(modifier) = &measure_modifier { + order_by + .iter() + .map(|o| -> Result<_, CubeError> { + if !schema.find_member_positions(&o.name()).is_empty() { + return Ok(o.clone()); + } + Ok(OrderByItem::new( + transforms::measures_render_modifier(&o.member_symbol(), modifier)?, + o.desc(), + )) + }) + .collect::, _>>()? + } else { + order_by + }; + select_builder.set_order_by(self.builder.make_order_by(&schema, &order_by)?); let res = Rc::new(select_builder.build(query_tools.clone(), context_factory)); Ok(res) diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/filter/base_filter.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/filter/base_filter.rs index 8548c9b3db5f3..b059b7baecef6 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/filter/base_filter.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/filter/base_filter.rs @@ -98,11 +98,11 @@ impl BaseFilter { &self, member_evaluator: Rc, ) -> Result, CubeError> { - // No compiler here (called from static-filter symbol rewriting, which - // has none). A member swap keeps the same operator/values, so the only - // branch that would need a compiler is a to_date rolling window — not - // reachable from this path. FIXME: removed once granularities are - // resolved during early compilation rather than at filter-build time. + // No compiler here. A member swap keeps the same operator/values, so + // the only branch that would need a compiler is a to_date rolling + // window — not reachable from this path. FIXME: removed once + // granularities are resolved during early compilation rather than at + // filter-build time. let typed_filter = self .typed_filter .to_builder() diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/member_query_planner.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/member_query_planner.rs index e16859a349567..f5d4849cad38e 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/member_query_planner.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/member_query_planner.rs @@ -5,6 +5,7 @@ use super::{ use crate::logical_plan::*; use crate::planner::planners::{multi_stage::RollingWindowType, QueryPlanner, SimpleQueryPlanner}; use crate::planner::state::State; +use crate::planner::symbols::transforms; use crate::planner::GranularityHelper; use crate::planner::MemberSymbol; use crate::planner::MultiStageGrain; @@ -424,7 +425,16 @@ impl MultiStageMemberQueryPlanner { &self, scope: &mut PlanningScope, ) -> Result, CubeError> { - let member_node = self.description.member_node(); + // An aggregating stage on top (a rolling window) merges the leaf's + // values, so measures with a mergeable state form must materialize + // the state, not the final value. + let leaf_as_state = self.description.member().has_aggregates_on_top(); + let member_node = if leaf_as_state { + transforms::measures_as_state(self.description.member_node())? + } else { + self.description.member_node().clone() + }; + let member_node = &member_node; let mut dimensions = self.description.state().dimensions().clone(); let mut time_dimensions = self.description.state().time_dimensions().clone(); let mut measures = vec![]; @@ -451,6 +461,15 @@ impl MultiStageMemberQueryPlanner { } } + let mut measures_filters = self.description.state().measures_filters().clone(); + if leaf_as_state { + for filter_item in measures_filters.iter_mut() { + *filter_item = transforms::map_filter_item_symbols( + filter_item, + &transforms::measures_as_state, + )?; + } + } let cte_query_properties = QueryProperties::builder() .query_tools(self.query_tools.clone()) .measures(measures) @@ -458,7 +477,7 @@ impl MultiStageMemberQueryPlanner { .time_dimensions(time_dimensions) .time_dimensions_filters(self.description.state().time_dimensions_filters().clone()) .dimensions_filters(self.description.state().dimensions_filters().clone()) - .measures_filters(self.description.state().measures_filters().clone()) + .measures_filters(measures_filters) .segments(self.description.state().segments().clone()) .ignore_cumulative(true) .ungrouped(self.description.member().is_ungrupped()) @@ -476,7 +495,6 @@ impl MultiStageMemberQueryPlanner { // itself renders with. let evaluation_context = EvaluationContext { time_shifts: self.description.state().time_shifts().clone(), - measure_as_state: self.description.member().has_aggregates_on_top(), measure_for_ungrouped: self.description.member().is_ungrupped(), }; let query = scope.with_evaluation_context(evaluation_context.clone(), |scope| { diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/multi_stage_query_planner.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/multi_stage_query_planner.rs index a652fcbbb1c63..a52869d71a828 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/multi_stage_query_planner.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/multi_stage_query_planner.rs @@ -6,7 +6,6 @@ use super::{ use crate::cube_bridge::base_query_options::FilterValue; use crate::cube_bridge::measure_definition::RollingWindow; use crate::logical_plan::*; -use crate::planner::apply_static_filter_to_symbol; use crate::planner::collectors::has_multi_stage_members; use crate::planner::collectors::member_childs; use crate::planner::filter::base_filter::FilterType; @@ -14,6 +13,7 @@ use crate::planner::filter::BaseFilter; use crate::planner::filter::FilterItem; use crate::planner::filter::FilterOperator; use crate::planner::state::State; +use crate::planner::symbols::transforms; use crate::planner::symbols::AggregationType; use crate::planner::Case; use crate::planner::CaseSwitchDefinition; @@ -160,7 +160,10 @@ impl MultiStageQueryPlanner { let member_type = match measure.kind() { MeasureKind::Rank => MultiStageInodeMemberType::Rank, MeasureKind::Calculated(_) => MultiStageInodeMemberType::Calculate, - _ => MultiStageInodeMemberType::Aggregate, + MeasureKind::Count(_) + | MeasureKind::MultipliedCount(_) + | MeasureKind::Aggregated(_) + | MeasureKind::AggregatedState(_) => MultiStageInodeMemberType::Aggregate, }; let time_shift = measure.time_shift().cloned(); @@ -276,7 +279,10 @@ impl MultiStageQueryPlanner { match inner.kind() { MeasureKind::Count(_) => true, MeasureKind::Aggregated(a) => a.agg_type() == AggregationType::Sum, - _ => false, + MeasureKind::MultipliedCount(_) + | MeasureKind::AggregatedState(_) + | MeasureKind::Calculated(_) + | MeasureKind::Rank => false, } } @@ -465,7 +471,8 @@ impl MultiStageQueryPlanner { scope: &mut PlanningScope, ) -> Result, CubeError> { let member = member.resolve_reference_chain(); - let member = apply_static_filter_to_symbol(&member, state.dimensions_filters())?; + let member = + transforms::apply_static_filter_to_symbol(&member, state.dimensions_filters())?; let state = if member.is_dimension() { let mut new_state = state.as_ref().clone(); new_state.remove_multistage_dimensions(resolved_multi_stage_dimensions)?; @@ -701,7 +708,7 @@ impl MultiStageQueryPlanner { } } - let base_member = MemberSymbol::new_measure(measure.new_unrolling()); + let base_member = MemberSymbol::new_measure(transforms::unroll_rolling(&measure)); if time_dimensions.is_empty() { let base_state = diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/planning_scope.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/planning_scope.rs index fea3932b53917..adc4799eee865 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/planning_scope.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/planning_scope.rs @@ -34,13 +34,12 @@ impl CteState { } } -/// How values are evaluated within a multi-stage leaf scope: the -/// time basis of its dimensions and the measure evaluation shape -/// (mergeable state for aggregates-on-top, ungrouped evaluation). +/// How values are evaluated within a multi-stage leaf scope: the time +/// basis of its dimensions and whether measures are evaluated at row +/// grain for an enclosing aggregation. #[derive(Clone, Default)] pub struct EvaluationContext { pub time_shifts: TimeShiftState, - pub measure_as_state: bool, pub measure_for_ungrouped: bool, } diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/planners/multiplied_measures_query_planner.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/planners/multiplied_measures_query_planner.rs index c1c2115f3864c..b20eb9bfdc0a9 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/planners/multiplied_measures_query_planner.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/planners/multiplied_measures_query_planner.rs @@ -6,6 +6,7 @@ use crate::planner::collectors::{ }; use crate::planner::planners::multi_stage::{EvaluationContext, PlanningScope}; use crate::planner::state::State; +use crate::planner::symbols::transforms; use crate::planner::JoinTree; use crate::planner::MemberSymbol; use crate::planner::{FullKeyAggregateMeasures, QueryProperties}; @@ -203,7 +204,7 @@ impl MultipliedMeasuresQueryPlanner { key_cube_name: &String, ) -> Result { for measure in measures.iter() { - let owned_measure = measure.with_stripped_join_prefix(); + let owned_measure = transforms::strip_join_prefix(measure); let member_expression_over_dimensions_cubes = if let Ok(member_expression) = owned_measure.as_member_expression() { member_expression.cube_names_if_dimension_only_expression()? diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/query_properties.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/query_properties.rs index 70ba0941b5b73..c69ad672ab318 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/query_properties.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/query_properties.rs @@ -16,10 +16,8 @@ use crate::planner::filter::{Filter, FilterGroup, FilterItem, FilterOperator}; use crate::planner::join_hints::JoinHints; use crate::planner::multi_fact_join_groups::{MeasuresJoinHints, MultiFactJoinGroups}; use crate::planner::planners::multi_stage::TimeShiftState; -use crate::planner::{ - apply_static_filter_to_filter_item, apply_static_filter_to_symbol, DimensionTimeShift, - JoinTree, MeasureTimeShifts, -}; +use crate::planner::symbols::transforms; +use crate::planner::{DimensionTimeShift, JoinTree, MeasureTimeShifts}; use cubenativeutils::CubeError; use itertools::Itertools; use std::cell::OnceCell; @@ -114,13 +112,7 @@ impl FullKeyAggregateMeasures { /// form recorded during classification. Measures with no multiplied /// count pass through unchanged. pub fn render(&self, measure: &Rc) -> Result, CubeError> { - measure.apply_recursive(&|node| { - Ok(self - .render_forms - .get(&node.full_name()) - .cloned() - .unwrap_or_else(|| node.clone())) - }) + transforms::substitute_by_name(measure, &self.render_forms) } } @@ -214,6 +206,20 @@ impl From for Result, CubeError> { }); } qp.apply_static_filters()?; + // A pre-aggregation build stores aggregations for later rollup, so + // measures with a mergeable state form must materialize the state, + // not the final value. + if qp.pre_aggregation_query { + for meas in qp.measures.iter_mut() { + *meas = transforms::measures_as_state(meas)?; + } + for filter_item in qp.measures_filters.iter_mut() { + *filter_item = transforms::map_filter_item_symbols( + filter_item, + &transforms::measures_as_state, + )?; + } + } Ok(Rc::new(qp)) } } @@ -233,29 +239,35 @@ impl QueryProperties { fn apply_static_filters(&mut self) -> Result<(), CubeError> { let dimensions_filters = self.dimensions_filters.clone(); for dim in self.dimensions.iter_mut() { - *dim = apply_static_filter_to_symbol(dim, &dimensions_filters)?; + *dim = transforms::apply_static_filter_to_symbol(dim, &dimensions_filters)?; } for dim in self.time_dimensions.iter_mut() { - *dim = apply_static_filter_to_symbol(dim, &dimensions_filters)?; + *dim = transforms::apply_static_filter_to_symbol(dim, &dimensions_filters)?; } for meas in self.measures.iter_mut() { - *meas = apply_static_filter_to_symbol(meas, &dimensions_filters)?; + *meas = transforms::apply_static_filter_to_symbol(meas, &dimensions_filters)?; } for filter_item in self.dimensions_filters.iter_mut() { - *filter_item = apply_static_filter_to_filter_item(filter_item, &dimensions_filters)?; + *filter_item = + transforms::apply_static_filter_to_filter_item(filter_item, &dimensions_filters)?; } for filter_item in self.measures_filters.iter_mut() { - *filter_item = apply_static_filter_to_filter_item(filter_item, &dimensions_filters)?; + *filter_item = + transforms::apply_static_filter_to_filter_item(filter_item, &dimensions_filters)?; } for filter_item in self.time_dimensions_filters.iter_mut() { - *filter_item = apply_static_filter_to_filter_item(filter_item, &dimensions_filters)?; + *filter_item = + transforms::apply_static_filter_to_filter_item(filter_item, &dimensions_filters)?; } for filter_item in self.segments.iter_mut() { - *filter_item = apply_static_filter_to_filter_item(filter_item, &dimensions_filters)?; + *filter_item = + transforms::apply_static_filter_to_filter_item(filter_item, &dimensions_filters)?; } for order_item in self.order_by.iter_mut().flatten() { - order_item.member_evaluator = - apply_static_filter_to_symbol(&order_item.member_evaluator, &dimensions_filters)?; + order_item.member_evaluator = transforms::apply_static_filter_to_symbol( + &order_item.member_evaluator, + &dimensions_filters, + )?; } Ok(()) } @@ -542,7 +554,7 @@ impl QueryProperties { // main query or moves to a multiplied subquery. let rendered = match measure .as_ref() - .and_then(|m| m.convert_multiplied_to_regular()) + .and_then(|m| transforms::regular_in_multiplied(m)) { Some(regular) => { result.regular_measures.push(regular.clone()); @@ -550,7 +562,7 @@ impl QueryProperties { } None => { let rendered = measure - .map(|m| m.into_multiplied()) + .map(|m| transforms::into_multiplied(&m)) .unwrap_or_else(|| item.measure.clone()); result .multiplied_measures diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/query_properties_compiler.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/query_properties_compiler.rs index c184016389ac0..758d33fb85ea3 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/query_properties_compiler.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/query_properties_compiler.rs @@ -23,6 +23,7 @@ use super::filter::{BaseSegment, FilterItem}; use super::join_hints::JoinHints; use super::query_properties::{OrderByItem, QueryProperties}; use super::state::State; +use super::symbols::transforms::patch_measure; use super::{ Compiler, GranularityHelper, MemberExpressionExpression, MemberExpressionSymbol, MemberSymbol, TimeDimensionSymbol, @@ -355,7 +356,7 @@ impl QueryPropertiesCompiler { let resolved_source = source_measure_compiled.clone().resolve_reference_chain(); let symbol = if let Ok(source_measure) = resolved_source.as_measure() { let patched_measure = - source_measure.new_patched(new_measure_type, filters_to_add)?; + patch_measure(&source_measure, new_measure_type, filters_to_add)?; MemberSymbol::new_measure(patched_measure) } else { source_measure_compiled diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/symbols/common/case.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/symbols/common/case.rs index 8bf4d4e97eb81..827d14c0f1ef2 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/symbols/common/case.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/symbols/common/case.rs @@ -5,7 +5,7 @@ use crate::{ case_switch_definition::CaseSwitchDefinition as NativeCaseSwitchDefinition, case_variant::CaseVariant, string_or_sql::StringOrSql, }, - planner::{find_value_restriction, Compiler, MemberSymbol, SqlCall}, + planner::{symbols::transforms::find_value_restriction, Compiler, MemberSymbol, SqlCall}, }; use cubenativeutils::CubeError; use itertools::Itertools; diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/symbols/common/mod.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/symbols/common/mod.rs index 1e7681c3847e4..7c0ec5f12e08b 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/symbols/common/mod.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/symbols/common/mod.rs @@ -3,7 +3,6 @@ mod case; mod compiled_member_path; mod dimension_type; mod multi_stage; -mod static_filter; mod symbol_path; pub use aggregation_type::*; @@ -11,5 +10,4 @@ pub use case::*; pub use compiled_member_path::*; pub use dimension_type::*; pub use multi_stage::*; -pub use static_filter::*; pub use symbol_path::*; diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/symbols/dimension_symbol.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/symbols/dimension_symbol.rs index 4d17518723fc7..05a513f9d12b9 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/symbols/dimension_symbol.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/symbols/dimension_symbol.rs @@ -33,17 +33,17 @@ pub struct CalendarDimensionTimeShift { /// group, filter or order by, but never aggregate. #[derive(Clone)] pub struct DimensionSymbol { - compiled_path: CompiledMemberPath, - kind: DimensionKind, - is_reference: bool, // Symbol is a direct reference to another symbol without any calculations - is_view: bool, - multi_stage: Option, - time_shift: Vec, - time_shift_pk_full_name: Option, - is_self_time_shift_pk: bool, // If the dimension itself is a primary key and has time shifts, we can not reevaluate itself again while processing time shifts to avoid infinite recursion. So we raise this flag instead. - is_sub_query: bool, - propagate_filters_to_sub_query: bool, - mask_sql: Option>, + pub(super) compiled_path: CompiledMemberPath, + pub(super) kind: DimensionKind, + pub(super) is_reference: bool, // Symbol is a direct reference to another symbol without any calculations + pub(super) is_view: bool, + pub(super) multi_stage: Option, + pub(super) time_shift: Vec, + pub(super) time_shift_pk_full_name: Option, + pub(super) is_self_time_shift_pk: bool, // If the dimension itself is a primary key and has time shifts, we can not reevaluate itself again while processing time shifts to avoid infinite recursion. So we raise this flag instead. + pub(super) is_sub_query: bool, + pub(super) propagate_filters_to_sub_query: bool, + pub(super) mask_sql: Option>, } symbol_deps! { @@ -104,18 +104,6 @@ impl DimensionSymbol { } } - pub(super) fn replace_case(&self, new_case: Case) -> Rc { - let mut new = self.clone(); - if new_case.is_single_value() { - //FIXME - Hack: we don't treat a single-element case as a multi-stage dimension - new.multi_stage = None; - } - if let DimensionKind::Case(ref c) = new.kind { - new.kind = DimensionKind::Case(c.replace_case(new_case)); - } - Rc::new(new) - } - /// Case-expression body for `DimensionKind::Case`; `None` otherwise. pub fn case(&self) -> Option<&Case> { match &self.kind { @@ -146,12 +134,6 @@ impl DimensionSymbol { &self.compiled_path } - /// Trims the join-chain prefix from `compiled_path` in place so the - /// path points only at the owning cube. - pub fn strip_join_prefix(&mut self) { - self.compiled_path = self.compiled_path.strip_join_prefix(); - } - /// Full unique identifier of the symbol: cube path, member name and /// any suffix that distinguishes one symbol from another. pub fn full_name(&self) -> String { diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/symbols/measure_kinds/mod.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/symbols/measure_kinds/mod.rs index 5f93315d88e78..8781d1f336db5 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/symbols/measure_kinds/mod.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/symbols/measure_kinds/mod.rs @@ -15,12 +15,15 @@ use std::rc::Rc; /// How a measure kind wraps its inner SQL when rendered: no wrapper /// at all, a named SQL aggregate function, or one of the distinct- -/// count special forms. +/// count special forms. `CountDistinctApproxState` is the mergeable +/// intermediate form of `CountDistinctApprox` — an HLL state instead +/// of a cardinality. pub enum AggregateWrap<'a> { PassThrough, Function(&'a str), CountDistinct, CountDistinctApprox, + CountDistinctApproxState, } /// Form of a measure's aggregation, classified from the data-model @@ -41,6 +44,12 @@ pub enum MeasureKind { /// `Count` in every respect except the final aggregate wrap. MultipliedCount(CountMeasure), Aggregated(AggregatedMeasure), + /// An `Aggregated` that renders as a mergeable intermediate state + /// (an HLL state for `count_distinct_approx`) instead of a final + /// value, because an outer aggregation merges it — a rolling + /// window over a multi-stage leaf, or a rollup read from a + /// pre-aggregation. Constructed only via [`Self::as_state`]. + AggregatedState(AggregatedMeasure), Calculated(CalculatedMeasure), Rank, } @@ -81,7 +90,7 @@ impl MeasureKind { pub fn iter_sql_calls(&self) -> Box> + '_> { match self { Self::Count(c) | Self::MultipliedCount(c) => c.iter_sql_calls(), - Self::Aggregated(a) => a.iter_sql_calls(), + Self::Aggregated(a) | Self::AggregatedState(a) => a.iter_sql_calls(), Self::Calculated(c) => c.iter_sql_calls(), Self::Rank => Box::new(std::iter::empty()), } @@ -90,7 +99,7 @@ impl MeasureKind { pub fn is_owned_by_cube(&self) -> bool { match self { Self::Count(c) | Self::MultipliedCount(c) => c.is_owned_by_cube(), - Self::Aggregated(a) => a.is_owned_by_cube(), + Self::Aggregated(a) | Self::AggregatedState(a) => a.is_owned_by_cube(), Self::Calculated(c) => c.is_owned_by_cube(), Self::Rank => false, } @@ -106,15 +115,15 @@ impl MeasureKind { pub fn is_additive(&self) -> bool { match self { Self::Count(_) | Self::MultipliedCount(_) => true, - Self::Aggregated(a) => a.agg_type().is_additive(), - _ => false, + Self::Aggregated(a) | Self::AggregatedState(a) => a.agg_type().is_additive(), + Self::Calculated(_) | Self::Rank => false, } } pub fn measure_type_str(&self) -> &str { match self { Self::Count(_) | Self::MultipliedCount(_) => "count", - Self::Aggregated(a) => a.agg_type().as_str(), + Self::Aggregated(a) | Self::AggregatedState(a) => a.agg_type().as_str(), Self::Calculated(c) => c.calc_type().as_str(), Self::Rank => "rank", } @@ -140,10 +149,14 @@ impl MeasureKind { AggregationType::CountDistinct | AggregationType::CountDistinctApprox => { matches!(new_type, "count_distinct" | "count_distinct_approx") } - _ => false, + AggregationType::NumberAgg => false, } } - _ => false, + Self::Count(_) + | Self::MultipliedCount(_) + | Self::AggregatedState(_) + | Self::Calculated(_) + | Self::Rank => false, } } @@ -162,7 +175,7 @@ impl MeasureKind { | AggregationType::CountDistinct | AggregationType::CountDistinctApprox ), - _ => false, + Self::AggregatedState(_) | Self::Calculated(_) | Self::Rank => false, } } @@ -172,7 +185,7 @@ impl MeasureKind { CountSql::Explicit(sql) => Some(sql), CountSql::Auto(_) => None, }, - Self::Aggregated(a) => a.member_sql(), + Self::Aggregated(a) | Self::AggregatedState(a) => a.member_sql(), Self::Calculated(c) => c.member_sql(), Self::Rank => None, } @@ -188,8 +201,12 @@ impl MeasureKind { AggregationType::NumberAgg => AggregateWrap::PassThrough, AggregationType::CountDistinctApprox => AggregateWrap::CountDistinctApprox, AggregationType::CountDistinct => AggregateWrap::CountDistinct, - _ => AggregateWrap::Function(a.agg_type().as_str()), + AggregationType::Sum + | AggregationType::Avg + | AggregationType::Min + | AggregationType::Max => AggregateWrap::Function(a.agg_type().as_str()), }, + Self::AggregatedState(_) => AggregateWrap::CountDistinctApproxState, Self::Count(_) => AggregateWrap::Function("count"), Self::MultipliedCount(_) => AggregateWrap::CountDistinct, Self::Rank => AggregateWrap::PassThrough, @@ -203,17 +220,23 @@ impl MeasureKind { pub fn pre_aggregate_wrap(&self) -> AggregateWrap<'_> { match self { Self::Count(_) | Self::MultipliedCount(_) => AggregateWrap::Function("sum"), + Self::AggregatedState(_) => AggregateWrap::CountDistinctApproxState, Self::Aggregated(a) => match a.agg_type() { AggregationType::CountDistinctApprox => AggregateWrap::CountDistinctApprox, AggregationType::Min => AggregateWrap::Function("min"), AggregationType::Max => AggregateWrap::Function("max"), - _ => AggregateWrap::Function("sum"), + AggregationType::Sum + | AggregationType::Avg + | AggregationType::CountDistinct + | AggregationType::NumberAgg => AggregateWrap::Function("sum"), }, Self::Calculated(c) => match c.calc_type() { CalculatedMeasureType::Number => AggregateWrap::Function("sum"), - _ => AggregateWrap::Function("max"), + CalculatedMeasureType::String + | CalculatedMeasureType::Time + | CalculatedMeasureType::Boolean => AggregateWrap::Function("max"), }, - _ => AggregateWrap::Function("sum"), + Self::Rank => AggregateWrap::Function("sum"), } } @@ -221,10 +244,12 @@ impl MeasureKind { let member_sql = self.member_sql().cloned(); let pk_sqls = match self { Self::Count(c) | Self::MultipliedCount(c) => match c.sql() { + CountSql::Explicit(_) => vec![], CountSql::Auto(pks) => pks.clone(), - _ => vec![], }, - _ => vec![], + Self::Aggregated(_) | Self::AggregatedState(_) | Self::Calculated(_) | Self::Rank => { + vec![] + } }; Self::from_type_str(new_type, member_sql, pk_sqls) } @@ -235,7 +260,29 @@ impl MeasureKind { pub fn into_multiplied(&self) -> Self { match self { Self::Count(c) => Self::MultipliedCount(c.clone()), - other => other.clone(), + Self::MultipliedCount(_) + | Self::Aggregated(_) + | Self::AggregatedState(_) + | Self::Calculated(_) + | Self::Rank => self.clone(), + } + } + + /// `Some(render form)` when the kind's aggregation can render as a + /// mergeable intermediate state for an outer aggregation to merge: + /// only `count_distinct_approx` has such a form (an HLL state). + /// `None` when the kind has no state form or is already in it. + pub fn as_state(&self) -> Option { + match self { + Self::Aggregated(a) if a.agg_type() == AggregationType::CountDistinctApprox => { + Some(Self::AggregatedState(a.clone())) + } + Self::Count(_) + | Self::MultipliedCount(_) + | Self::Aggregated(_) + | Self::AggregatedState(_) + | Self::Calculated(_) + | Self::Rank => None, } } @@ -246,8 +293,15 @@ impl MeasureKind { pub fn regular_in_multiplied(&self) -> Option { match self { Self::Count(c) if c.convertible_to_distinct() => Some(Self::MultipliedCount(c.clone())), - Self::Aggregated(a) if a.agg_type().is_distinct() => Some(self.clone()), - _ => None, + Self::Aggregated(a) | Self::AggregatedState(a) if a.agg_type().is_distinct() => { + Some(self.clone()) + } + Self::Count(_) + | Self::MultipliedCount(_) + | Self::Aggregated(_) + | Self::AggregatedState(_) + | Self::Calculated(_) + | Self::Rank => None, } } } @@ -256,7 +310,7 @@ impl SymbolDeps for MeasureKind { fn visit_deps(&self, visitor: &mut dyn DepVisitor) -> ControlFlow<()> { match self { Self::Count(c) | Self::MultipliedCount(c) => c.visit_deps(visitor), - Self::Aggregated(a) => a.visit_deps(visitor), + Self::Aggregated(a) | Self::AggregatedState(a) => a.visit_deps(visitor), Self::Calculated(c) => c.visit_deps(visitor), Self::Rank => ControlFlow::Continue(()), } @@ -265,7 +319,7 @@ impl SymbolDeps for MeasureKind { fn visit_deps_mut(&mut self, visitor: &mut dyn DepVisitorMut) -> Result<(), CubeError> { match self { Self::Count(c) | Self::MultipliedCount(c) => c.visit_deps_mut(visitor), - Self::Aggregated(a) => a.visit_deps_mut(visitor), + Self::Aggregated(a) | Self::AggregatedState(a) => a.visit_deps_mut(visitor), Self::Calculated(c) => c.visit_deps_mut(visitor), Self::Rank => Ok(()), } diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/symbols/measure_symbol.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/symbols/measure_symbol.rs index c283ce5fb47f4..76ead709fe2c6 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/symbols/measure_symbol.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/symbols/measure_symbol.rs @@ -1,6 +1,6 @@ use super::common::{Case, CompiledMemberPath, MultiStageProperties}; use super::deps::{self, symbol_deps}; -use super::measure_kinds::{CalculatedMeasure, CalculatedMeasureType, MeasureKind}; +use super::measure_kinds::MeasureKind; use super::SymbolPath; use super::{MemberSymbol, SymbolFactory}; use crate::cube_bridge::evaluator::CubeEvaluator; @@ -81,23 +81,92 @@ pub enum MeasureTimeShifts { Named(String), } +/// Render-time modifier of how the measure's value is emitted in its +/// select. +/// +/// `None` on the symbol means both "no stamping pass has decided yet" +/// and "the usual final aggregation" — the two coincide because +/// stamping only ever fills `None`, so nothing needs to express +/// "explicitly the default" to defend it against a later pass. +#[derive(Clone, Debug)] +pub enum MeasureRenderModifier { + /// Raw row-level value without the aggregation wrap, re-aggregated + /// by an enclosing select (measure subqueries, ungrouped + /// multi-stage leaves). + RawValue, + /// Final row-level output of an ungrouped query: count-like + /// measures render a not-null indicator so an outer count can sum + /// each row's contribution. + UngroupedFinal, + /// Merge of the window's partial values in a rolling-window + /// select: mergeable aggregations combine the input column + /// (`sum` for sums and counts, `min`/`max`, an HLL merge for + /// `count_distinct_approx`); the rest re-aggregate the raw rows. + RollingMerge, + /// `rank() OVER (PARTITION BY ...)` in the multi-stage select + /// that computes a rank measure. + MultiStageRank { partition: Vec> }, + /// A window aggregation `agg(agg(x)) OVER (PARTITION BY ...)` in + /// the multi-stage select whose partition is narrower than the + /// full dimension set. + MultiStageWindow { partition: Vec> }, +} + +impl MeasureRenderModifier { + /// True when the measure can take this form. The single authority + /// for the decision: stamping consults it, render nodes assert it. + pub fn applies_to(&self, measure: &MeasureSymbol) -> bool { + match self { + Self::RawValue | Self::UngroupedFinal => true, + Self::RollingMerge => measure.is_cumulative(), + Self::MultiStageRank { .. } => { + measure.is_multi_stage() && matches!(measure.kind(), MeasureKind::Rank) + } + Self::MultiStageWindow { .. } => measure.is_multi_stage() && !measure.is_calculated(), + } + } + + /// Render-side check that the measure reaching a form's node really + /// carries that form's prerequisites. + pub fn ensure_applies_to(&self, measure: &MeasureSymbol) -> Result<(), CubeError> { + if self.applies_to(measure) { + return Ok(()); + } + Err(CubeError::internal(format!( + "{} render modifier on incompatible measure {}", + self.name(), + measure.full_name() + ))) + } + + fn name(&self) -> &'static str { + match self { + Self::RawValue => "RawValue", + Self::UngroupedFinal => "UngroupedFinal", + Self::RollingMerge => "RollingMerge", + Self::MultiStageRank { .. } => "MultiStageRank", + Self::MultiStageWindow { .. } => "MultiStageWindow", + } + } +} + /// `MemberSymbol::Measure` body: Tesseract representation of a /// `measure` declared in the data model — an aggregation, count or /// calculated value the query exposes. #[derive(Clone)] pub struct MeasureSymbol { - compiled_path: CompiledMemberPath, - kind: MeasureKind, - rolling_window: Option, - multi_stage: Option, - is_reference: bool, - is_view: bool, - case: Option, - measure_filters: Vec>, - measure_drill_filters: Vec>, - measure_order_by: Vec, - is_splitted_source: bool, - mask_sql: Option>, + pub(super) compiled_path: CompiledMemberPath, + pub(super) kind: MeasureKind, + pub(super) rolling_window: Option, + pub(super) multi_stage: Option, + pub(super) is_reference: bool, + pub(super) is_view: bool, + pub(super) case: Option, + pub(super) measure_filters: Vec>, + pub(super) measure_drill_filters: Vec>, + pub(super) measure_order_by: Vec, + pub(super) mask_sql: Option>, + pub(super) render_modifier: Option, } symbol_deps! { @@ -113,7 +182,7 @@ symbol_deps! { multi_stage: skip, is_reference: skip, is_view: skip, - is_splitted_source: skip, + render_modifier: skip, } } @@ -142,115 +211,17 @@ impl MeasureSymbol { measure_drill_filters, measure_order_by, multi_stage, - is_splitted_source: false, mask_sql, + render_modifier: None, }) } - /// Returns a non-rolling copy of the symbol. A rolling-window - /// measure carries both the windowing context and the SQL of the - /// inner value it operates on; unrolling drops the window and - /// yields that inner value. Multi-stage rolling measures collapse - /// to a `Calculated` kind so they can be rendered without window- - /// function machinery. - pub fn new_unrolling(&self) -> Rc { - if self.is_rolling_window() { - let kind = if self.is_multi_stage() { - if let Some(sql) = self.kind.member_sql() { - MeasureKind::Calculated(CalculatedMeasure::new( - CalculatedMeasureType::Number, - sql.clone(), - )) - } else { - MeasureKind::Calculated(CalculatedMeasure::new_without_sql( - CalculatedMeasureType::Number, - )) - } - } else { - self.kind.clone() - }; - Rc::new(Self { - compiled_path: self.compiled_path.clone(), - kind, - rolling_window: None, - multi_stage: None, - is_reference: false, - is_view: self.is_view, - case: self.case.clone(), - measure_filters: self.measure_filters.clone(), - measure_drill_filters: self.measure_drill_filters.clone(), - measure_order_by: self.measure_order_by.clone(), - is_splitted_source: self.is_splitted_source, - mask_sql: self.mask_sql.clone(), - }) - } else { - Rc::new(self.clone()) - } - } - - /// Returns a copy of the symbol with the measure type optionally - /// replaced (subject to per-kind compatibility checks) and - /// additional measure filters merged in. - pub fn new_patched( - &self, - new_measure_type: Option, - add_filters: Vec>, - ) -> Result, CubeError> { - let result_kind = if let Some(new_measure_type) = new_measure_type { - if !self.kind.can_replace_type_with(&new_measure_type) { - return Err(CubeError::user(format!( - "Unsupported measure type replacement for {}: {} => {}", - self.compiled_path.name(), - self.kind.measure_type_str(), - new_measure_type - ))); - } - self.kind.with_new_type(&new_measure_type)? - } else { - self.kind.clone() - }; - - let mut measure_filters = self.measure_filters.clone(); - if !add_filters.is_empty() { - if !result_kind.supports_additional_filters() { - return Err(CubeError::user(format!( - "Unsupported additional filters for measure {} type {}", - self.compiled_path.name(), - result_kind.measure_type_str() - ))); - } - measure_filters.extend(add_filters); - } - Ok(Rc::new(Self { - compiled_path: self.compiled_path.clone(), - kind: result_kind, - rolling_window: self.rolling_window.clone(), - multi_stage: self.multi_stage.clone(), - is_reference: self.is_reference, - is_view: self.is_view, - case: self.case.clone(), - measure_filters, - measure_drill_filters: self.measure_drill_filters.clone(), - measure_order_by: self.measure_order_by.clone(), - is_splitted_source: self.is_splitted_source, - mask_sql: self.mask_sql.clone(), - })) - } - - pub(super) fn replace_case(&self, new_case: Case) -> Rc { - let mut new = self.clone(); - new.case = Some(new_case); - Rc::new(new) - } - pub fn compiled_path(&self) -> &CompiledMemberPath { &self.compiled_path } - /// Trims the join-chain prefix from `compiled_path` in place so - /// the path points only at the owning cube. - pub fn strip_join_prefix(&mut self) { - self.compiled_path = self.compiled_path.strip_join_prefix(); + pub fn render_modifier(&self) -> Option<&MeasureRenderModifier> { + self.render_modifier.as_ref() } /// Full unique identifier of the symbol: cube path, member name @@ -265,10 +236,6 @@ impl MeasureSymbol { self.compiled_path.alias().clone() } - pub fn is_splitted_source(&self) -> bool { - self.is_splitted_source - } - pub fn time_shift(&self) -> Option<&MeasureTimeShifts> { self.multi_stage .as_ref() @@ -315,30 +282,6 @@ impl MeasureSymbol { Box::new(result) } - /// Render form of this measure when it sits under a row-multiplying - /// join: a `count` switches to a distinct `MultipliedCount`, every - /// other kind is returned unchanged. - pub fn into_multiplied(&self) -> Rc { - self.with_kind(self.kind.into_multiplied()) - } - - /// `Some(render form)` when this measure, under a row-multiplying - /// join, can still be computed directly in the main query (it stays - /// additive there): a key-based count rolls up as a distinct - /// `MultipliedCount`, distinct aggregations are already immune. - /// `None` when it must be isolated in a multiplied subquery instead. - pub fn convert_multiplied_to_regular(&self) -> Option> { - self.kind - .regular_in_multiplied() - .map(|kind| self.with_kind(kind)) - } - - fn with_kind(&self, kind: MeasureKind) -> Rc { - let mut new = self.clone(); - new.kind = kind; - MemberSymbol::new_measure(Rc::new(new)) - } - /// True when the cube on the symbol's path is required in the /// join to read the measure from the database. Multi-stage /// measures are never owned by a cube; otherwise ownership is the diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/symbols/member_expression_symbol.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/symbols/member_expression_symbol.rs index 4eacf87aae1e5..bf52f8acd17fa 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/symbols/member_expression_symbol.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/symbols/member_expression_symbol.rs @@ -43,16 +43,16 @@ impl SymbolDeps for MemberExpressionExpression { /// full name lives in the `expr:` namespace. #[derive(Clone)] pub struct MemberExpressionSymbol { - compiled_path: CompiledMemberPath, - expression: MemberExpressionExpression, + pub(super) compiled_path: CompiledMemberPath, + pub(super) expression: MemberExpressionExpression, #[allow(dead_code)] - definition: Option, - is_reference: bool, - parenthesized: bool, + pub(super) definition: Option, + pub(super) is_reference: bool, + pub(super) parenthesized: bool, /// True when this expression materialises a `segments:` entry used as a /// selected dimension (in a pre-aggregation). Such a boolean must be /// wrapped per dialect when projected/grouped (e.g. MSSQL `BIT`). - is_segment: bool, + pub(super) is_segment: bool, } symbol_deps! { @@ -124,12 +124,6 @@ impl MemberExpressionSymbol { &self.compiled_path } - /// Trims the join-chain prefix from `compiled_path` in place so - /// the path points only at the owning cube. - pub fn strip_join_prefix(&mut self) { - self.compiled_path = self.compiled_path.strip_join_prefix(); - } - /// Full unique identifier of the symbol; lives in the `expr:` /// namespace to keep it disjoint from data-model member names. pub fn full_name(&self) -> String { diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/symbols/member_symbol.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/symbols/member_symbol.rs index c16b296347198..05c578fc324c0 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/symbols/member_symbol.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/symbols/member_symbol.rs @@ -50,6 +50,12 @@ impl Debug for MemberSymbol { } } +/// Member identity: two symbols are equal when they refer to the same +/// data-model member (same `full_name`) as the same variant. The +/// symbol *content* does not participate — derived forms of a member +/// (a time shift, a state aggregation, a stripped join prefix) compare +/// equal to the original. Do not use this equality to distinguish +/// forms; it answers "the same member?", not "the same symbol?". impl PartialEq for MemberSymbol { fn eq(&self, other: &Self) -> bool { self.full_name() == other.full_name() @@ -233,33 +239,6 @@ impl MemberSymbol { false } - /// Returns a copy of this symbol with the path reduced to just the owning cube, - /// stripping any join chain prefix (e.g. from views or cross-cube references). - pub fn with_stripped_join_prefix(&self) -> Rc { - match self { - Self::Dimension(d) => { - let mut new = (**d).clone(); - new.strip_join_prefix(); - Rc::new(Self::Dimension(Rc::new(new))) - } - Self::TimeDimension(d) => { - let mut new = (**d).clone(); - new.strip_join_prefix(); - Rc::new(Self::TimeDimension(Rc::new(new))) - } - Self::Measure(m) => { - let mut new = (**m).clone(); - new.strip_join_prefix(); - Rc::new(Self::Measure(Rc::new(new))) - } - Self::MemberExpression(e) => { - let mut new = (**e).clone(); - new.strip_join_prefix(); - Rc::new(Self::MemberExpression(Rc::new(new))) - } - } - } - /// `MemberExpression` symbols are never owned by a cube; for the other /// variants, the answer comes from the underlying member definition. pub fn owned_by_cube(&self) -> bool { diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/symbols/mod.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/symbols/mod.rs index dc5b14cba2721..af3092fd52914 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/symbols/mod.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/symbols/mod.rs @@ -9,6 +9,7 @@ mod member_expression_symbol; mod member_symbol; mod symbol_factory; mod time_dimension_symbol; +pub mod transforms; pub use common::*; pub use cube_symbol::{ @@ -21,7 +22,8 @@ pub use measure_kinds::{ CountSql, MeasureKind, }; pub use measure_symbol::{ - DimensionTimeShift, MeasureSymbol, MeasureSymbolFactory, MeasureTimeShifts, + DimensionTimeShift, MeasureRenderModifier, MeasureSymbol, MeasureSymbolFactory, + MeasureTimeShifts, }; pub use member_expression_symbol::{MemberExpressionExpression, MemberExpressionSymbol}; pub use member_symbol::MemberSymbol; diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/symbols/time_dimension_symbol.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/symbols/time_dimension_symbol.rs index f1d0fe0a4bc8b..d3cf0b05d8f0d 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/symbols/time_dimension_symbol.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/symbols/time_dimension_symbol.rs @@ -16,13 +16,18 @@ use std::rc::Rc; /// top of an existing time dimension. #[derive(Clone)] pub struct TimeDimensionSymbol { - base_symbol: Rc, - compiled_path: CompiledMemberPath, - granularity: Option, - granularity_obj: Option, - date_range: Option<(String, String)>, - alias_suffix: String, - alias_override: Option, + pub(super) base_symbol: Rc, + pub(super) compiled_path: CompiledMemberPath, + pub(super) granularity: Option, + pub(super) granularity_obj: Option, + pub(super) date_range: Option<(String, String)>, + pub(super) alias_suffix: String, + pub(super) alias_override: Option, + /// The value arrives already timezone-converted from the source + /// (a pre-aggregation rollup or an input CTE), so rendering must + /// not apply the conversion again. Composes with other forms: a + /// symbol may carry this together with future derived forms. + pub(super) tz_converted_at_source: bool, } symbol_deps! { @@ -34,6 +39,7 @@ symbol_deps! { date_range: skip, alias_suffix: skip, alias_override: skip, + tz_converted_at_source: skip, } } @@ -87,6 +93,7 @@ impl TimeDimensionSymbol { date_range, alias_suffix: name_suffix, alias_override, + tz_converted_at_source: false, }) } @@ -94,6 +101,10 @@ impl TimeDimensionSymbol { &self.base_symbol } + pub fn tz_converted_at_source(&self) -> bool { + self.tz_converted_at_source + } + pub fn granularity(&self) -> &Option { &self.granularity } @@ -137,23 +148,41 @@ impl TimeDimensionSymbol { new_granularity.clone(), )?; let date_range_tuple = self.date_range.clone(); - let result = TimeDimensionSymbol::new( + Ok(self.derive( self.base_symbol.clone(), new_granularity.clone(), new_granularity_obj.clone(), date_range_tuple, - ); - Ok(result) + None, + )) } - pub fn compiled_path(&self) -> &CompiledMemberPath { - &self.compiled_path + /// Another form of the same time dimension — a different + /// granularity of it, or the member it references. Render marks + /// describe where the value comes from, which such a re-wrap does + /// not change, so they are carried over. + fn derive( + &self, + base_symbol: Rc, + granularity: Option, + granularity_obj: Option, + date_range: Option<(String, String)>, + alias_override: Option, + ) -> Rc { + let mut new = (*Self::new_with_alias( + base_symbol, + granularity, + granularity_obj, + date_range, + alias_override, + )) + .clone(); + new.tz_converted_at_source = self.tz_converted_at_source; + Rc::new(new) } - /// Trims the join-chain prefix from `compiled_path` in place so - /// the path points only at the owning cube. - pub fn strip_join_prefix(&mut self) { - self.compiled_path = self.compiled_path.strip_join_prefix(); + pub fn compiled_path(&self) -> &CompiledMemberPath { + &self.compiled_path } /// Full unique identifier of the symbol: cube path, base @@ -246,7 +275,7 @@ impl TimeDimensionSymbol { /// range. `None` if the base is not a reference. pub fn reference_member(&self) -> Option> { if let Some(base_symbol) = self.base_symbol.clone().reference_member() { - let new_time_dim = Self::new_with_alias( + let new_time_dim = self.derive( base_symbol, self.granularity.clone(), self.granularity_obj.clone(), diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/symbols/transforms/filter_symbols.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/symbols/transforms/filter_symbols.rs new file mode 100644 index 0000000000000..5f28447d55b20 --- /dev/null +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/symbols/transforms/filter_symbols.rs @@ -0,0 +1,48 @@ +use super::super::MemberSymbol; +use crate::planner::filter::{Filter, FilterItem}; +use cubenativeutils::CubeError; +use std::rc::Rc; + +/// Rebuilds a whole filter with every member evaluator replaced by +/// `f` of itself. +pub fn map_filter_symbols(filter: Option, f: &F) -> Result, CubeError> +where + F: Fn(&Rc) -> Result, CubeError>, +{ + filter + .map(|filter| -> Result { + Ok(Filter { + items: filter + .items + .iter() + .map(|item| map_filter_item_symbols(item, f)) + .collect::, _>>()?, + }) + }) + .transpose() +} + +/// Rebuilds a filter tree with every member evaluator replaced by +/// `f` of itself. +pub fn map_filter_item_symbols(filter_item: &FilterItem, f: &F) -> Result +where + F: Fn(&Rc) -> Result, CubeError>, +{ + let mut result = filter_item.clone(); + match &mut result { + FilterItem::Group(group) => { + let mut new_group = group.as_ref().clone(); + for item in new_group.items.iter_mut() { + *item = map_filter_item_symbols(item, f)?; + } + *group = Rc::new(new_group); + } + FilterItem::Item(item) => { + *item = item.with_member_evaluator(f(&item.raw_member_evaluator())?)?; + } + FilterItem::Segment(item) => { + *item = item.with_member_evaluator(f(&item.member_evaluator())?); + } + } + Ok(result) +} diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/symbols/transforms/measures_as_state.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/symbols/transforms/measures_as_state.rs new file mode 100644 index 0000000000000..a7f9381a02f6b --- /dev/null +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/symbols/transforms/measures_as_state.rs @@ -0,0 +1,21 @@ +use super::super::MemberSymbol; +use cubenativeutils::CubeError; +use std::rc::Rc; + +/// Rebuilds the symbol tree with every measure that has a mergeable +/// state form (`count_distinct_approx` → HLL state) switched to it — +/// for queries whose aggregations feed an outer merge instead of +/// producing final values: pre-aggregation builds and multi-stage +/// leaves under an aggregating stage. +pub fn measures_as_state(symbol: &Rc) -> Result, CubeError> { + symbol.apply_recursive(&|node| { + if let MemberSymbol::Measure(measure) = node.as_ref() { + if let Some(kind) = measure.kind().as_state() { + let mut new = (**measure).clone(); + new.kind = kind; + return Ok(Rc::new(MemberSymbol::Measure(Rc::new(new)))); + } + } + Ok(node.clone()) + }) +} diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/symbols/transforms/mod.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/symbols/transforms/mod.rs new file mode 100644 index 0000000000000..7f0fd2484ae61 --- /dev/null +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/symbols/transforms/mod.rs @@ -0,0 +1,34 @@ +//! Transformations that derive new symbols from existing ones. +//! +//! Symbols are immutable values; every "modified copy" — whether a +//! node-local rebuild (unrolling a rolling window, patching a measure +//! type) or a rewrite of a whole dependency tree (static-filter +//! pruning, render-form substitution) — lives here, not on the symbol +//! types themselves. +//! +//! Rebuild style: a transform that decides per field rebuilds via a +//! full struct literal, so adding a field fails to compile until the +//! transform classifies it; a transform that stamps a single field +//! uses clone-and-mutate, so new fields flow through untouched. + +mod filter_symbols; +mod measures_as_state; +mod multiplied; +mod patch_measure; +mod render_modifier; +mod static_filter; +mod strip_join_prefix; +mod substitute; +mod tz_converted_at_source; +mod unroll_rolling; + +pub use filter_symbols::*; +pub use measures_as_state::*; +pub use multiplied::*; +pub use patch_measure::*; +pub use render_modifier::*; +pub use static_filter::*; +pub use strip_join_prefix::*; +pub use substitute::*; +pub use tz_converted_at_source::*; +pub use unroll_rolling::*; diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/symbols/transforms/multiplied.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/symbols/transforms/multiplied.rs new file mode 100644 index 0000000000000..a491e7c897284 --- /dev/null +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/symbols/transforms/multiplied.rs @@ -0,0 +1,28 @@ +use super::super::measure_kinds::MeasureKind; +use super::super::{MeasureSymbol, MemberSymbol}; +use std::rc::Rc; + +/// Render form of a measure when it sits under a row-multiplying +/// join: a `count` switches to a distinct `MultipliedCount`, every +/// other kind is returned unchanged. +pub fn into_multiplied(measure: &MeasureSymbol) -> Rc { + with_kind(measure, measure.kind.into_multiplied()) +} + +/// `Some(render form)` when the measure, under a row-multiplying +/// join, can still be computed directly in the main query (it stays +/// additive there): a key-based count rolls up as a distinct +/// `MultipliedCount`, distinct aggregations are already immune. +/// `None` when it must be isolated in a multiplied subquery instead. +pub fn regular_in_multiplied(measure: &MeasureSymbol) -> Option> { + measure + .kind + .regular_in_multiplied() + .map(|kind| with_kind(measure, kind)) +} + +fn with_kind(measure: &MeasureSymbol, kind: MeasureKind) -> Rc { + let mut new = measure.clone(); + new.kind = kind; + MemberSymbol::new_measure(Rc::new(new)) +} diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/symbols/transforms/patch_measure.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/symbols/transforms/patch_measure.rs new file mode 100644 index 0000000000000..10f72831af88c --- /dev/null +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/symbols/transforms/patch_measure.rs @@ -0,0 +1,53 @@ +use super::super::MeasureSymbol; +use crate::planner::SqlCall; +use cubenativeutils::CubeError; +use std::rc::Rc; + +/// Returns a copy of the measure with the measure type optionally +/// replaced (subject to per-kind compatibility checks) and +/// additional measure filters merged in. +pub fn patch_measure( + measure: &MeasureSymbol, + new_measure_type: Option, + add_filters: Vec>, +) -> Result, CubeError> { + let result_kind = if let Some(new_measure_type) = new_measure_type { + if !measure.kind.can_replace_type_with(&new_measure_type) { + return Err(CubeError::user(format!( + "Unsupported measure type replacement for {}: {} => {}", + measure.compiled_path.name(), + measure.kind.measure_type_str(), + new_measure_type + ))); + } + measure.kind.with_new_type(&new_measure_type)? + } else { + measure.kind.clone() + }; + + let mut measure_filters = measure.measure_filters.clone(); + if !add_filters.is_empty() { + if !result_kind.supports_additional_filters() { + return Err(CubeError::user(format!( + "Unsupported additional filters for measure {} type {}", + measure.compiled_path.name(), + result_kind.measure_type_str() + ))); + } + measure_filters.extend(add_filters); + } + Ok(Rc::new(MeasureSymbol { + compiled_path: measure.compiled_path.clone(), + kind: result_kind, + rolling_window: measure.rolling_window.clone(), + multi_stage: measure.multi_stage.clone(), + is_reference: measure.is_reference, + is_view: measure.is_view, + case: measure.case.clone(), + measure_filters, + measure_drill_filters: measure.measure_drill_filters.clone(), + measure_order_by: measure.measure_order_by.clone(), + mask_sql: measure.mask_sql.clone(), + render_modifier: measure.render_modifier.clone(), + })) +} diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/symbols/transforms/render_modifier.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/symbols/transforms/render_modifier.rs new file mode 100644 index 0000000000000..d93eca3398e52 --- /dev/null +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/symbols/transforms/render_modifier.rs @@ -0,0 +1,24 @@ +use super::super::{MeasureRenderModifier, MemberSymbol}; +use cubenativeutils::CubeError; +use std::rc::Rc; + +/// Rebuilds the symbol tree with `modifier` set on every measure the +/// form applies to that has no render modifier yet. A measure already +/// carrying a form keeps it, so stamping a tree twice is a no-op +/// rather than a silent overwrite — each select decides the form of +/// the measures it renders exactly once. +pub fn measures_render_modifier( + symbol: &Rc, + modifier: &MeasureRenderModifier, +) -> Result, CubeError> { + symbol.apply_recursive(&|node| { + if let MemberSymbol::Measure(measure) = node.as_ref() { + if measure.render_modifier().is_none() && modifier.applies_to(measure) { + let mut new = (**measure).clone(); + new.render_modifier = Some(modifier.clone()); + return Ok(Rc::new(MemberSymbol::Measure(Rc::new(new)))); + } + } + Ok(node.clone()) + }) +} diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/symbols/common/static_filter.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/symbols/transforms/static_filter.rs similarity index 62% rename from rust/cube/cubesqlplanner/cubesqlplanner/src/planner/symbols/common/static_filter.rs rename to rust/cube/cubesqlplanner/cubesqlplanner/src/planner/symbols/transforms/static_filter.rs index 26bb739f80c42..7cd890964e51b 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/symbols/common/static_filter.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/symbols/transforms/static_filter.rs @@ -1,7 +1,8 @@ -use cubenativeutils::CubeError; - +use super::super::common::Case; +use super::super::dimension_kinds::DimensionKind; +use super::super::{DimensionSymbol, MeasureSymbol, MemberSymbol}; use crate::planner::filter::{Filter, FilterGroup, FilterGroupOperator, FilterItem}; -use crate::planner::MemberSymbol; +use cubenativeutils::CubeError; use std::rc::Rc; pub fn find_value_restriction( @@ -45,14 +46,18 @@ pub fn apply_static_filter_to_symbol( MemberSymbol::Dimension(dim) => { if let Some(case) = dim.case() { if let Some(new_case) = case.apply_static_filter(filters) { - return Ok(MemberSymbol::new_dimension(dim.replace_case(new_case))); + return Ok(MemberSymbol::new_dimension(replace_dimension_case( + dim, new_case, + ))); } } } MemberSymbol::Measure(meas) => { if let Some(case) = meas.case() { if let Some(new_case) = case.apply_static_filter(filters) { - return Ok(MemberSymbol::new_measure(meas.replace_case(new_case))); + return Ok(MemberSymbol::new_measure(replace_measure_case( + meas, new_case, + ))); } } } @@ -66,27 +71,25 @@ pub fn apply_static_filter_to_filter_item( filter_item: &FilterItem, filters: &Vec, ) -> Result { - let mut result = filter_item.clone(); - match &mut result { - FilterItem::Group(group) => { - let mut new_group = group.as_ref().clone(); - for item in new_group.items.iter_mut() { - *item = apply_static_filter_to_filter_item(item, filters)?; - } - *group = Rc::new(new_group); - } - FilterItem::Item(item) => { - *item = item.with_member_evaluator(apply_static_filter_to_symbol( - &item.raw_member_evaluator(), - filters, - )?)?; - } - FilterItem::Segment(item) => { - *item = item.with_member_evaluator(apply_static_filter_to_symbol( - &item.member_evaluator(), - filters, - )?); - } + super::map_filter_item_symbols(filter_item, &|symbol| { + apply_static_filter_to_symbol(symbol, filters) + }) +} + +fn replace_measure_case(measure: &MeasureSymbol, new_case: Case) -> Rc { + let mut new = measure.clone(); + new.case = Some(new_case); + Rc::new(new) +} + +fn replace_dimension_case(dimension: &DimensionSymbol, new_case: Case) -> Rc { + let mut new = dimension.clone(); + if new_case.is_single_value() { + //FIXME - Hack: we don't treat a single-element case as a multi-stage dimension + new.multi_stage = None; + } + if let DimensionKind::Case(ref c) = new.kind { + new.kind = DimensionKind::Case(c.replace_case(new_case)); } - Ok(result) + Rc::new(new) } diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/symbols/transforms/strip_join_prefix.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/symbols/transforms/strip_join_prefix.rs new file mode 100644 index 0000000000000..b7520b2aad0ad --- /dev/null +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/symbols/transforms/strip_join_prefix.rs @@ -0,0 +1,30 @@ +use super::super::MemberSymbol; +use std::rc::Rc; + +/// Returns a copy of the symbol with the path reduced to just the +/// owning cube, stripping any join chain prefix (e.g. from views or +/// cross-cube references). +pub fn strip_join_prefix(symbol: &Rc) -> Rc { + match symbol.as_ref() { + MemberSymbol::Dimension(d) => { + let mut new = (**d).clone(); + new.compiled_path = new.compiled_path.strip_join_prefix(); + Rc::new(MemberSymbol::Dimension(Rc::new(new))) + } + MemberSymbol::TimeDimension(d) => { + let mut new = (**d).clone(); + new.compiled_path = new.compiled_path.strip_join_prefix(); + Rc::new(MemberSymbol::TimeDimension(Rc::new(new))) + } + MemberSymbol::Measure(m) => { + let mut new = (**m).clone(); + new.compiled_path = new.compiled_path.strip_join_prefix(); + Rc::new(MemberSymbol::Measure(Rc::new(new))) + } + MemberSymbol::MemberExpression(e) => { + let mut new = (**e).clone(); + new.compiled_path = new.compiled_path.strip_join_prefix(); + Rc::new(MemberSymbol::MemberExpression(Rc::new(new))) + } + } +} diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/symbols/transforms/substitute.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/symbols/transforms/substitute.rs new file mode 100644 index 0000000000000..f54236977119a --- /dev/null +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/symbols/transforms/substitute.rs @@ -0,0 +1,19 @@ +use super::super::MemberSymbol; +use cubenativeutils::CubeError; +use std::collections::HashMap; +use std::rc::Rc; + +/// Rebuilds the symbol tree with every node whose `full_name` appears +/// in `replacements` substituted by the mapped symbol; the walk then +/// descends into the substitute's dependencies, not the original's. +pub fn substitute_by_name( + symbol: &Rc, + replacements: &HashMap>, +) -> Result, CubeError> { + symbol.apply_recursive(&|node| { + Ok(replacements + .get(&node.full_name()) + .cloned() + .unwrap_or_else(|| node.clone())) + }) +} diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/symbols/transforms/tz_converted_at_source.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/symbols/transforms/tz_converted_at_source.rs new file mode 100644 index 0000000000000..4e8eb474cf491 --- /dev/null +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/symbols/transforms/tz_converted_at_source.rs @@ -0,0 +1,27 @@ +use super::super::MemberSymbol; +use cubenativeutils::CubeError; +use std::collections::HashSet; +use std::rc::Rc; + +/// Marks the time dimensions listed in `names` as timezone-converted +/// at the source, everywhere in the symbol tree: their value comes +/// converted from a pre-aggregation rollup or an input CTE, so +/// rendering must not apply the timezone conversion again. +pub fn mark_tz_converted_at_source( + symbol: &Rc, + names: &HashSet, +) -> Result, CubeError> { + if names.is_empty() { + return Ok(symbol.clone()); + } + symbol.apply_recursive(&|node| { + if let MemberSymbol::TimeDimension(td) = node.as_ref() { + if !td.tz_converted_at_source() && names.contains(&node.full_name()) { + let mut new = (**td).clone(); + new.tz_converted_at_source = true; + return Ok(Rc::new(MemberSymbol::TimeDimension(Rc::new(new)))); + } + } + Ok(node.clone()) + }) +} diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/symbols/transforms/unroll_rolling.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/symbols/transforms/unroll_rolling.rs new file mode 100644 index 0000000000000..3de109081277b --- /dev/null +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/symbols/transforms/unroll_rolling.rs @@ -0,0 +1,46 @@ +use super::super::measure_kinds::{CalculatedMeasure, CalculatedMeasureType, MeasureKind}; +use super::super::MeasureSymbol; +use std::rc::Rc; + +/// Returns a non-rolling copy of the measure. A rolling-window +/// measure carries both the windowing context and the SQL of the +/// inner value it operates on; unrolling drops the window and +/// yields that inner value. Multi-stage rolling measures collapse +/// to a `Calculated` kind so they can be rendered without window- +/// function machinery. +pub fn unroll_rolling(measure: &MeasureSymbol) -> Rc { + if !measure.is_rolling_window() { + return Rc::new(measure.clone()); + } + let kind = if measure.is_multi_stage() { + if let Some(sql) = measure.kind.member_sql() { + MeasureKind::Calculated(CalculatedMeasure::new( + CalculatedMeasureType::Number, + sql.clone(), + )) + } else { + MeasureKind::Calculated(CalculatedMeasure::new_without_sql( + CalculatedMeasureType::Number, + )) + } + } else { + measure.kind.clone() + }; + Rc::new(MeasureSymbol { + compiled_path: measure.compiled_path.clone(), + kind, + rolling_window: None, + multi_stage: None, + is_reference: false, + is_view: measure.is_view, + case: measure.case.clone(), + measure_filters: measure.measure_filters.clone(), + measure_drill_filters: measure.measure_drill_filters.clone(), + measure_order_by: measure.measure_order_by.clone(), + mask_sql: measure.mask_sql.clone(), + // Unrolling erases the properties (cumulativeness, multi-stage) + // a render modifier is stamped against, so a carried-over + // modifier would contradict the resulting symbol. + render_modifier: None, + }) +} diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/top_level_planner.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/top_level_planner.rs index 9ed93b0b05099..28eb149adc310 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/top_level_planner.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/top_level_planner.rs @@ -80,7 +80,6 @@ impl TopLevelPlanner { optimized_plan, original_sql_pre_aggregations, self.request.is_total_query(), - self.request.is_pre_aggregation_query(), )?; let sql = physical_plan.to_sql(&templates)?; diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/test_fixtures/schemas/yaml_files/common/integration_masking.yaml b/rust/cube/cubesqlplanner/cubesqlplanner/src/test_fixtures/schemas/yaml_files/common/integration_masking.yaml new file mode 100644 index 0000000000000..7f9dbe447f74c --- /dev/null +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/test_fixtures/schemas/yaml_files/common/integration_masking.yaml @@ -0,0 +1,29 @@ +cubes: + - name: orders + sql: "SELECT * FROM orders" + dimensions: + - name: id + type: number + sql: id + primary_key: true + - name: status + type: string + sql: status + - name: amount + type: number + sql: amount + measures: + - name: count + type: count + # Mask whose SQL references another member (has symbol + # dependencies) — exercises the row-level masking path. + - name: masked_total + type: sum + sql: amount + mask: + sql: "{CUBE.amount} * 0 - 1" + # Constant mask with no dependencies. + - name: masked_total_const + type: sum + sql: amount + mask: -1 diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/test_fixtures/schemas/yaml_files/common/measure_kind_tests.yaml b/rust/cube/cubesqlplanner/cubesqlplanner/src/test_fixtures/schemas/yaml_files/common/measure_kind_tests.yaml index c2fc615f6d83b..3a1221bd3d1a9 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/test_fixtures/schemas/yaml_files/common/measure_kind_tests.yaml +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/test_fixtures/schemas/yaml_files/common/measure_kind_tests.yaml @@ -47,6 +47,17 @@ cubes: rolling_window: trailing: "7 day" offset: start + - name: multi_stage_rank + type: rank + multi_stage: true + - name: multi_stage_total + type: sum + sql: "{CUBE.total}" + multi_stage: true + - name: multi_stage_calculated + type: number + sql: "{CUBE.multi_stage_total} * 2" + multi_stage: true - name: filtered_total type: sum sql: amount diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/test_fixtures/schemas/yaml_files/common/symbol_transforms.yaml b/rust/cube/cubesqlplanner/cubesqlplanner/src/test_fixtures/schemas/yaml_files/common/symbol_transforms.yaml new file mode 100644 index 0000000000000..ec9b74c611790 --- /dev/null +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/test_fixtures/schemas/yaml_files/common/symbol_transforms.yaml @@ -0,0 +1,22 @@ +cubes: + - name: events + sql: "SELECT * FROM events" + dimensions: + - name: id + type: number + sql: id + - name: created_at + type: time + sql: created_at + # A granularity reference compiles into an embedded + # TimeDimension symbol inside this dimension's tree. + - name: created_day + type: time + sql: "{CUBE.created_at.day}" + # A measure reached through a dimension's expression tree. + - name: count_label + type: string + sql: "CASE WHEN {CUBE.count} > 3 THEN 'high' ELSE 'low' END" + measures: + - name: count + type: count diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/test_fixtures/test_utils/test_context.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/test_fixtures/test_utils/test_context.rs index ff0e4f4a2e2ee..3c7d997410506 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/test_fixtures/test_utils/test_context.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/test_fixtures/test_utils/test_context.rs @@ -381,7 +381,6 @@ impl TestContext { group_by_members: Vec, ) -> Result { let mut nodes_factory = SqlNodesFactory::default(); - nodes_factory.set_ungrouped(false); nodes_factory.set_group_by_members(group_by_members.into_iter().collect()); let cube_ref_evaluator = Rc::new(nodes_factory.cube_ref_evaluator()); let visitor = SqlEvaluatorVisitor::new( diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/mod.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/mod.rs index 3d93a9e2731de..eb3176e488533 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/mod.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/mod.rs @@ -22,5 +22,6 @@ mod subquery_dimensions; mod subquery_in_join; mod time_dimensions; mod transitive_joins; +mod ungrouped_forms; mod view_default_filters; mod views; diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/snapshots/cubesqlplanner__tests__integration__ungrouped_forms__grouped_masked_measures_control.snap b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/snapshots/cubesqlplanner__tests__integration__ungrouped_forms__grouped_masked_measures_control.snap new file mode 100644 index 0000000000000..9543132bb70ec --- /dev/null +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/snapshots/cubesqlplanner__tests__integration__ungrouped_forms__grouped_masked_measures_control.snap @@ -0,0 +1,9 @@ +--- +source: cubesqlplanner/cubesqlplanner/src/tests/integration/ungrouped_forms.rs +expression: result +--- +orders__status | orders__masked_total_const +---------------+--------------------------- +cancelled | -1 +completed | -1 +pending | -1 diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/snapshots/cubesqlplanner__tests__integration__ungrouped_forms__ungrouped_conditional_dep_mask.snap b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/snapshots/cubesqlplanner__tests__integration__ungrouped_forms__ungrouped_conditional_dep_mask.snap new file mode 100644 index 0000000000000..b2db299fafba6 --- /dev/null +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/snapshots/cubesqlplanner__tests__integration__ungrouped_forms__ungrouped_conditional_dep_mask.snap @@ -0,0 +1,16 @@ +--- +source: cubesqlplanner/cubesqlplanner/src/tests/integration/ungrouped_forms.rs +assertion_line: 96 +expression: result +--- +orders__id | orders__status | orders__masked_total +-----------+----------------+--------------------- +1 | completed | -1.00 +2 | completed | -1.00 +3 | pending | -1.00 +4 | completed | -1.00 +5 | cancelled | -1.00 +6 | completed | -1.00 +7 | pending | -1.00 +8 | completed | -1.00 +9 | pending | -1.00 diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/snapshots/cubesqlplanner__tests__integration__ungrouped_forms__ungrouped_masked_measures.snap b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/snapshots/cubesqlplanner__tests__integration__ungrouped_forms__ungrouped_masked_measures.snap new file mode 100644 index 0000000000000..df10e2b221ad1 --- /dev/null +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/snapshots/cubesqlplanner__tests__integration__ungrouped_forms__ungrouped_masked_measures.snap @@ -0,0 +1,15 @@ +--- +source: cubesqlplanner/cubesqlplanner/src/tests/integration/ungrouped_forms.rs +expression: result +--- +orders__id | orders__status | orders__masked_total | orders__masked_total_const +-----------+----------------+----------------------+--------------------------- +1 | completed | -1.00 | -1 +2 | completed | -1.00 | -1 +3 | pending | -1.00 | -1 +4 | completed | -1.00 | -1 +5 | cancelled | -1.00 | -1 +6 | completed | -1.00 | -1 +7 | pending | -1.00 | -1 +8 | completed | -1.00 | -1 +9 | pending | -1.00 | -1 diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/snapshots/cubesqlplanner__tests__integration__ungrouped_forms__ungrouped_multiplied_count.snap b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/snapshots/cubesqlplanner__tests__integration__ungrouped_forms__ungrouped_multiplied_count.snap new file mode 100644 index 0000000000000..ebd339de6b1fb --- /dev/null +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/snapshots/cubesqlplanner__tests__integration__ungrouped_forms__ungrouped_multiplied_count.snap @@ -0,0 +1,24 @@ +--- +source: cubesqlplanner/cubesqlplanner/src/tests/integration/ungrouped_forms.rs +expression: result +--- +orders__status | customers__count | orders__count +---------------+------------------+-------------- +cancelled | NULL | 1 +completed | 1 | 1 +completed | 1 | 1 +completed | 1 | 1 +completed | 1 | 1 +completed | 1 | 1 +completed | 1 | 1 +completed | 1 | 1 +completed | 1 | 1 +completed | 1 | 1 +completed | 1 | 1 +completed | 1 | 1 +completed | 1 | 1 +completed | 1 | 1 +completed | 1 | 1 +completed | 1 | 1 +completed | 1 | 1 +pending | 1 | 1 diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/snapshots/cubesqlplanner__tests__integration__ungrouped_forms__ungrouped_order_by_unselected_measure.snap b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/snapshots/cubesqlplanner__tests__integration__ungrouped_forms__ungrouped_order_by_unselected_measure.snap new file mode 100644 index 0000000000000..0aa4d30a179f0 --- /dev/null +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/snapshots/cubesqlplanner__tests__integration__ungrouped_forms__ungrouped_order_by_unselected_measure.snap @@ -0,0 +1,15 @@ +--- +source: cubesqlplanner/cubesqlplanner/src/tests/integration/ungrouped_forms.rs +expression: result +--- +orders__id | orders__status | orders__count +-----------+----------------+-------------- +8 | completed | 1 +4 | completed | 1 +2 | completed | 1 +6 | completed | 1 +1 | completed | 1 +7 | pending | 1 +3 | pending | 1 +9 | pending | 1 +5 | cancelled | 1 diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/snapshots/cubesqlplanner__tests__integration__ungrouped_forms__ungrouped_rolling_count_distinct.snap b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/snapshots/cubesqlplanner__tests__integration__ungrouped_forms__ungrouped_rolling_count_distinct.snap new file mode 100644 index 0000000000000..4706da91636e2 --- /dev/null +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/snapshots/cubesqlplanner__tests__integration__ungrouped_forms__ungrouped_rolling_count_distinct.snap @@ -0,0 +1,47 @@ +--- +source: cubesqlplanner/cubesqlplanner/src/tests/integration/ungrouped_forms.rs +expression: result +--- +orders__created_at_day | orders__rolling_unique_customers_7d +-----------------------+------------------------------------ +2024-01-10 00:00:00 | 1 +2024-01-10 00:00:00 | 1 +2024-01-10 00:00:00 | 1 +2024-01-11 00:00:00 | 1 +2024-01-11 00:00:00 | 1 +2024-01-12 00:00:00 | 1 +2024-01-12 00:00:00 | 1 +2024-01-12 00:00:00 | 1 +2024-01-13 00:00:00 | 1 +2024-01-13 00:00:00 | 1 +2024-01-13 00:00:00 | 1 +2024-01-14 00:00:00 | 1 +2024-01-14 00:00:00 | 1 +2024-01-14 00:00:00 | 1 +2024-01-14 00:00:00 | 1 +2024-01-15 00:00:00 | 1 +2024-01-15 00:00:00 | 1 +2024-01-15 00:00:00 | 1 +2024-01-15 00:00:00 | 1 +2024-01-16 00:00:00 | 1 +2024-01-16 00:00:00 | 1 +2024-01-16 00:00:00 | 1 +2024-01-16 00:00:00 | 1 +2024-01-16 00:00:00 | 1 +2024-01-17 00:00:00 | 1 +2024-01-17 00:00:00 | 1 +2024-01-17 00:00:00 | 1 +2024-01-17 00:00:00 | 1 +2024-01-18 00:00:00 | 1 +2024-01-18 00:00:00 | 1 +2024-01-18 00:00:00 | 1 +2024-01-18 00:00:00 | 1 +2024-01-18 00:00:00 | 1 +2024-01-19 00:00:00 | 1 +2024-01-19 00:00:00 | 1 +2024-01-19 00:00:00 | 1 +2024-01-19 00:00:00 | 1 +2024-01-20 00:00:00 | 1 +2024-01-20 00:00:00 | 1 +2024-01-20 00:00:00 | 1 +2024-01-20 00:00:00 | 1 diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/ungrouped_forms.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/ungrouped_forms.rs new file mode 100644 index 0000000000000..7064e70bdbe1a --- /dev/null +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/ungrouped_forms.rs @@ -0,0 +1,200 @@ +//! Ungrouped rendering of measures in the contexts where it combines +//! with other machinery: masking, multiplied measures, ORDER BY of an +//! unselected measure, rolling windows over count-like measures. + +use crate::test_fixtures::cube_bridge::MockSchema; +use crate::test_fixtures::test_utils::TestContext; +use indoc::indoc; + +const BASIC_SEED: &str = "integration_basic_tables.sql"; +const JOINS_SEED: &str = "integration_joins_tables.sql"; +const ROLLING_SEED: &str = "integration_rolling_window_tables.sql"; + +fn create_basic_context() -> TestContext { + let schema = MockSchema::from_yaml_file("common/integration_basic.yaml"); + TestContext::new(schema).unwrap() +} + +fn create_joins_context() -> TestContext { + let schema = MockSchema::from_yaml_file("common/integration_joins.yaml"); + TestContext::new(schema).unwrap() +} + +fn create_rolling_context() -> TestContext { + let schema = MockSchema::from_yaml_file("common/integration_rolling_window.yaml"); + TestContext::new(schema).unwrap() +} + +fn create_masking_context() -> TestContext { + let schema = MockSchema::from_yaml_file("common/integration_masking.yaml"); + TestContext::new(schema).unwrap() +} + +const MASKED_MEMBERS: &str = indoc! {" + maskedMembers: + - member: orders.masked_total + - member: orders.masked_total_const +"}; + +// A masked measure in an ungrouped query must stay masked. The mask +// whose SQL references another member takes the row-level masking +// path; the constant mask is the control. +#[tokio::test(flavor = "multi_thread")] +async fn test_ungrouped_masked_measures() { + let ctx = create_masking_context(); + + let query = indoc! {" + measures: + - orders.masked_total + - orders.masked_total_const + dimensions: + - orders.id + - orders.status + order: + - id: orders.id + ungrouped: true + "}; + let query = format!("{}{}", query, MASKED_MEMBERS); + + ctx.build_sql(&query).unwrap(); + + if let Some(result) = ctx.try_execute_pg(&query, BASIC_SEED).await { + insta::assert_snapshot!(result); + } +} + +// A conditional mask (mask filter) on a measure whose mask SQL has +// row-level dependencies, in an ungrouped query. The dependency- +// carrying mask is applied with grouped semantics at the evaluate +// position, where the filter cannot be turned into a CASE WHEN — so +// every row must render the mask value; the original value must not +// leak through rows matching the filter. +#[tokio::test(flavor = "multi_thread")] +async fn test_ungrouped_conditional_dep_mask() { + let ctx = create_masking_context(); + + let query = indoc! {" + measures: + - orders.masked_total + dimensions: + - orders.id + - orders.status + order: + - id: orders.id + ungrouped: true + maskedMembers: + - member: orders.masked_total + filter: + member: orders.status + operator: equals + values: ['completed'] + "}; + + ctx.build_sql(query).unwrap(); + + if let Some(result) = ctx.try_execute_pg(query, BASIC_SEED).await { + insta::assert_snapshot!(result); + } +} + +// Control: a constant-masked measure in a grouped query. The mask +// whose SQL has row-level dependencies is not representable in a +// grouped select and is exercised by the ungrouped test only. +#[tokio::test(flavor = "multi_thread")] +async fn test_grouped_masked_measures_control() { + let ctx = create_masking_context(); + + let query = indoc! {" + measures: + - orders.masked_total_const + dimensions: + - orders.status + order: + - id: orders.status + "}; + let query = format!("{}{}", query, MASKED_MEMBERS); + + ctx.build_sql(&query).unwrap(); + + if let Some(result) = ctx.try_execute_pg(&query, BASIC_SEED).await { + insta::assert_snapshot!(result); + } +} + +// A multiplied measure (customers.count is multiplied by the join to +// orders) in an ungrouped query: the multiplied subquery must keep its +// distinct aggregation while the outer select emits row-level values. +#[tokio::test(flavor = "multi_thread")] +async fn test_ungrouped_multiplied_count() { + let ctx = create_joins_context(); + + let query = indoc! {" + measures: + - customers.count + - orders.count + dimensions: + - orders.status + order: + - id: orders.status + ungrouped: true + "}; + + ctx.build_sql(query).unwrap(); + + if let Some(result) = ctx.try_execute_pg(query, JOINS_SEED).await { + insta::assert_snapshot!(result); + } +} + +// ORDER BY a measure that is not in the selection list of an +// ungrouped query: the sort key must be the row-level value, not an +// aggregate. +#[tokio::test(flavor = "multi_thread")] +async fn test_ungrouped_order_by_unselected_measure() { + let ctx = create_basic_context(); + + let query = indoc! {" + measures: + - orders.count + dimensions: + - orders.id + - orders.status + order: + - id: orders.total_amount + desc: true + - id: orders.id + ungrouped: true + "}; + + ctx.build_sql(query).unwrap(); + + if let Some(result) = ctx.try_execute_pg(query, BASIC_SEED).await { + insta::assert_snapshot!(result); + } +} + +// A rolling count-distinct in an ungrouped query: the leaf emits the +// raw distinct key for the window stage to count, not a not-null +// indicator. +#[tokio::test(flavor = "multi_thread")] +async fn test_ungrouped_rolling_count_distinct() { + let ctx = create_rolling_context(); + + let query = indoc! {r#" + measures: + - orders.rolling_unique_customers_7d + time_dimensions: + - dimension: orders.created_at + granularity: day + dateRange: + - "2024-01-10" + - "2024-01-20" + ungrouped: true + "#}; + + ctx.build_sql(query).unwrap(); + + if let Some(result) = ctx.try_execute_pg(query, ROLLING_SEED).await { + insta::assert_snapshot!(result); + } +} diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/measure_symbol.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/measure_symbol.rs index 1de8d4d2869d8..1cc0390e7d30b 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/measure_symbol.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/measure_symbol.rs @@ -1,6 +1,9 @@ -//! Tests for MeasureSymbol: kind classification, new_patched, and helper methods +//! Tests for MeasureSymbol: kind classification, the patch_measure transform, and helper methods -use crate::planner::{AggregationType, CalculatedMeasureType, MeasureKind, SqlCall}; +use crate::planner::symbols::transforms::{measures_as_state, patch_measure}; +use crate::planner::{ + AggregationType, CalculatedMeasureType, MeasureKind, MeasureRenderModifier, SqlCall, +}; use crate::test_fixtures::cube_bridge::MockSchema; use crate::test_fixtures::test_utils::TestContext; use std::rc::Rc; @@ -177,10 +180,10 @@ fn measure_rolling_window_properties() { assert!(measure.is_cumulative()); } -// ─── new_patched: valid type replacements ─────────────────────────────────── +// ─── patch_measure: valid type replacements ─────────────────────────────────── #[test] -fn new_patched_sum_to_all_valid_targets() { +fn patch_sum_to_all_valid_targets() { let ctx = ctx(); let m = ctx.create_measure("test_measures.total").unwrap(); let measure = m.as_measure().unwrap(); @@ -197,8 +200,7 @@ fn new_patched_sum_to_all_valid_targets() { ), ]; for (new_type, expected_agg) in cases { - let patched = measure - .new_patched(Some(new_type.to_string()), vec![]) + let patched = patch_measure(&measure, Some(new_type.to_string()), vec![]) .unwrap_or_else(|e| panic!("sum -> {} should succeed: {}", new_type, e)); assert!( matches!(patched.kind(), MeasureKind::Aggregated(a) if a.agg_type() == expected_agg), @@ -210,14 +212,10 @@ fn new_patched_sum_to_all_valid_targets() { } #[test] -fn new_patched_avg_to_sum() { +fn patch_avg_to_sum() { let ctx = ctx(); let m = ctx.create_measure("test_measures.average").unwrap(); - let patched = m - .as_measure() - .unwrap() - .new_patched(Some("sum".to_string()), vec![]) - .unwrap(); + let patched = patch_measure(&m.as_measure().unwrap(), Some("sum".to_string()), vec![]).unwrap(); assert!(matches!( patched.kind(), MeasureKind::Aggregated(a) if a.agg_type() == AggregationType::Sum @@ -225,45 +223,45 @@ fn new_patched_avg_to_sum() { } #[test] -fn new_patched_count_distinct_family() { +fn patch_count_distinct_family() { let ctx = ctx(); let cd = ctx.create_measure("test_measures.distinct_count").unwrap(); - let patched = cd - .as_measure() - .unwrap() - .new_patched(Some("count_distinct_approx".to_string()), vec![]) - .unwrap(); + let patched = patch_measure( + &cd.as_measure().unwrap(), + Some("count_distinct_approx".to_string()), + vec![], + ) + .unwrap(); assert!(matches!( patched.kind(), MeasureKind::Aggregated(a) if a.agg_type() == AggregationType::CountDistinctApprox )); let cda = ctx.create_measure("test_measures.approx_count").unwrap(); - let patched = cda - .as_measure() - .unwrap() - .new_patched(Some("count_distinct".to_string()), vec![]) - .unwrap(); + let patched = patch_measure( + &cda.as_measure().unwrap(), + Some("count_distinct".to_string()), + vec![], + ) + .unwrap(); assert!(matches!( patched.kind(), MeasureKind::Aggregated(a) if a.agg_type() == AggregationType::CountDistinct )); } -// ─── new_patched: invalid type replacements ───────────────────────────────── +// ─── patch_measure: invalid type replacements ───────────────────────────────── #[test] -fn new_patched_sum_invalid_targets() { +fn patch_sum_invalid_targets() { let ctx = ctx(); let m = ctx.create_measure("test_measures.total").unwrap(); let measure = m.as_measure().unwrap(); for invalid in ["number", "count", "rank", "numberAgg"] { assert!( - measure - .new_patched(Some(invalid.to_string()), vec![]) - .is_err(), + patch_measure(&measure, Some(invalid.to_string()), vec![]).is_err(), "sum -> {} should fail", invalid ); @@ -271,18 +269,14 @@ fn new_patched_sum_invalid_targets() { } #[test] -fn new_patched_count_distinct_to_sum_error() { +fn patch_count_distinct_to_sum_error() { let ctx = ctx(); let m = ctx.create_measure("test_measures.distinct_count").unwrap(); - assert!(m - .as_measure() - .unwrap() - .new_patched(Some("sum".to_string()), vec![]) - .is_err()); + assert!(patch_measure(&m.as_measure().unwrap(), Some("sum".to_string()), vec![]).is_err()); } #[test] -fn new_patched_non_patchable_types() { +fn patch_non_patchable_types() { let ctx = ctx(); let non_patchable = [ @@ -293,49 +287,46 @@ fn new_patched_non_patchable_types() { for path in non_patchable { let m = ctx.create_measure(path).unwrap(); assert!( - m.as_measure() - .unwrap() - .new_patched(Some("sum".to_string()), vec![]) - .is_err(), + patch_measure(&m.as_measure().unwrap(), Some("sum".to_string()), vec![]).is_err(), "{} -> sum should fail", path ); } } -// ─── new_patched: no type change (None) ───────────────────────────────────── +// ─── patch_measure: no type change (None) ───────────────────────────────────── #[test] -fn new_patched_none_preserves_kind() { +fn patch_none_preserves_kind() { let ctx = ctx(); let m = ctx.create_measure("test_measures.total").unwrap(); - let patched = m.as_measure().unwrap().new_patched(None, vec![]).unwrap(); + let patched = patch_measure(&m.as_measure().unwrap(), None, vec![]).unwrap(); assert!(matches!( patched.kind(), MeasureKind::Aggregated(a) if a.agg_type() == AggregationType::Sum )); let m = ctx.create_measure("test_measures.cnt").unwrap(); - let patched = m.as_measure().unwrap().new_patched(None, vec![]).unwrap(); + let patched = patch_measure(&m.as_measure().unwrap(), None, vec![]).unwrap(); assert!(matches!(patched.kind(), MeasureKind::Count(_))); let m = ctx.create_measure("test_measures.calculated").unwrap(); - let patched = m.as_measure().unwrap().new_patched(None, vec![]).unwrap(); + let patched = patch_measure(&m.as_measure().unwrap(), None, vec![]).unwrap(); assert!(matches!( patched.kind(), MeasureKind::Calculated(c) if c.calc_type() == CalculatedMeasureType::Number )); let m = ctx.create_measure("test_measures.rank_measure").unwrap(); - let patched = m.as_measure().unwrap().new_patched(None, vec![]).unwrap(); + let patched = patch_measure(&m.as_measure().unwrap(), None, vec![]).unwrap(); assert!(matches!(patched.kind(), MeasureKind::Rank)); } -// ─── new_patched: filter addition validation ──────────────────────────────── +// ─── patch_measure: filter addition validation ──────────────────────────────── #[test] -fn new_patched_filters_accepted_for_aggregatable_types() { +fn patch_filters_accepted_for_aggregatable_types() { let ctx = ctx(); let filters = get_filter_calls(&ctx); @@ -348,10 +339,7 @@ fn new_patched_filters_accepted_for_aggregatable_types() { ]; for path in accept_filters { let m = ctx.create_measure(path).unwrap(); - let patched = m - .as_measure() - .unwrap() - .new_patched(None, filters.clone()) + let patched = patch_measure(&m.as_measure().unwrap(), None, filters.clone()) .unwrap_or_else(|e| panic!("{} + filters should succeed: {}", path, e)); assert!( !patched.measure_filters().is_empty(), @@ -364,17 +352,14 @@ fn new_patched_filters_accepted_for_aggregatable_types() { // Fixed: countDistinct/countDistinctApprox now correctly support filters // via MeasureKind::supports_additional_filters() pattern matching. #[test] -fn new_patched_count_distinct_accepts_filters() { +fn patch_count_distinct_accepts_filters() { let ctx = ctx(); let filters = get_filter_calls(&ctx); for path in ["test_measures.distinct_count", "test_measures.approx_count"] { let m = ctx.create_measure(path).unwrap(); assert!( - m.as_measure() - .unwrap() - .new_patched(None, filters.clone()) - .is_ok(), + patch_measure(&m.as_measure().unwrap(), None, filters.clone()).is_ok(), "{} + filters should be Ok", path ); @@ -382,7 +367,7 @@ fn new_patched_count_distinct_accepts_filters() { } #[test] -fn new_patched_filters_rejected_for_non_aggregatable_types() { +fn patch_filters_rejected_for_non_aggregatable_types() { let ctx = ctx(); let filters = get_filter_calls(&ctx); @@ -394,29 +379,27 @@ fn new_patched_filters_rejected_for_non_aggregatable_types() { for path in reject_filters { let m = ctx.create_measure(path).unwrap(); assert!( - m.as_measure() - .unwrap() - .new_patched(None, filters.clone()) - .is_err(), + patch_measure(&m.as_measure().unwrap(), None, filters.clone()).is_err(), "{} + filters should fail", path ); } } -// ─── new_patched: combined type change + filters ──────────────────────────── +// ─── patch_measure: combined type change + filters ──────────────────────────── #[test] -fn new_patched_type_change_with_filters() { +fn patch_type_change_with_filters() { let ctx = ctx(); let filters = get_filter_calls(&ctx); let m = ctx.create_measure("test_measures.total").unwrap(); - let patched = m - .as_measure() - .unwrap() - .new_patched(Some("count_distinct".to_string()), filters) - .unwrap(); + let patched = patch_measure( + &m.as_measure().unwrap(), + Some("count_distinct".to_string()), + filters, + ) + .unwrap(); assert!(matches!( patched.kind(), MeasureKind::Aggregated(a) if a.agg_type() == AggregationType::CountDistinct @@ -425,7 +408,7 @@ fn new_patched_type_change_with_filters() { } #[test] -fn new_patched_appends_to_existing_filters() { +fn patch_appends_to_existing_filters() { let ctx = ctx(); let m = ctx.create_measure("test_measures.filtered_total").unwrap(); let measure = m.as_measure().unwrap(); @@ -433,13 +416,148 @@ fn new_patched_appends_to_existing_filters() { assert!(original_count > 0); let new_filters = get_filter_calls(&ctx); - let patched = measure.new_patched(None, new_filters.clone()).unwrap(); + let patched = patch_measure(&measure, None, new_filters.clone()).unwrap(); assert_eq!( patched.measure_filters().len(), original_count + new_filters.len() ); } +// ─── State form ───────────────────────────────────────────────────────────── + +// Only an aggregation with a mergeable partial value has a state form, +// and taking it twice is not possible. +#[test] +fn state_form_exists_only_for_count_distinct_approx() { + let ctx = ctx(); + let stateful = ctx.create_measure("test_measures.approx_count").unwrap(); + let state = stateful.as_measure().unwrap().kind().as_state(); + assert!(matches!( + state, + Some(MeasureKind::AggregatedState(ref a)) if a.agg_type() == AggregationType::CountDistinctApprox + )); + assert!( + state.unwrap().as_state().is_none(), + "a state form has no state form of its own" + ); + + for path in [ + "test_measures.total", + "test_measures.cnt", + "test_measures.distinct_count", + "test_measures.calculated", + "test_measures.rank_measure", + ] { + let m = ctx.create_measure(path).unwrap(); + assert!( + m.as_measure().unwrap().kind().as_state().is_none(), + "{path} must have no state form" + ); + } +} + +// A stored state is not a value the query can filter or re-type: it is +// consumed by a merge, not by an aggregation over rows. +#[test] +fn state_form_classification() { + let ctx = ctx(); + let m = ctx.create_measure("test_measures.approx_count").unwrap(); + let state = measures_as_state(&m).unwrap(); + let state = state.as_measure().unwrap(); + + assert!(matches!(state.kind(), MeasureKind::AggregatedState(_))); + assert!(!state.kind().supports_additional_filters()); + assert!(!state.kind().can_replace_type_with("count_distinct")); + assert_eq!( + state.measure_type(), + m.as_measure().unwrap().measure_type(), + "the state form keeps the aggregation it stores" + ); +} + +// ─── Render modifiers ─────────────────────────────────────────────────────── + +// `applies_to` is the single authority for which measures may take a +// form: stamping consults it and the render nodes assert it, so both +// sides move together and only a test can pin the answers. +#[test] +fn render_modifier_row_level_forms_apply_to_every_measure() { + let ctx = ctx(); + for path in [ + "test_measures.total", + "test_measures.cnt", + "test_measures.calculated", + "test_measures.rank_measure", + "test_measures.rolling_sum", + ] { + let m = ctx.create_measure(path).unwrap(); + let measure = m.as_measure().unwrap(); + assert!(MeasureRenderModifier::RawValue.applies_to(&measure)); + assert!(MeasureRenderModifier::UngroupedFinal.applies_to(&measure)); + } +} + +#[test] +fn render_modifier_rolling_merge_requires_a_cumulative_measure() { + let ctx = ctx(); + let rolling = ctx.create_measure("test_measures.rolling_sum").unwrap(); + let plain = ctx.create_measure("test_measures.total").unwrap(); + + assert!(MeasureRenderModifier::RollingMerge.applies_to(&rolling.as_measure().unwrap())); + assert!(!MeasureRenderModifier::RollingMerge.applies_to(&plain.as_measure().unwrap())); +} + +#[test] +fn render_modifier_multi_stage_forms_require_multi_stage_measures() { + let ctx = ctx(); + let rank = MeasureRenderModifier::MultiStageRank { partition: vec![] }; + let window = MeasureRenderModifier::MultiStageWindow { partition: vec![] }; + + let ms_rank = ctx + .create_measure("test_measures.multi_stage_rank") + .unwrap(); + let ms_total = ctx + .create_measure("test_measures.multi_stage_total") + .unwrap(); + let ms_calculated = ctx + .create_measure("test_measures.multi_stage_calculated") + .unwrap(); + let plain_rank = ctx.create_measure("test_measures.rank_measure").unwrap(); + + // A rank window ranks a multi-stage rank measure and nothing else. + assert!(rank.applies_to(&ms_rank.as_measure().unwrap())); + assert!(!rank.applies_to(&ms_total.as_measure().unwrap())); + assert!(!rank.applies_to(&plain_rank.as_measure().unwrap())); + + // A value window aggregates over a partition, which a calculated + // measure has no aggregation for. + assert!(window.applies_to(&ms_total.as_measure().unwrap())); + assert!(!window.applies_to(&ms_calculated.as_measure().unwrap())); + assert!(!window.applies_to(&plain_rank.as_measure().unwrap())); +} + +#[test] +fn ensure_applies_to_names_the_rejected_form() { + let ctx = ctx(); + let plain = ctx.create_measure("test_measures.total").unwrap(); + let measure = plain.as_measure().unwrap(); + + assert!(MeasureRenderModifier::RollingMerge + .ensure_applies_to(&measure) + .is_err()); + let err = MeasureRenderModifier::RollingMerge + .ensure_applies_to(&measure) + .unwrap_err(); + assert!( + err.message.contains("RollingMerge") && err.message.contains("test_measures.total"), + "unexpected error message: {}", + err.message + ); + assert!(MeasureRenderModifier::RawValue + .ensure_applies_to(&measure) + .is_ok()); +} + // ─── Multi-stage properties + filter directive ────────────────────────────── mod multi_stage { diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/mod.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/mod.rs index b1e1b0bc5055a..5cb0406fda012 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/mod.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/mod.rs @@ -13,6 +13,7 @@ mod no_query_tools_leak; mod positional_params; mod string_measures; mod subquery_dimensions; +mod symbol_transforms; mod time_dimension_symbol; mod utils; mod view_default_filters; diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/symbol_transforms.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/symbol_transforms.rs new file mode 100644 index 0000000000000..c41bbe17ebaf6 --- /dev/null +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/symbol_transforms.rs @@ -0,0 +1,144 @@ +//! Tests for schema-level symbol transforms. + +use crate::logical_plan::transforms::{ + mark_tz_converted_at_source_in_schema, measures_render_modifier_in_schema, +}; +use crate::logical_plan::LogicalSchema; +use crate::planner::symbols::transforms; +use crate::planner::{MeasureRenderModifier, MemberSymbol}; +use crate::test_fixtures::cube_bridge::MockSchema; +use crate::test_fixtures::test_utils::TestContext; +use std::cell::RefCell; +use std::collections::HashSet; +use std::rc::Rc; + +fn ctx() -> TestContext { + let schema = MockSchema::from_yaml_file("common/symbol_transforms.yaml"); + TestContext::new(schema).unwrap() +} + +/// Mark of the time dimension `name` anywhere in the tree, `None` +/// when the tree has no such time dimension. +fn tz_mark_of(symbol: &Rc, name: &str) -> Option { + let found = RefCell::new(None); + symbol + .apply_recursive(&|node| { + if let MemberSymbol::TimeDimension(td) = node.as_ref() { + if node.full_name() == name { + *found.borrow_mut() = Some(td.tz_converted_at_source()); + } + } + Ok(node.clone()) + }) + .unwrap(); + found.into_inner() +} + +// Deriving another form of the same member keeps the mark: it says +// where the value comes from, which re-reading it at a different +// granularity does not change. +#[test] +fn tz_mark_survives_granularity_change() { + let ctx = ctx(); + let td = ctx + .create_time_dimension("events.created_at", Some("day")) + .unwrap(); + let names = HashSet::from(["events.created_at_day".to_string()]); + let marked = transforms::mark_tz_converted_at_source(&td, &names).unwrap(); + let marked = marked.as_time_dimension().unwrap(); + assert!(marked.tz_converted_at_source()); + + let regranulated = marked + .change_granularity(ctx.query_tools().clone(), Some("month".to_string())) + .unwrap(); + assert!( + regranulated.tz_converted_at_source(), + "the mark must survive a granularity change" + ); +} + +/// Whether the measure `name` anywhere in the tree carries a render +/// modifier, `None` when the tree has no such measure. +fn has_render_modifier(symbol: &Rc, name: &str) -> Option { + let found = RefCell::new(None); + symbol + .apply_recursive(&|node| { + if let MemberSymbol::Measure(m) = node.as_ref() { + if node.full_name() == name { + *found.borrow_mut() = Some(m.render_modifier().is_some()); + } + } + Ok(node.clone()) + }) + .unwrap(); + found.into_inner() +} + +// A render form belongs to the measure as rendered in the select, so a +// measure reached through a dimension's expression tree takes the same +// form as the schema's own measure entries. +#[test] +fn render_modifier_reaches_measures_embedded_in_dimensions() { + let ctx = ctx(); + let label = ctx.create_dimension("events.count_label").unwrap(); + let count = ctx.create_measure("events.count").unwrap(); + + assert_eq!( + has_render_modifier(&label, "events.count"), + Some(false), + "the dimension embeds an unstamped measure" + ); + + let schema = LogicalSchema::default() + .set_dimensions(vec![label]) + .set_measures(vec![count]) + .into_rc(); + let stamped = + measures_render_modifier_in_schema(&schema, &MeasureRenderModifier::UngroupedFinal) + .unwrap(); + + assert_eq!( + has_render_modifier(&stamped.measures[0], "events.count"), + Some(true) + ); + assert_eq!( + has_render_modifier(&stamped.dimensions[0], "events.count"), + Some(true), + "the embedded occurrence must carry the form too" + ); +} + +// The mark is a property of the member in the select, not of its +// schema position: an occurrence embedded in another member's +// expression tree (a granularity reference) must be marked the same +// way as the schema's own time-dimension entry. +#[test] +fn tz_mark_reaches_time_dimensions_embedded_in_other_members() { + let ctx = ctx(); + let td = ctx + .create_time_dimension("events.created_at", Some("day")) + .unwrap(); + let proxy = ctx.create_dimension("events.created_day").unwrap(); + + assert_eq!( + tz_mark_of(&proxy, "events.created_at_day"), + Some(false), + "the granularity reference embeds an unmarked time dimension" + ); + + let schema = LogicalSchema::default() + .set_time_dimensions(vec![td]) + .set_dimensions(vec![proxy]) + .into_rc(); + let marked = mark_tz_converted_at_source_in_schema(&schema).unwrap(); + + assert_eq!( + tz_mark_of(&marked.time_dimensions[0], "events.created_at_day"), + Some(true) + ); + assert_eq!( + tz_mark_of(&marked.dimensions[0], "events.created_at_day"), + Some(true), + "the embedded occurrence must carry the mark too" + ); +}