From 0730e31df3cb9e0f7ac0a2f73ccb41d3b20013e5 Mon Sep 17 00:00:00 2001 From: Pavel Tiunov Date: Fri, 24 Jul 2026 19:42:58 -0700 Subject: [PATCH 1/2] feat(tesseract): pre-agg matching without stored calc-group dims; collapse trivial pass-through subqueries; e2e specs for cross-cube rolling measures over rollups (#11346) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test(schema-compiler): e2e spec for cross-cube rolling measures with a shared calc-group switch Adds an integration spec covering rollup pre-aggregation matching for multi-stage case entrypoint measures dispatched by a `type: switch` dimension (calc group) across two joined fact cubes combined in a view: - when each cube declares its own switch dimension, a query combining rolling measures from both cubes pins only one switch with a filter, the other stays unresolved, and no pre-aggregation can match (anti-pattern case, asserted); - when the switch lives on a shared single-row calc-group cube joined `1 = 1` into both fact cubes, a single rolling_window filter resolves every case measure and the query is served from both rollups, verified end to end against Postgres; - rolling window and time-shift arithmetic is pinned with deterministic value assertions on a rollup-free copy of the model. Two currently unsupported paths are documented as skipped tests: querying with a granular time dimension fails with "Alias not found for partition_by dimension" (the calc-group dimension is not projected into the multi-stage window input CTE), and a date-range-only time dimension stops matching the rollups even with allow_non_strict_date_range_match. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01QxbxLWpBwunw3FypC8zfnY * test(cubejs-testing): birdbox smoke test for shared calc-group rollups in Cube Store Adds a birdbox e2e smoke test running the shared calc-group model against a real Cube server with rollups stored and queried in Cube Store (CUBEJS_ROLLUP_ONLY, Tesseract planner), covering single-cube, cross-cube, and full multi-stage rolling-window queries dispatched by a shared switch dimension. Unlike the schema-compiler integration spec (Postgres-backed rollups), this exercises the Cube Store execution path used in production. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01QxbxLWpBwunw3FypC8zfnY * test(cubejs-testing): deepen smoke query with growth percentage layer Adds a growth-percentage entrypoint (an extra multi-stage layer over the same rolling leaves) and a four-measure query matching the production query shape, to exercise the deep FullKeyAggregate plan that can overflow Cube Store's serialized-plan decode recursion limit. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01QxbxLWpBwunw3FypC8zfnY * test(cubejs-testing): skip deep multi-stage Cube Store query, document decode limit The four-measure query reproduces the Cube Store failure: the deep FullKeyAggregate plan overflows the serialized-plan protobuf decode recursion limit ("Error decoding expr as protobuf: ... recursion limit reached"). Skipped with a FIXME until Cube Store can decode plans of this depth; the three-measure query continues to run as a regression guard for the working path. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01QxbxLWpBwunw3FypC8zfnY * feat(tesseract): collapse trivial pass-through subqueries in physical plan Multi-stage planning wraps every CTE reference into a derived table of the form (SELECT * FROM cte_n AS cte_n) AS alias. Each wrapper is a semantic no-op but costs two plan nodes (projection + subquery alias) per usage in engines that inline CTE bodies at every reference, such as Cube Store / DataFusion; deep multi-stage queries can overflow Cube Store's serialized-plan decode recursion limit because of them. Adds a physical-plan optimizer pass that rewrites (SELECT * FROM AS x) AS alias into a direct
AS alias everywhere in the plan: the top-level select, CTE bodies, join sides, unions and nested subqueries. Only pure pass-through selects are collapsed: SELECT * projection with no filter, grouping, having, ordering, distinct, limit, offset or own CTEs, over a single table reference. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01QxbxLWpBwunw3FypC8zfnY * test(tesseract): cover trivial pass-through subquery collapse Asserts that multi-stage SQL no longer contains (SELECT * FROM
) wrappers, that collapsed CTE references keep their outer alias, and that executed results are unchanged. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01QxbxLWpBwunw3FypC8zfnY * test(cubejs-testing): unskip deep multi-stage Cube Store query With the trivial-subquery collapse optimizer in the Tesseract physical plan builder, the four-measure query's plan fits under Cube Store's serialized-plan decode recursion limit and executes from the rollups. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01QxbxLWpBwunw3FypC8zfnY * fix(tesseract): rustfmt optimizer, adapt Oracle alias assertions to collapsed SQL - cargo fmt on trivial_subquery.rs (CI fmt/clippy + lint failures) - OracleQuery alias assertions now accept a bare CTE reference before the alias, since the trivial-subquery collapse renders join sides as `cte_0 "q_0"` instead of `(SELECT ...) "q_0"`; the no-AS-keyword invariant they guard is unchanged - symmetric rollup assertions in the per-cube switch anti-pattern spec Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01QxbxLWpBwunw3FypC8zfnY * chore: retrigger CI after Docker Hub registry and runner network incidents Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01QxbxLWpBwunw3FypC8zfnY * feat(tesseract): match pre-aggregations without stored calc-group dimensions Calc-group (switch) dimensions are virtual enumerations: they carry no stored data, so a rollup gains nothing from materializing them — the build SQL just CROSS JOINs the value list, multiplying rollup size by the number of values while adding zero information. Until now they still had to be listed in every rollup definition purely because the pre-aggregation matcher treated them like ordinary dimensions. - dimension_matcher: a calc-group dimension now always matches (MatchState::Full) regardless of the rollup's stored dimensions. - query processor: calc-group resolution (filter-pinned value rendered as a literal, otherwise a virtual values cross-join) now also applies when the query source is a pre-aggregation, covering exactly the calc groups NOT stored in the rollup. Stored ones keep resolving to the rollup column, so existing rollups behave as before. Literal render references are re-added after the pre-aggregation render-reference reset so they survive it. - specs: new slim-rollup model + test in the shared-calc-group Postgres suite; the per-cube-switch test now asserts rollups match (calc groups no longer block matching); birdbox fixtures drop the switch dimension from both rollups and all Cube Store smoke tests pass. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01QxbxLWpBwunw3FypC8zfnY * test: pin rollup sets and demonstrate per-cube switch divergence Address review feedback: the anti-pattern block asserted only rollup selection after calc groups stopped blocking matching, so the semantic bug it warns about was no longer demonstrated. A new deterministic test on a no-pre-aggs variant of the per-cube switch model shows the unresolved share_metrics switch emitting one row per enumeration value (a duplicate P1 row with a null share change from the YTD branch) instead of the single 0.3 row the shared-switch model returns. The Cube Store smoke tests now assert the exact set of rollup tables used instead of some() containment: usedPreAggregations keys are per-usage (__usage_N suffix), so they are deduped to distinct rollup tables and compared with toEqual, catching any unexpected extra pre-aggregation being served. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01QxbxLWpBwunw3FypC8zfnY --------- Co-authored-by: Claude --- ...pre-aggregations-shared-calc-group.test.ts | 790 ++++++++++++++++++ .../test/unit/oracle-query.test.ts | 11 +- .../shared-calc-group/cube.js | 7 + .../schema/performance_view.yml | 22 + .../schema/rolling_window_dim.yml | 19 + .../shared-calc-group/schema/sales.yml | 175 ++++ .../schema/share_metrics.yml | 212 +++++ packages/cubejs-testing/package.json | 3 +- .../test/smoke-shared-calc-group.test.ts | 156 ++++ .../pre_aggregation/dimension_matcher.rs | 7 + .../cubesqlplanner/src/physical_plan/mod.rs | 2 + .../src/physical_plan/optimizers/mod.rs | 3 + .../optimizers/trivial_subquery.rs | 152 ++++ .../src/physical_plan_builder/builder.rs | 1 + .../physical_plan_builder/processors/query.rs | 31 +- .../src/tests/integration/multi_stage/mod.rs | 1 + .../multi_stage/plan_optimizers.rs | 65 ++ ...s__trivial_cte_wrappers_are_collapsed.snap | 9 + 18 files changed, 1657 insertions(+), 9 deletions(-) create mode 100644 packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations-shared-calc-group.test.ts create mode 100644 packages/cubejs-testing/birdbox-fixtures/shared-calc-group/cube.js create mode 100644 packages/cubejs-testing/birdbox-fixtures/shared-calc-group/schema/performance_view.yml create mode 100644 packages/cubejs-testing/birdbox-fixtures/shared-calc-group/schema/rolling_window_dim.yml create mode 100644 packages/cubejs-testing/birdbox-fixtures/shared-calc-group/schema/sales.yml create mode 100644 packages/cubejs-testing/birdbox-fixtures/shared-calc-group/schema/share_metrics.yml create mode 100644 packages/cubejs-testing/test/smoke-shared-calc-group.test.ts create mode 100644 rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan/optimizers/mod.rs create mode 100644 rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan/optimizers/trivial_subquery.rs create mode 100644 rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/multi_stage/plan_optimizers.rs create mode 100644 rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/multi_stage/snapshots/cubesqlplanner__tests__integration__multi_stage__plan_optimizers__trivial_cte_wrappers_are_collapsed.snap diff --git a/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations-shared-calc-group.test.ts b/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations-shared-calc-group.test.ts new file mode 100644 index 0000000000000..a1dd04612b576 --- /dev/null +++ b/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations-shared-calc-group.test.ts @@ -0,0 +1,790 @@ +import { + getEnv, +} from '@cubejs-backend/shared'; +import { PostgresQuery } from '../../../src/adapter/PostgresQuery'; +import { prepareYamlCompiler } from '../../unit/PrepareCompiler'; +import { dbRunner } from './PostgresDBRunner'; + +// Reproduces a real-world scenario: two fact cubes (sales and share +// metrics), each exposing rolling-window metrics through multi-stage `case` +// entrypoint measures dispatched by a `type: switch` dimension (calc group), +// combined in one view and accelerated with rollup pre-aggregations. +// +// When every cube declares its OWN switch dimension, a query that combines +// measures from both cubes can only pin one of the switches with a filter; +// the other stays unresolved and falls through to its cross-joined +// enumeration. Hosting the switch dimension on a shared single-row cube +// (joined with `1 = 1` into both fact cubes) makes all case entrypoints +// dispatch on the SAME dimension, so a single filter resolves every measure. +// +// Calc-group dimensions are virtual (no stored data), so rollups serve them +// whether or not they are listed in the rollup definition: the pinned value +// renders as a literal over the rollup scan, an unresolved enumeration is +// re-cross-joined — same semantics as over the raw source. + +const ROLLING_WINDOW_DIM_CUBE = ` + - name: rolling_window_dim + sql: SELECT 1 AS one + public: false + dimensions: + - name: one + sql: one + type: number + primary_key: true + public: false + + - name: rolling_window + type: switch + values: + - R3 + - YTD +`; + +const SALES_SQL = ` + SELECT 'A1' AS account, 'P1' AS product, '2017-01-15T00:00:00.000Z'::timestamptz AS sale_date, 10.0 AS amount UNION ALL + SELECT 'A1' AS account, 'P1' AS product, '2017-02-15T00:00:00.000Z'::timestamptz AS sale_date, 20.0 AS amount UNION ALL + SELECT 'A1' AS account, 'P1' AS product, '2017-03-15T00:00:00.000Z'::timestamptz AS sale_date, 30.0 AS amount UNION ALL + SELECT 'A1' AS account, 'P1' AS product, '2017-04-15T00:00:00.000Z'::timestamptz AS sale_date, 40.0 AS amount UNION ALL + SELECT 'A1' AS account, 'P1' AS product, '2017-05-15T00:00:00.000Z'::timestamptz AS sale_date, 50.0 AS amount UNION ALL + SELECT 'A1' AS account, 'P1' AS product, '2017-06-15T00:00:00.000Z'::timestamptz AS sale_date, 60.0 AS amount UNION ALL + SELECT 'A1' AS account, 'P2' AS product, '2017-01-15T00:00:00.000Z'::timestamptz AS sale_date, 5.0 AS amount UNION ALL + SELECT 'A1' AS account, 'P2' AS product, '2017-02-15T00:00:00.000Z'::timestamptz AS sale_date, 5.0 AS amount UNION ALL + SELECT 'A1' AS account, 'P2' AS product, '2017-03-15T00:00:00.000Z'::timestamptz AS sale_date, 5.0 AS amount UNION ALL + SELECT 'A1' AS account, 'P2' AS product, '2017-04-15T00:00:00.000Z'::timestamptz AS sale_date, 5.0 AS amount UNION ALL + SELECT 'A1' AS account, 'P2' AS product, '2017-05-15T00:00:00.000Z'::timestamptz AS sale_date, 5.0 AS amount UNION ALL + SELECT 'A1' AS account, 'P2' AS product, '2017-06-15T00:00:00.000Z'::timestamptz AS sale_date, 5.0 AS amount UNION ALL + SELECT 'A2' AS account, 'P1' AS product, '2017-01-15T00:00:00.000Z'::timestamptz AS sale_date, 1.0 AS amount UNION ALL + SELECT 'A2' AS account, 'P1' AS product, '2017-02-15T00:00:00.000Z'::timestamptz AS sale_date, 1.0 AS amount UNION ALL + SELECT 'A2' AS account, 'P1' AS product, '2017-03-15T00:00:00.000Z'::timestamptz AS sale_date, 1.0 AS amount UNION ALL + SELECT 'A2' AS account, 'P1' AS product, '2017-04-15T00:00:00.000Z'::timestamptz AS sale_date, 1.0 AS amount UNION ALL + SELECT 'A2' AS account, 'P1' AS product, '2017-05-15T00:00:00.000Z'::timestamptz AS sale_date, 1.0 AS amount UNION ALL + SELECT 'A2' AS account, 'P1' AS product, '2017-06-15T00:00:00.000Z'::timestamptz AS sale_date, 1.0 AS amount +`; + +const SHARE_METRICS_SQL = ` + SELECT 'A1' AS account, 'P1' AS product, 'P1' AS competitor_product, '2017-01-15T00:00:00.000Z'::timestamptz AS sale_date, 1.0 AS qty UNION ALL + SELECT 'A1' AS account, 'P1' AS product, 'P1' AS competitor_product, '2017-02-15T00:00:00.000Z'::timestamptz AS sale_date, 2.0 AS qty UNION ALL + SELECT 'A1' AS account, 'P1' AS product, 'P1' AS competitor_product, '2017-03-15T00:00:00.000Z'::timestamptz AS sale_date, 3.0 AS qty UNION ALL + SELECT 'A1' AS account, 'P1' AS product, 'P1' AS competitor_product, '2017-04-15T00:00:00.000Z'::timestamptz AS sale_date, 4.0 AS qty UNION ALL + SELECT 'A1' AS account, 'P1' AS product, 'P1' AS competitor_product, '2017-05-15T00:00:00.000Z'::timestamptz AS sale_date, 5.0 AS qty UNION ALL + SELECT 'A1' AS account, 'P1' AS product, 'P1' AS competitor_product, '2017-06-15T00:00:00.000Z'::timestamptz AS sale_date, 6.0 AS qty UNION ALL + SELECT 'A1' AS account, 'P1' AS product, 'Q' AS competitor_product, '2017-01-15T00:00:00.000Z'::timestamptz AS sale_date, 9.0 AS qty UNION ALL + SELECT 'A1' AS account, 'P1' AS product, 'Q' AS competitor_product, '2017-02-15T00:00:00.000Z'::timestamptz AS sale_date, 8.0 AS qty UNION ALL + SELECT 'A1' AS account, 'P1' AS product, 'Q' AS competitor_product, '2017-03-15T00:00:00.000Z'::timestamptz AS sale_date, 7.0 AS qty UNION ALL + SELECT 'A1' AS account, 'P1' AS product, 'Q' AS competitor_product, '2017-04-15T00:00:00.000Z'::timestamptz AS sale_date, 6.0 AS qty UNION ALL + SELECT 'A1' AS account, 'P1' AS product, 'Q' AS competitor_product, '2017-05-15T00:00:00.000Z'::timestamptz AS sale_date, 5.0 AS qty UNION ALL + SELECT 'A1' AS account, 'P1' AS product, 'Q' AS competitor_product, '2017-06-15T00:00:00.000Z'::timestamptz AS sale_date, 4.0 AS qty +`; + +// switchRef is the member the case entrypoints dispatch on: +// shared model: '{rolling_window_dim.rolling_window}', per-cube model: '{CUBE.rolling_window}'. +// rollingWindowPreAggDim is the rolling window dimension stored in rollups. +// extra is injected into the cube body (own switch dimension and/or joins). +function salesCube(switchRef: string, rollingWindowPreAggDim: string, extraJoins: string, extraDimensions: string, includePreAggs: boolean = true) { + return ` + - name: sales + sql: >${SALES_SQL} + public: false + joins:${extraJoins} + + dimensions: + - name: id + sql: "{CUBE}.account || '|' || {CUBE}.product || '|' || CAST({CUBE}.sale_date AS TEXT)" + type: string + primary_key: true + public: false + + - name: account + sql: account + type: string + + - name: product + sql: product + type: string + + - name: date + sql: sale_date + type: time +${extraDimensions} + measures: + - name: total + sql: amount + type: sum + + - name: r3_amount + sql: amount + type: sum + public: false + rolling_window: + trailing: 3 month + + - name: prev_r3_amount + multi_stage: true + sql: "{r3_amount}" + type: number + public: false + time_shift: + - interval: 3 month + type: prior + + - name: r3_amount_change + multi_stage: true + type: number + public: false + sql: "({r3_amount} - {prev_r3_amount})" + + - name: ytd_amount + sql: amount + type: sum + public: false + rolling_window: + type: to_date + granularity: year + + - name: prev_ytd_amount + multi_stage: true + sql: "{ytd_amount}" + type: number + public: false + time_shift: + - interval: 1 year + type: prior + + - name: ytd_amount_change + multi_stage: true + type: number + public: false + sql: "({ytd_amount} - {prev_ytd_amount})" + + - name: rolling_amount + multi_stage: true + type: number + case: + switch: "${switchRef}" + when: + - value: R3 + sql: "{CUBE.r3_amount}" + else: + sql: "{CUBE.ytd_amount}" + + - name: rolling_amount_change + multi_stage: true + type: number + case: + switch: "${switchRef}" + when: + - value: R3 + sql: "{CUBE.r3_amount_change}" + else: + sql: "{CUBE.ytd_amount_change}" + +${includePreAggs ? ` + pre_aggregations: + - name: perf_rolling + measures: + - total + - r3_amount + - ytd_amount + dimensions: + - account + - product${rollingWindowPreAggDim ? ` + - ${rollingWindowPreAggDim}` : ''} + time_dimension: date + granularity: month + allow_non_strict_date_range_match: true +` : ''} +`; +} + +function shareMetricsCube(switchRef: string, rollingWindowPreAggDim: string, extraJoins: string, extraDimensions: string, includePreAggs: boolean = true) { + return ` + - name: share_metrics + sql: >${SHARE_METRICS_SQL} + public: false + joins: + - name: sales + sql: "{CUBE}.account = {sales.account} AND {CUBE}.product = {sales.product} AND {CUBE}.sale_date = {sales.date}" + relationship: many_to_one${extraJoins} + + dimensions: + - name: id + sql: "{CUBE}.account || '|' || {CUBE}.product || '|' || {CUBE}.competitor_product || '|' || CAST({CUBE}.sale_date AS TEXT)" + type: string + primary_key: true + public: false + + - name: account + sql: account + type: string + + - name: product + sql: product + type: string + + - name: competitor_product + sql: competitor_product + type: string + + - name: date + sql: sale_date + type: time +${extraDimensions} + measures: + - name: numerator_r3 + type: sum + public: false + rolling_window: + trailing: 3 month + sql: "CASE WHEN {CUBE}.competitor_product = {CUBE}.product THEN {CUBE}.qty ELSE 0 END" + + - name: denominator_r3 + type: sum + public: false + rolling_window: + trailing: 3 month + sql: qty + + - name: numerator_r3_new + multi_stage: true + sql: "{numerator_r3}" + type: number + public: false + + - name: denominator_r3_new + multi_stage: true + sql: "{denominator_r3}" + type: number + public: false + + - name: share_r3 + multi_stage: true + type: number + public: false + sql: "CASE WHEN {denominator_r3_new} = 0 THEN NULL ELSE {numerator_r3_new} / {denominator_r3_new} END" + + - name: prev_numerator_r3 + multi_stage: true + sql: "{numerator_r3}" + type: number + public: false + time_shift: + - time_dimension: date + interval: 3 month + type: prior + + - name: prev_denominator_r3 + multi_stage: true + sql: "{denominator_r3}" + type: number + public: false + time_shift: + - time_dimension: date + interval: 3 month + type: prior + + - name: prev_share_r3 + multi_stage: true + type: number + public: false + sql: "CASE WHEN {prev_denominator_r3} = 0 THEN NULL ELSE {prev_numerator_r3} / {prev_denominator_r3} END" + + - name: share_change_r3 + multi_stage: true + type: number + public: false + sql: "({share_r3} - {prev_share_r3})" + + - name: numerator_ytd + type: sum + public: false + rolling_window: + type: to_date + granularity: year + sql: "CASE WHEN {CUBE}.competitor_product = {CUBE}.product THEN {CUBE}.qty ELSE 0 END" + + - name: denominator_ytd + type: sum + public: false + rolling_window: + type: to_date + granularity: year + sql: qty + + - name: numerator_ytd_new + multi_stage: true + sql: "{numerator_ytd}" + type: number + public: false + + - name: denominator_ytd_new + multi_stage: true + sql: "{denominator_ytd}" + type: number + public: false + + - name: share_ytd + multi_stage: true + type: number + public: false + sql: "CASE WHEN {denominator_ytd_new} = 0 THEN NULL ELSE {numerator_ytd_new} / {denominator_ytd_new} END" + + - name: prev_numerator_ytd + multi_stage: true + sql: "{numerator_ytd}" + type: number + public: false + time_shift: + - time_dimension: date + interval: 1 year + type: prior + + - name: prev_denominator_ytd + multi_stage: true + sql: "{denominator_ytd}" + type: number + public: false + time_shift: + - time_dimension: date + interval: 1 year + type: prior + + - name: prev_share_ytd + multi_stage: true + type: number + public: false + sql: "CASE WHEN {prev_denominator_ytd} = 0 THEN NULL ELSE {prev_numerator_ytd} / {prev_denominator_ytd} END" + + - name: share_change_ytd + multi_stage: true + type: number + public: false + sql: "({share_ytd} - {prev_share_ytd})" + + - name: rolling_share_change + multi_stage: true + type: number + case: + switch: "${switchRef}" + when: + - value: R3 + sql: "{CUBE.share_change_r3}" + else: + sql: "{CUBE.share_change_ytd}" + +${includePreAggs ? ` + pre_aggregations: + - name: perf_share + measures: + - numerator_r3 + - denominator_r3 + - numerator_ytd + - denominator_ytd + dimensions: + - account + - product + - sales.account + - sales.product${rollingWindowPreAggDim ? ` + - ${rollingWindowPreAggDim}` : ''} + time_dimension: date + granularity: month + allow_non_strict_date_range_match: true +` : ''} +`; +} + +const SHARED_JOIN = ` + - name: rolling_window_dim + sql: "1 = 1" + relationship: many_to_one +`; + +const OWN_SWITCH_DIMENSION = ` + - name: rolling_window + type: switch + values: + - R3 + - YTD +`; + +// Model where the rolling window selector lives on one shared calc-group +// cube joined into both fact cubes: one filter drives every case measure. +const sharedSwitchModel = ` +cubes: +${ROLLING_WINDOW_DIM_CUBE} +${salesCube('{rolling_window_dim.rolling_window}', 'rolling_window_dim.rolling_window', SHARED_JOIN, '')} +${shareMetricsCube('{rolling_window_dim.rolling_window}', 'rolling_window_dim.rolling_window', SHARED_JOIN, '')} +views: + - name: performance_view + cubes: + - join_path: sales + includes: + - account + - product + - date + - total + - rolling_amount + - rolling_amount_change + + - join_path: share_metrics + includes: + - rolling_share_change + + - join_path: rolling_window_dim + includes: + - rolling_window +`; + +// Model where each fact cube declares its OWN switch dimension: the view +// filter pins only the sales switch, the share_metrics one stays +// unresolved and pre-aggregations can't match. +const perCubeSwitchModel = ` +cubes: +${salesCube('{CUBE.rolling_window}', 'rolling_window', '', OWN_SWITCH_DIMENSION)} +${shareMetricsCube('{CUBE.rolling_window}', 'rolling_window', '', OWN_SWITCH_DIMENSION)} +views: + - name: performance_view + cubes: + - join_path: sales + includes: + - account + - product + - date + - total + - rolling_amount + - rolling_amount_change + - rolling_window + + - join_path: share_metrics + includes: + - rolling_share_change + - name: rolling_window + alias: share_metrics_rolling_window +`; + +// Shared calc-group switch, but the rollups do NOT store the calc-group +// dimension: the planner must still match them (the dimension is virtual) +// and resolve the filtered value as a literal over the rollup scan. +const sharedSwitchModelSlimRollups = ` +cubes: +${ROLLING_WINDOW_DIM_CUBE} +${salesCube('{rolling_window_dim.rolling_window}', '', SHARED_JOIN, '')} +${shareMetricsCube('{rolling_window_dim.rolling_window}', '', SHARED_JOIN, '')} +views: + - name: performance_view + cubes: + - join_path: sales + includes: + - account + - product + - date + - total + - rolling_amount + - rolling_amount_change + + - join_path: share_metrics + includes: + - rolling_share_change + + - join_path: rolling_window_dim + includes: + - rolling_window +`; + +// Per-cube switch model without pre-aggregations: used to demonstrate the +// semantic problem of the anti-pattern with deterministic values — the view +// filter pins only the sales switch, so share_metrics computes its case +// measure across its whole cross-joined enumeration. +const perCubeSwitchModelNoPreAggs = ` +cubes: +${salesCube('{CUBE.rolling_window}', 'rolling_window', '', OWN_SWITCH_DIMENSION, false)} +${shareMetricsCube('{CUBE.rolling_window}', 'rolling_window', '', OWN_SWITCH_DIMENSION, false)} +views: + - name: performance_view + cubes: + - join_path: sales + includes: + - account + - product + - date + - total + - rolling_amount + - rolling_amount_change + - rolling_window + + - join_path: share_metrics + includes: + - rolling_share_change + - name: rolling_window + alias: share_metrics_rolling_window + - name: date + alias: ms_date +`; + +// Same model without pre-aggregations: used to verify the plain-SQL results +// of the repro query deterministically (rolling windows anchored to the +// query date range instead of the current date). +const sharedSwitchModelNoPreAggs = ` +cubes: +${ROLLING_WINDOW_DIM_CUBE} +${salesCube('{rolling_window_dim.rolling_window}', 'rolling_window_dim.rolling_window', SHARED_JOIN, '', false)} +${shareMetricsCube('{rolling_window_dim.rolling_window}', 'rolling_window_dim.rolling_window', SHARED_JOIN, '', false)} +views: + - name: performance_view + cubes: + - join_path: sales + includes: + - account + - product + - date + - total + - rolling_amount + - rolling_amount_change + + - join_path: share_metrics + includes: + - rolling_share_change + - name: date + alias: ms_date + + - join_path: rolling_window_dim + includes: + - rolling_window +`; + +const REPRO_QUERY = { + measures: [ + 'performance_view.rolling_amount', + 'performance_view.rolling_amount_change', + 'performance_view.rolling_share_change', + ], + dimensions: ['performance_view.product'], + filters: [ + { member: 'performance_view.account', operator: 'equals', values: ['A1'] }, + { member: 'performance_view.rolling_window', operator: 'equals', values: ['R3'] }, + ], + timezone: 'UTC', + order: [{ id: 'performance_view.product' }], + preAggregationsSchema: '', + cubestoreSupportMultistage: true, +}; + +describe('PreAggregationsSharedCalcGroup', () => { + jest.setTimeout(200000); + + if (getEnv('nativeSqlPlanner')) { + describe('shared calc-group switch dimension', () => { + const { compiler, joinGraph, cubeEvaluator } = prepareYamlCompiler(sharedSwitchModel); + + it('matches rollups without a time dimension in the query', () => compiler.compile().then(() => { + const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, REPRO_QUERY); + + const preAggregationsDescription: any = query.preAggregations?.preAggregationsDescription(); + const sqlAndParams = query.buildSqlAndParams(); + const tableNames = preAggregationsDescription.map((d: any) => d.tableName); + expect(tableNames).toContain('sales_perf_rolling'); + expect(tableNames).toContain('share_metrics_perf_share'); + expect(sqlAndParams[0]).toContain('sales_perf_rolling'); + expect(sqlAndParams[0]).toContain('share_metrics_perf_share'); + + // Rolling windows without a date range are anchored to the current + // date, so only assert the query is served by the rollups end to end. + return dbRunner.evaluateQueryWithPreAggregations(query).then(res => { + expect(Array.isArray(res)).toBe(true); + }); + })); + + // FIXME: with a date-range-only time dimension (no granularity) the + // rolling windows are anchored to the range end, but the query stops + // matching the rollups even though allow_non_strict_date_range_match + // is set and the range is month-aligned. Unskip once matching + // supports it. + it.skip('matches rollups with a date-range-only time dimension', () => compiler.compile().then(() => { + const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { + ...REPRO_QUERY, + timeDimensions: [{ + dimension: 'performance_view.date', + dateRange: ['2017-01-01', '2017-06-30'], + }], + }); + + const preAggregationsDescription: any = query.preAggregations?.preAggregationsDescription(); + const tableNames = preAggregationsDescription.map((d: any) => d.tableName); + expect(tableNames).toContain('sales_perf_rolling'); + expect(tableNames).toContain('share_metrics_perf_share'); + })); + + // FIXME: querying the same measures with a granular time dimension fails + // inside the Tesseract physical plan builder with "Alias not found for + // partition_by dimension rolling_window_dim.rolling_window": the + // calc-group dimension is not projected into the multi-stage window + // input CTE. Unskip once the planner supports it. + it.skip('cross-cube rolling measures with a granular time dimension', () => compiler.compile().then(() => { + const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { + ...REPRO_QUERY, + timeDimensions: [{ + dimension: 'performance_view.date', + granularity: 'month', + dateRange: ['2017-06-01', '2017-06-30'], + }], + }); + + query.buildSqlAndParams(); + })); + }); + + describe('shared calc-group switch dimension with slim rollups', () => { + const { compiler, joinGraph, cubeEvaluator } = prepareYamlCompiler(sharedSwitchModelSlimRollups); + + // The calc-group dimension is virtual (a cross-joined enumeration), + // so rollups that don't store it must still match: the filtered value + // is rendered as a literal over the rollup scan. + it('matches rollups that do not store the calc-group dimension', () => compiler.compile().then(() => { + const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, REPRO_QUERY); + + const preAggregationsDescription: any = query.preAggregations?.preAggregationsDescription(); + const sqlAndParams = query.buildSqlAndParams(); + const tableNames = preAggregationsDescription.map((d: any) => d.tableName); + expect(tableNames).toContain('sales_perf_rolling'); + expect(tableNames).toContain('share_metrics_perf_share'); + expect(sqlAndParams[0]).toContain('sales_perf_rolling'); + expect(sqlAndParams[0]).toContain('share_metrics_perf_share'); + // The rollup build must not cross-join the calc-group values table. + const loadSql = preAggregationsDescription + .map((d: any) => d.loadSql[0]) + .join('\n'); + expect(loadSql).not.toContain('rolling_window_values'); + + return dbRunner.evaluateQueryWithPreAggregations(query).then(res => { + expect(Array.isArray(res)).toBe(true); + }); + })); + }); + + describe('shared calc-group switch dimension without rollups', () => { + const { compiler, joinGraph, cubeEvaluator } = prepareYamlCompiler(sharedSwitchModelNoPreAggs); + + it('computes deterministic sales rolling values anchored to the date range', async () => { + await dbRunner.runQueryTest({ + ...REPRO_QUERY, + measures: [ + 'performance_view.rolling_amount', + 'performance_view.rolling_amount_change', + ], + timeDimensions: [{ + dimension: 'performance_view.date', + dateRange: ['2017-01-01', '2017-06-30'], + }], + }, [ + { + performance_view__product: 'P1', + performance_view__rolling_amount: '150.0', + performance_view__rolling_amount_change: '90.0', + }, + { + performance_view__product: 'P2', + performance_view__rolling_amount: '15.0', + performance_view__rolling_amount_change: '0.0', + }, + ], + { joinGraph, cubeEvaluator, compiler }); + }); + + // The time_shift of the share-of-total measures is declared on + // share_metrics.date, so the anchor date range must be set on that + // dimension (exposed as ms_date) for the prior window to move. + it('computes deterministic share-of-total change anchored to its own date range', async () => { + await dbRunner.runQueryTest({ + ...REPRO_QUERY, + measures: [ + 'performance_view.rolling_share_change', + ], + timeDimensions: [{ + dimension: 'performance_view.ms_date', + dateRange: ['2017-01-01', '2017-06-30'], + }], + }, [ + { + performance_view__product: 'P1', + performance_view__rolling_share_change: '0.30000000000000000000', + }, + ], + { joinGraph, cubeEvaluator, compiler }); + }); + }); + + describe('per-cube switch dimensions (anti-pattern)', () => { + const { compiler, joinGraph, cubeEvaluator } = prepareYamlCompiler(perCubeSwitchModel); + const noPreAggs = prepareYamlCompiler(perCubeSwitchModelNoPreAggs); + + // This is what makes per-cube switches an anti-pattern: the view + // filter pins only the sales switch, while share_metrics falls + // through to its own cross-joined enumeration, so every enumeration + // value emits its own result row even though the switch dimension is + // not projected. The same query against the shared-switch model + // returns a single row of 0.3 (see the deterministic test above); + // here a duplicate P1 row from the YTD branch (null: its 1-year prior + // window has no data) leaks into the result. + it('produces a divergent share change because one switch stays unresolved', async () => { + await dbRunner.runQueryTest({ + ...REPRO_QUERY, + measures: [ + 'performance_view.rolling_share_change', + ], + timeDimensions: [{ + dimension: 'performance_view.ms_date', + dateRange: ['2017-01-01', '2017-06-30'], + }], + }, [ + { + performance_view__product: 'P1', + performance_view__rolling_share_change: '0.30000000000000000000', + }, + { + performance_view__product: 'P1', + performance_view__rolling_share_change: null, + }, + ], + { joinGraph: noPreAggs.joinGraph, cubeEvaluator: noPreAggs.cubeEvaluator, compiler: noPreAggs.compiler }); + }); + + // Calc-group dimensions are virtual, so rollups serve this query too. + // The anti-pattern remains semantic: the view filter pins only the + // sales switch, while the share_metrics switch stays unresolved and + // falls through to its cross-joined enumeration — same behavior as + // over the raw source, just accelerated. + it('cross-cube rolling measures still match rollups with per-cube switches', () => compiler.compile().then(() => { + const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, REPRO_QUERY); + + const preAggregationsDescription: any = query.preAggregations?.preAggregationsDescription(); + const sqlAndParams = query.buildSqlAndParams(); + const tableNames = preAggregationsDescription.map((d: any) => d.tableName); + expect(tableNames).toContain('sales_perf_rolling'); + expect(tableNames).toContain('share_metrics_perf_share'); + expect(sqlAndParams[0]).toContain('sales_perf_rolling'); + expect(sqlAndParams[0]).toContain('share_metrics_perf_share'); + })); + + it('single-cube rolling measures still match their own rollup', () => compiler.compile().then(() => { + const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { + ...REPRO_QUERY, + measures: [ + 'performance_view.rolling_amount', + 'performance_view.rolling_amount_change', + ], + }); + + const preAggregationsDescription: any = query.preAggregations?.preAggregationsDescription(); + const sqlAndParams = query.buildSqlAndParams(); + const tableNames = preAggregationsDescription.map((d: any) => d.tableName); + expect(tableNames).toContain('sales_perf_rolling'); + expect(tableNames).not.toContain('share_metrics_perf_share'); + expect(sqlAndParams[0]).toContain('sales_perf_rolling'); + expect(sqlAndParams[0]).not.toContain('share_metrics_perf_share'); + })); + }); + } else { + it.skip('shared calc-group pre-aggregations', () => { + // Works only with the Tesseract SQL planner + }); + } +}); diff --git a/packages/cubejs-schema-compiler/test/unit/oracle-query.test.ts b/packages/cubejs-schema-compiler/test/unit/oracle-query.test.ts index e8c63d596ca67..58bf4f2696d70 100644 --- a/packages/cubejs-schema-compiler/test/unit/oracle-query.test.ts +++ b/packages/cubejs-schema-compiler/test/unit/oracle-query.test.ts @@ -159,8 +159,9 @@ describe('OracleQuery', () => { expect(sql).not.toMatch(/\bas\s+q_\d+/); // Should have q_0 alias (with space around it, indicating no AS). - // The native planner quotes the alias ("q_0"). - expect(sql).toMatch(/\)\s+"?q_0"?/); + // The native planner quotes the alias ("q_0") and may reference the + // source as a bare CTE name after trivial-subquery collapse. + expect(sql).toMatch(/[)\w]\s+"?q_0"?/); }); it('does not use AS keyword with multiple rolling window measures (YoY scenario)', async () => { @@ -191,7 +192,7 @@ describe('OracleQuery', () => { expect(sql).not.toMatch(/\bas\s+q_\d+/); // Verify pattern is ) q_X not ) AS q_X (the native planner quotes the alias) - expect(sql).toMatch(/\)\s+"?q_\d+"?/); + expect(sql).toMatch(/[)\w]\s+"?q_\d+"?/); }); it('does not use AS keyword in INNER JOIN subqueries', async () => { @@ -280,8 +281,8 @@ describe('OracleQuery', () => { // Should have multiple subquery aliases without AS (the native planner // quotes the aliases and joins subqueries explicitly rather than with commas). - expect(sql).toMatch(/\)\s+"?q_0"?/); - expect(sql).toMatch(/\)\s+"?q_1"?/); + expect(sql).toMatch(/[)\w]\s+"?q_0"?/); + expect(sql).toMatch(/[)\w]\s+"?q_1"?/); // Should NOT have AS before q_ aliases expect(sql).not.toMatch(/\bAS\s+q_\d+/i); diff --git a/packages/cubejs-testing/birdbox-fixtures/shared-calc-group/cube.js b/packages/cubejs-testing/birdbox-fixtures/shared-calc-group/cube.js new file mode 100644 index 0000000000000..628dde65fb9c2 --- /dev/null +++ b/packages/cubejs-testing/birdbox-fixtures/shared-calc-group/cube.js @@ -0,0 +1,7 @@ +module.exports = { + orchestratorOptions: { + preAggregationsOptions: { + externalRefresh: false, + }, + }, +}; diff --git a/packages/cubejs-testing/birdbox-fixtures/shared-calc-group/schema/performance_view.yml b/packages/cubejs-testing/birdbox-fixtures/shared-calc-group/schema/performance_view.yml new file mode 100644 index 0000000000000..e3cbcf888c905 --- /dev/null +++ b/packages/cubejs-testing/birdbox-fixtures/shared-calc-group/schema/performance_view.yml @@ -0,0 +1,22 @@ +views: + - name: performance_view + cubes: + - join_path: sales + includes: + - account + - product + - date + - total + - rolling_amount + - rolling_amount_change + - rolling_amount_growth_pct + + - join_path: share_metrics + includes: + - rolling_share_change + - name: date + alias: ms_date + + - join_path: rolling_window_dim + includes: + - rolling_window diff --git a/packages/cubejs-testing/birdbox-fixtures/shared-calc-group/schema/rolling_window_dim.yml b/packages/cubejs-testing/birdbox-fixtures/shared-calc-group/schema/rolling_window_dim.yml new file mode 100644 index 0000000000000..821fd3427ad0b --- /dev/null +++ b/packages/cubejs-testing/birdbox-fixtures/shared-calc-group/schema/rolling_window_dim.yml @@ -0,0 +1,19 @@ +# Shared rolling-window selector (calc group): a single-row cube hosting a +# `type: switch` dimension that case entrypoint measures in several fact +# cubes dispatch on, so one filter resolves the window for all of them. +cubes: + - name: rolling_window_dim + sql: SELECT 1 AS one + public: false + dimensions: + - name: one + sql: one + type: number + primary_key: true + public: false + + - name: rolling_window + type: switch + values: + - R3 + - YTD diff --git a/packages/cubejs-testing/birdbox-fixtures/shared-calc-group/schema/sales.yml b/packages/cubejs-testing/birdbox-fixtures/shared-calc-group/schema/sales.yml new file mode 100644 index 0000000000000..71726b0818c11 --- /dev/null +++ b/packages/cubejs-testing/birdbox-fixtures/shared-calc-group/schema/sales.yml @@ -0,0 +1,175 @@ +cubes: + - name: sales + sql: > + SELECT 'A1' AS account, 'P1' AS product, '2017-01-15T00:00:00.000Z'::timestamptz AS sale_date, 10.0 AS amount UNION ALL + SELECT 'A1' AS account, 'P1' AS product, '2017-02-15T00:00:00.000Z'::timestamptz AS sale_date, 20.0 AS amount UNION ALL + SELECT 'A1' AS account, 'P1' AS product, '2017-03-15T00:00:00.000Z'::timestamptz AS sale_date, 30.0 AS amount UNION ALL + SELECT 'A1' AS account, 'P1' AS product, '2017-04-15T00:00:00.000Z'::timestamptz AS sale_date, 40.0 AS amount UNION ALL + SELECT 'A1' AS account, 'P1' AS product, '2017-05-15T00:00:00.000Z'::timestamptz AS sale_date, 50.0 AS amount UNION ALL + SELECT 'A1' AS account, 'P1' AS product, '2017-06-15T00:00:00.000Z'::timestamptz AS sale_date, 60.0 AS amount UNION ALL + SELECT 'A1' AS account, 'P2' AS product, '2017-01-15T00:00:00.000Z'::timestamptz AS sale_date, 5.0 AS amount UNION ALL + SELECT 'A1' AS account, 'P2' AS product, '2017-02-15T00:00:00.000Z'::timestamptz AS sale_date, 5.0 AS amount UNION ALL + SELECT 'A1' AS account, 'P2' AS product, '2017-03-15T00:00:00.000Z'::timestamptz AS sale_date, 5.0 AS amount UNION ALL + SELECT 'A1' AS account, 'P2' AS product, '2017-04-15T00:00:00.000Z'::timestamptz AS sale_date, 5.0 AS amount UNION ALL + SELECT 'A1' AS account, 'P2' AS product, '2017-05-15T00:00:00.000Z'::timestamptz AS sale_date, 5.0 AS amount UNION ALL + SELECT 'A1' AS account, 'P2' AS product, '2017-06-15T00:00:00.000Z'::timestamptz AS sale_date, 5.0 AS amount UNION ALL + SELECT 'A2' AS account, 'P1' AS product, '2017-01-15T00:00:00.000Z'::timestamptz AS sale_date, 1.0 AS amount UNION ALL + SELECT 'A2' AS account, 'P1' AS product, '2017-02-15T00:00:00.000Z'::timestamptz AS sale_date, 1.0 AS amount UNION ALL + SELECT 'A2' AS account, 'P1' AS product, '2017-03-15T00:00:00.000Z'::timestamptz AS sale_date, 1.0 AS amount UNION ALL + SELECT 'A2' AS account, 'P1' AS product, '2017-04-15T00:00:00.000Z'::timestamptz AS sale_date, 1.0 AS amount UNION ALL + SELECT 'A2' AS account, 'P1' AS product, '2017-05-15T00:00:00.000Z'::timestamptz AS sale_date, 1.0 AS amount UNION ALL + SELECT 'A2' AS account, 'P1' AS product, '2017-06-15T00:00:00.000Z'::timestamptz AS sale_date, 1.0 AS amount + public: false + + joins: + # Virtual edge to the shared rolling-window calc group cube (single + # row, 1 = 1) so the join graph connects; the switch dimension itself + # is cross-joined as a virtual values table, not via this SQL. + - name: rolling_window_dim + sql: "1 = 1" + relationship: many_to_one + + dimensions: + - name: id + sql: "{CUBE}.account || '|' || {CUBE}.product || '|' || CAST({CUBE}.sale_date AS TEXT)" + type: string + primary_key: true + public: false + + - name: account + sql: account + type: string + + - name: product + sql: product + type: string + + - name: date + sql: sale_date + type: time + + measures: + - name: total + sql: amount + type: sum + + - name: r3_amount + sql: amount + type: sum + public: false + rolling_window: + trailing: 3 month + + - name: prev_r3_amount + multi_stage: true + sql: "{r3_amount}" + type: number + public: false + time_shift: + - interval: 3 month + type: prior + + - name: r3_amount_change + multi_stage: true + type: number + public: false + sql: "({r3_amount} - {prev_r3_amount})" + + - name: ytd_amount + sql: amount + type: sum + public: false + rolling_window: + type: to_date + granularity: year + + - name: prev_ytd_amount + multi_stage: true + sql: "{ytd_amount}" + type: number + public: false + time_shift: + - interval: 1 year + type: prior + + - name: ytd_amount_change + multi_stage: true + type: number + public: false + sql: "({ytd_amount} - {prev_ytd_amount})" + + - name: r3_amount_growth_pct + multi_stage: true + type: number + public: false + format: percent + sql: > + CASE + WHEN {r3_amount} IS NULL OR {prev_r3_amount} IS NULL OR {prev_r3_amount} = 0 + THEN NULL + ELSE ({r3_amount} - {prev_r3_amount}) / {prev_r3_amount} + END + + - name: ytd_amount_growth_pct + multi_stage: true + type: number + public: false + format: percent + sql: > + CASE + WHEN {ytd_amount} IS NULL OR {prev_ytd_amount} IS NULL OR {prev_ytd_amount} = 0 + THEN NULL + ELSE ({ytd_amount} - {prev_ytd_amount}) / {prev_ytd_amount} + END + + - name: rolling_amount + multi_stage: true + type: number + case: + switch: "{rolling_window_dim.rolling_window}" + when: + - value: R3 + sql: "{CUBE.r3_amount}" + else: + sql: "{CUBE.ytd_amount}" + + - name: rolling_amount_change + multi_stage: true + type: number + case: + switch: "{rolling_window_dim.rolling_window}" + when: + - value: R3 + sql: "{CUBE.r3_amount_change}" + else: + sql: "{CUBE.ytd_amount_change}" + + - name: rolling_amount_growth_pct + multi_stage: true + type: number + format: percent + case: + switch: "{rolling_window_dim.rolling_window}" + when: + - value: R3 + sql: "{CUBE.r3_amount_growth_pct}" + else: + sql: "{CUBE.ytd_amount_growth_pct}" + + pre_aggregations: + - name: perf_rolling + measures: + - total + - r3_amount + - ytd_amount + # The rolling_window calc group is virtual and resolved at query + # time, so it is intentionally NOT stored in the rollup. + dimensions: + - account + - product + time_dimension: date + granularity: month + allow_non_strict_date_range_match: true + scheduled_refresh: false + refresh_key: + every: 1 hour diff --git a/packages/cubejs-testing/birdbox-fixtures/shared-calc-group/schema/share_metrics.yml b/packages/cubejs-testing/birdbox-fixtures/shared-calc-group/schema/share_metrics.yml new file mode 100644 index 0000000000000..8d71a203cf578 --- /dev/null +++ b/packages/cubejs-testing/birdbox-fixtures/shared-calc-group/schema/share_metrics.yml @@ -0,0 +1,212 @@ +cubes: + - name: share_metrics + sql: > + SELECT 'A1' AS account, 'P1' AS product, 'P1' AS competitor_product, '2017-01-15T00:00:00.000Z'::timestamptz AS sale_date, 1.0 AS qty UNION ALL + SELECT 'A1' AS account, 'P1' AS product, 'P1' AS competitor_product, '2017-02-15T00:00:00.000Z'::timestamptz AS sale_date, 2.0 AS qty UNION ALL + SELECT 'A1' AS account, 'P1' AS product, 'P1' AS competitor_product, '2017-03-15T00:00:00.000Z'::timestamptz AS sale_date, 3.0 AS qty UNION ALL + SELECT 'A1' AS account, 'P1' AS product, 'P1' AS competitor_product, '2017-04-15T00:00:00.000Z'::timestamptz AS sale_date, 4.0 AS qty UNION ALL + SELECT 'A1' AS account, 'P1' AS product, 'P1' AS competitor_product, '2017-05-15T00:00:00.000Z'::timestamptz AS sale_date, 5.0 AS qty UNION ALL + SELECT 'A1' AS account, 'P1' AS product, 'P1' AS competitor_product, '2017-06-15T00:00:00.000Z'::timestamptz AS sale_date, 6.0 AS qty UNION ALL + SELECT 'A1' AS account, 'P1' AS product, 'Q' AS competitor_product, '2017-01-15T00:00:00.000Z'::timestamptz AS sale_date, 9.0 AS qty UNION ALL + SELECT 'A1' AS account, 'P1' AS product, 'Q' AS competitor_product, '2017-02-15T00:00:00.000Z'::timestamptz AS sale_date, 8.0 AS qty UNION ALL + SELECT 'A1' AS account, 'P1' AS product, 'Q' AS competitor_product, '2017-03-15T00:00:00.000Z'::timestamptz AS sale_date, 7.0 AS qty UNION ALL + SELECT 'A1' AS account, 'P1' AS product, 'Q' AS competitor_product, '2017-04-15T00:00:00.000Z'::timestamptz AS sale_date, 6.0 AS qty UNION ALL + SELECT 'A1' AS account, 'P1' AS product, 'Q' AS competitor_product, '2017-05-15T00:00:00.000Z'::timestamptz AS sale_date, 5.0 AS qty UNION ALL + SELECT 'A1' AS account, 'P1' AS product, 'Q' AS competitor_product, '2017-06-15T00:00:00.000Z'::timestamptz AS sale_date, 4.0 AS qty + public: false + + joins: + # Virtual edge to the shared rolling-window calc group cube (single + # row, 1 = 1) so the join graph connects; the switch dimension itself + # is cross-joined as a virtual values table, not via this SQL. + - name: rolling_window_dim + sql: "1 = 1" + relationship: many_to_one + + - name: sales + sql: "{CUBE}.account = {sales.account} AND {CUBE}.product = {sales.product} AND {CUBE}.sale_date = {sales.date}" + relationship: many_to_one + + dimensions: + - name: id + sql: "{CUBE}.account || '|' || {CUBE}.product || '|' || {CUBE}.competitor_product || '|' || CAST({CUBE}.sale_date AS TEXT)" + type: string + primary_key: true + public: false + + - name: account + sql: account + type: string + + - name: product + sql: product + type: string + + - name: competitor_product + sql: competitor_product + type: string + + - name: date + sql: sale_date + type: time + + measures: + - name: numerator_r3 + type: sum + public: false + rolling_window: + trailing: 3 month + sql: "CASE WHEN {CUBE}.competitor_product = {CUBE}.product THEN {CUBE}.qty ELSE 0 END" + + - name: denominator_r3 + type: sum + public: false + rolling_window: + trailing: 3 month + sql: qty + + - name: numerator_r3_new + multi_stage: true + sql: "{numerator_r3}" + type: number + public: false + + - name: denominator_r3_new + multi_stage: true + sql: "{denominator_r3}" + type: number + public: false + + - name: share_r3 + multi_stage: true + type: number + public: false + sql: "CASE WHEN {denominator_r3_new} = 0 THEN NULL ELSE {numerator_r3_new} / {denominator_r3_new} END" + + - name: prev_numerator_r3 + multi_stage: true + sql: "{numerator_r3}" + type: number + public: false + time_shift: + - time_dimension: date + interval: 3 month + type: prior + + - name: prev_denominator_r3 + multi_stage: true + sql: "{denominator_r3}" + type: number + public: false + time_shift: + - time_dimension: date + interval: 3 month + type: prior + + - name: prev_share_r3 + multi_stage: true + type: number + public: false + sql: "CASE WHEN {prev_denominator_r3} = 0 THEN NULL ELSE {prev_numerator_r3} / {prev_denominator_r3} END" + + - name: share_change_r3 + multi_stage: true + type: number + public: false + sql: "({share_r3} - {prev_share_r3})" + + - name: numerator_ytd + type: sum + public: false + rolling_window: + type: to_date + granularity: year + sql: "CASE WHEN {CUBE}.competitor_product = {CUBE}.product THEN {CUBE}.qty ELSE 0 END" + + - name: denominator_ytd + type: sum + public: false + rolling_window: + type: to_date + granularity: year + sql: qty + + - name: numerator_ytd_new + multi_stage: true + sql: "{numerator_ytd}" + type: number + public: false + + - name: denominator_ytd_new + multi_stage: true + sql: "{denominator_ytd}" + type: number + public: false + + - name: share_ytd + multi_stage: true + type: number + public: false + sql: "CASE WHEN {denominator_ytd_new} = 0 THEN NULL ELSE {numerator_ytd_new} / {denominator_ytd_new} END" + + - name: prev_numerator_ytd + multi_stage: true + sql: "{numerator_ytd}" + type: number + public: false + time_shift: + - time_dimension: date + interval: 1 year + type: prior + + - name: prev_denominator_ytd + multi_stage: true + sql: "{denominator_ytd}" + type: number + public: false + time_shift: + - time_dimension: date + interval: 1 year + type: prior + + - name: prev_share_ytd + multi_stage: true + type: number + public: false + sql: "CASE WHEN {prev_denominator_ytd} = 0 THEN NULL ELSE {prev_numerator_ytd} / {prev_denominator_ytd} END" + + - name: share_change_ytd + multi_stage: true + type: number + public: false + sql: "({share_ytd} - {prev_share_ytd})" + + - name: rolling_share_change + multi_stage: true + type: number + case: + switch: "{rolling_window_dim.rolling_window}" + when: + - value: R3 + sql: "{CUBE.share_change_r3}" + else: + sql: "{CUBE.share_change_ytd}" + + pre_aggregations: + - name: perf_share + measures: + - numerator_r3 + - denominator_r3 + - numerator_ytd + - denominator_ytd + dimensions: + - account + - product + - sales.account + - sales.product + time_dimension: date + granularity: month + allow_non_strict_date_range_match: true + scheduled_refresh: false + refresh_key: + every: 1 hour diff --git a/packages/cubejs-testing/package.json b/packages/cubejs-testing/package.json index b10a25401d7d6..60e3dc6a00fc4 100644 --- a/packages/cubejs-testing/package.json +++ b/packages/cubejs-testing/package.json @@ -83,7 +83,8 @@ "smoke:mssql:snapshot": "jest --verbose --forceExit --updateSnapshot -i dist/test/smoke-mssql.test.js", "smoke:duckdb": "jest --verbose -i dist/test/smoke-duckdb.test.js", "smoke:duckdb:snapshot": "jest --verbose --updateSnapshot -i dist/test/smoke-duckdb.test.js", - "smoke:view-groups": "jest --verbose --forceExit -i dist/test/smoke-view-groups.test.js" + "smoke:view-groups": "jest --verbose --forceExit -i dist/test/smoke-view-groups.test.js", + "smoke:shared-calc-group": "TZ=UTC jest --verbose -i dist/test/smoke-shared-calc-group.test.js" }, "files": [ "dist/src", diff --git a/packages/cubejs-testing/test/smoke-shared-calc-group.test.ts b/packages/cubejs-testing/test/smoke-shared-calc-group.test.ts new file mode 100644 index 0000000000000..401f70cc0cc5e --- /dev/null +++ b/packages/cubejs-testing/test/smoke-shared-calc-group.test.ts @@ -0,0 +1,156 @@ +import { StartedTestContainer } from 'testcontainers'; +// eslint-disable-next-line import/no-extraneous-dependencies +import { afterAll, beforeAll, expect, jest } from '@jest/globals'; +import cubejs, { CubeApi, Query } from '@cubejs-client/core'; +import { PostgresDBRunner } from '@cubejs-backend/testing-shared'; +import { BirdBox, getBirdbox } from '../src'; +import { + DEFAULT_API_TOKEN, + DEFAULT_CONFIG, + JEST_AFTER_ALL_DEFAULT_TIMEOUT, + JEST_BEFORE_ALL_DEFAULT_TIMEOUT, +} from './smoke-tests'; + +// End-to-end pre-aggregation coverage for rolling-window metrics exposed +// through multi-stage `case` entrypoint measures dispatched by a shared +// `type: switch` dimension (calc group) across two joined fact cubes. +// Unlike the schema-compiler integration spec (which builds rollups in +// Postgres), rollups here are stored and queried in Cube Store, so the +// multi-stage plans are executed by the Cube Store engine like in +// production. +describe('shared calc group pre-aggregations in Cube Store', () => { + jest.setTimeout(60 * 5 * 1000); + let db: StartedTestContainer; + let birdbox: BirdBox; + let client: CubeApi; + + beforeAll(async () => { + db = await PostgresDBRunner.startContainer({}); + birdbox = await getBirdbox( + 'postgres', + { + ...DEFAULT_CONFIG, + CUBEJS_DB_HOST: db.getHost(), + CUBEJS_DB_PORT: `${db.getMappedPort(5432)}`, + CUBEJS_DB_NAME: 'test', + CUBEJS_DB_USER: 'test', + CUBEJS_DB_PASS: 'test', + CUBEJS_ROLLUP_ONLY: 'true', + CUBEJS_REFRESH_WORKER: 'false', + CUBEJS_TESSERACT_SQL_PLANNER: 'true', + }, + { + schemaDir: 'shared-calc-group/schema', + cubejsConfig: 'shared-calc-group/cube.js', + }, + ); + client = cubejs(async () => DEFAULT_API_TOKEN, { + apiUrl: birdbox.configuration.apiUrl, + }); + }, JEST_BEFORE_ALL_DEFAULT_TIMEOUT); + + afterAll(async () => { + await birdbox.stop(); + await db.stop(); + }, JEST_AFTER_ALL_DEFAULT_TIMEOUT); + + const REPRO_FILTERS: Query['filters'] = [ + { + member: 'performance_view.account', + operator: 'equals', + values: ['A1'], + }, + { + member: 'performance_view.rolling_window', + operator: 'equals', + values: ['R3'], + }, + ]; + + // Every reference to a rollup in a multi-stage plan is keyed separately + // (`__usage_N` suffix), so dedupe to the distinct rollup tables used. + function usedPreAggregations(resultSet: any): string[] { + const keys = Object.keys( + resultSet.serialize().loadResponse.results[0].usedPreAggregations || {} + ); + return [...new Set(keys.map(t => t.replace(/__usage_\d+$/, '')))].sort(); + } + + const SALES_ROLLUP = 'dev_pre_aggregations.sales_perf_rolling'; + const SHARE_ROLLUP = 'dev_pre_aggregations.share_metrics_perf_share'; + + test('single-cube rolling measure is served from the rollup', async () => { + const query: Query = { + measures: ['performance_view.rolling_amount'], + dimensions: ['performance_view.product'], + filters: REPRO_FILTERS, + order: { + 'performance_view.product': 'asc', + }, + }; + const result = await client.load(query); + expect(usedPreAggregations(result)).toEqual([SALES_ROLLUP]); + expect(result.rawData().map((r: any) => r['performance_view.product'])).toEqual(['P1', 'P2']); + }); + + test('cross-cube rolling measures are served from both rollups', async () => { + const query: Query = { + measures: [ + 'performance_view.rolling_amount', + 'performance_view.rolling_share_change', + ], + dimensions: ['performance_view.product'], + filters: REPRO_FILTERS, + order: { + 'performance_view.product': 'asc', + }, + }; + const result = await client.load(query); + expect(usedPreAggregations(result)).toEqual([SALES_ROLLUP, SHARE_ROLLUP]); + expect(result.rawData().map((r: any) => r['performance_view.product'])).toEqual(['P1', 'P2']); + }); + + test('full multi-stage query executes in Cube Store', async () => { + const query: Query = { + measures: [ + 'performance_view.rolling_amount', + 'performance_view.rolling_amount_change', + 'performance_view.rolling_share_change', + ], + dimensions: ['performance_view.product'], + filters: REPRO_FILTERS, + order: { + 'performance_view.product': 'asc', + }, + }; + const result = await client.load(query); + expect(usedPreAggregations(result)).toEqual([SALES_ROLLUP, SHARE_ROLLUP]); + expect(result.rawData().map((r: any) => r['performance_view.product'])).toEqual(['P1', 'P2']); + }); + + // Mirrors the production query shape: rolling amount + growth percentage + // (an extra multi-stage layer over the same rolling leaves) + cross-cube + // share change. Before the trivial-subquery collapse optimizer in the + // Tesseract physical plan builder, the deep FullKeyAggregate plan this + // produces overflowed Cube Store's serialized-plan decode recursion limit + // and the query failed with "Error during planning: Error decoding expr + // as protobuf: ... recursion limit reached". + test('deep multi-stage query with growth percentage executes in Cube Store', async () => { + const query: Query = { + measures: [ + 'performance_view.rolling_amount', + 'performance_view.rolling_amount_change', + 'performance_view.rolling_amount_growth_pct', + 'performance_view.rolling_share_change', + ], + dimensions: ['performance_view.product'], + filters: REPRO_FILTERS, + order: { + 'performance_view.product': 'asc', + }, + }; + const result = await client.load(query); + expect(usedPreAggregations(result)).toEqual([SALES_ROLLUP, SHARE_ROLLUP]); + expect(result.rawData().map((r: any) => r['performance_view.product'])).toEqual(['P1', 'P2']); + }); +}); diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/logical_plan/optimizers/pre_aggregation/dimension_matcher.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/logical_plan/optimizers/pre_aggregation/dimension_matcher.rs index ac7b1bf24414e..4aa074b3fdc5b 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/logical_plan/optimizers/pre_aggregation/dimension_matcher.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/logical_plan/optimizers/pre_aggregation/dimension_matcher.rs @@ -198,6 +198,13 @@ impl<'a> DimensionMatcher<'a> { *found = true; } Ok(MatchState::Full) + } else if dimension.is_calc_group() { + // Calc-group dimensions are virtual enumerations: they carry no + // stored data, so any rollup can serve them. At query time the + // value is either pinned by a filter (rendered as a literal) or + // re-cross-joined over the rollup scan, exactly like over a raw + // source. + Ok(MatchState::Full) } else if dimension.owned_by_cube() { Ok(MatchState::NotMatched) } else if dimension.is_multi_stage() { diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan/mod.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan/mod.rs index 9d7c0cfee21bd..a08db407095c0 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan/mod.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan/mod.rs @@ -6,6 +6,7 @@ pub mod expression; pub mod filter; pub mod from; pub mod join; +pub mod optimizers; pub mod order; pub mod query_plan; pub mod references_builder; @@ -25,6 +26,7 @@ pub use cube_ref_evaluator::CubeRefEvaluator; pub use expression::{Expr, MemberExpression}; pub use from::{From, FromSource, SingleAliasedSource, SingleSource}; pub use join::{Join, JoinCondition, JoinItem, RegularRollingWindowJoinCondition}; +pub use optimizers::collapse_trivial_subqueries; pub use order::OrderBy; pub use query_plan::QueryPlan; pub use references_builder::ReferencesBuilder; diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan/optimizers/mod.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan/optimizers/mod.rs new file mode 100644 index 0000000000000..7013c01bdf01e --- /dev/null +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan/optimizers/mod.rs @@ -0,0 +1,3 @@ +pub mod trivial_subquery; + +pub use trivial_subquery::collapse_trivial_subqueries; diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan/optimizers/trivial_subquery.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan/optimizers/trivial_subquery.rs new file mode 100644 index 0000000000000..22c21bd0e4df2 --- /dev/null +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan/optimizers/trivial_subquery.rs @@ -0,0 +1,152 @@ +//! Physical-plan optimizer collapsing trivial pass-through subqueries. +//! +//! Multi-stage planning wraps every CTE reference into a derived table of +//! the form `(SELECT * FROM cte_n AS cte_n) AS alias`. Each such wrapper is +//! semantically a no-op, but engines that inline CTE bodies at every +//! reference (e.g. Cube Store / DataFusion) pay for it with two extra plan +//! nodes (projection + subquery alias) per usage. Deep multi-stage queries +//! can overflow Cube Store's serialized-plan decode recursion limit purely +//! because of these wrappers. +//! +//! This pass rewrites `(SELECT * FROM
AS x) AS alias` into a +//! direct `
AS alias` reference everywhere in the plan: in the +//! top-level select, inside CTE bodies, join sides, unions and nested +//! subqueries. Only selects that are pure pass-throughs are collapsed: a +//! `SELECT *` projection with no filter, grouping, having, ordering, +//! distinct, limit, offset or own CTEs, reading from a single table +//! reference. + +use super::super::{ + Cte, From, FromSource, Join, JoinItem, QueryPlan, Select, SingleAliasedSource, SingleSource, +}; +use crate::physical_plan::CalcGroupsJoin; +use cubenativeutils::CubeError; +use std::rc::Rc; + +/// Entry point: returns a plan equivalent to `select` with all trivial +/// pass-through subqueries collapsed into direct table references. +pub fn collapse_trivial_subqueries(select: &Rc) -> Result, CubeError> { + let ctes = select + .ctes + .iter() + .map(|cte| -> Result<_, CubeError> { + Ok(Rc::new(Cte::new( + optimize_plan(cte.query())?, + cte.name().clone(), + cte.is_recursive(), + ))) + }) + .collect::, _>>()?; + + let from = optimize_from(&select.from)?; + + Ok(Rc::new(Select { + from, + ctes, + ..select.as_ref().clone() + })) +} + +fn optimize_plan(plan: &Rc) -> Result, CubeError> { + let result = match plan.as_ref() { + QueryPlan::Select(select) => QueryPlan::Select(optimize_select(select)?), + QueryPlan::Union(union) => { + let items = union + .union + .iter() + .map(|item| -> Result<_, CubeError> { + Ok(match item { + QueryPlan::Select(select) => QueryPlan::Select(optimize_select(select)?), + other => other.clone(), + }) + }) + .collect::, _>>()?; + QueryPlan::Union(Rc::new(super::super::Union::new(items))) + } + QueryPlan::TimeSeries(_) => return Ok(plan.clone()), + }; + Ok(Rc::new(result)) +} + +fn optimize_from(from: &Rc) -> Result, CubeError> { + let source = match &from.source { + FromSource::Empty => FromSource::Empty, + FromSource::Single(source) => FromSource::Single(optimize_single_source(source)?), + FromSource::Join(join) => { + let root = optimize_single_source(&join.root)?; + let joins = join + .joins + .iter() + .map(|item| -> Result<_, CubeError> { + Ok(JoinItem { + from: optimize_single_source(&item.from)?, + on: item.on.clone(), + join_type: item.join_type.clone(), + }) + }) + .collect::, _>>()?; + FromSource::Join(Rc::new(Join { root, joins })) + } + FromSource::CalcGroupsJoin(join) => { + let inner = optimize_from(join.from())?; + FromSource::CalcGroupsJoin(CalcGroupsJoin::try_new(inner, join.calc_groups().clone())?) + } + }; + Ok(From::new(source)) +} + +fn optimize_single_source(source: &SingleAliasedSource) -> Result { + match &source.source { + SingleSource::Subquery(plan) => { + let optimized = optimize_plan(plan)?; + if let Some(collapsed) = try_collapse(&optimized, &source.alias) { + return Ok(collapsed); + } + Ok(SingleAliasedSource::new_from_source( + SingleSource::Subquery(optimized), + source.alias.clone(), + )) + } + _ => Ok(source.clone()), + } +} + +/// If `plan` is a trivial pass-through select over a single table +/// reference, return that table reference re-aliased with the outer alias. +fn try_collapse(plan: &Rc, alias: &String) -> Option { + let QueryPlan::Select(select) = plan.as_ref() else { + return None; + }; + if !is_trivial_passthrough(select) { + return None; + } + let FromSource::Single(inner_source) = &select.from.source else { + return None; + }; + let SingleSource::TableReference(reference, _) = &inner_source.source else { + return None; + }; + Some(SingleAliasedSource::new_from_table_reference( + reference.clone(), + select.schema(), + Some(alias.clone()), + )) +} + +/// A select that renders as `SELECT * FROM ` with no other +/// clauses: removing it does not change the produced rows or column names. +fn is_trivial_passthrough(select: &Select) -> bool { + select.projection_columns.is_empty() + && select.filter.is_none() + && select.group_by.is_empty() + && select.having.is_none() + && select.order_by.is_empty() + && select.ctes.is_empty() + && !select.is_distinct + && select.limit.is_none() + && select.offset.is_none() +} diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan_builder/builder.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan_builder/builder.rs index 78a5fdebd9705..6cd6819b01c19 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan_builder/builder.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan_builder/builder.rs @@ -71,6 +71,7 @@ impl PhysicalPlanBuilder { } else { query }; + let query = collapse_trivial_subqueries(&query)?; Ok(query) } diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan_builder/processors/query.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan_builder/processors/query.rs index a388859ac3cb1..3276753c09d47 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan_builder/processors/query.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan_builder/processors/query.rs @@ -9,6 +9,7 @@ use crate::planner::collectors::collect_calc_group_dims_from_nodes; use crate::planner::get_filtered_values; use cubenativeutils::CubeError; use itertools::Itertools; +use std::collections::HashSet; use std::rc::Rc; pub struct QueryProcessor<'a> { @@ -66,10 +67,28 @@ impl<'a> LogicalNodeProcessor<'a, Query> for QueryProcessor<'a> { let filter = logical_plan.filter().all_filters(); let having = logical_plan.filter().measures_filter(); - //TODO pre-aggregations support for calc-groups - let from = if let QuerySource::LogicalJoin(_) = logical_plan.source() { + // Calc-group dimensions are resolved at query time: a value pinned by + // a filter renders as a literal, otherwise the enumeration is + // cross-joined as a virtual values table. Over a pre-aggregation this + // applies only to calc groups NOT stored in the rollup — stored ones + // keep resolving to the rollup column for backward compatibility. + let calc_group_stored_dims = match logical_plan.source() { + QuerySource::LogicalJoin(_) => Some(HashSet::new()), + QuerySource::PreAggregation(pre_aggregation) => Some( + pre_aggregation + .all_dimensions_refererences() + .into_keys() + .collect::>(), + ), + QuerySource::FullKeyAggregate(_) => None, + }; + let mut calc_group_value_references: Vec<(String, String)> = Vec::new(); + let from = if let Some(stored_dims) = calc_group_stored_dims { let all_symbols = all_symbols(&logical_plan.schema(), &logical_plan.filter()); - let calc_group_dims = collect_calc_group_dims_from_nodes(all_symbols.iter())?; + let calc_group_dims = collect_calc_group_dims_from_nodes(all_symbols.iter())? + .into_iter() + .filter(|dim| !stored_dims.contains(&dim.full_name())) + .collect_vec(); let calc_groups_items = calc_group_dims.into_iter().map(|dim| { let values = get_filtered_values(&dim, &filter); @@ -84,6 +103,7 @@ impl<'a> LogicalNodeProcessor<'a, Query> for QueryProcessor<'a> { { context_factory .add_render_reference(item.symbol.full_name(), item.values[0].clone()); + calc_group_value_references.push((item.symbol.full_name(), item.values[0].clone())); } let calc_groups_to_join = calc_groups_items .filter(|itm| itm.values.len() > 1) @@ -190,6 +210,11 @@ impl<'a> LogicalNodeProcessor<'a, Query> for QueryProcessor<'a> { if is_pre_aggregation { context_factory.clear_render_references(); + // Calc-group values are rendered as literals, not resolved from + // the rollup, so they must survive the render-reference reset. + for (name, value) in calc_group_value_references.into_iter() { + context_factory.add_render_reference(name, value); + } } if logical_plan.modifers().ungrouped { context_factory.set_ungrouped(true); diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/multi_stage/mod.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/multi_stage/mod.rs index dd627f0ba5622..de8f5ba041c99 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/multi_stage/mod.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/multi_stage/mod.rs @@ -12,6 +12,7 @@ mod joins; mod multi_fact; mod multiple_measures; mod order_limit; +mod plan_optimizers; mod rank; mod reduce_by; mod time_shift_basic; diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/multi_stage/plan_optimizers.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/multi_stage/plan_optimizers.rs new file mode 100644 index 0000000000000..dfa82d5f2426b --- /dev/null +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/multi_stage/plan_optimizers.rs @@ -0,0 +1,65 @@ +//! Tests for physical-plan optimizer passes applied to multi-stage queries. + +use crate::test_fixtures::cube_bridge::MockSchema; +use crate::test_fixtures::test_utils::TestContext; +use indoc::indoc; +use regex::Regex; + +fn create_context() -> TestContext { + let schema = MockSchema::from_yaml_file("common/integration_multi_stage.yaml"); + TestContext::new(schema).unwrap() +} + +const SEED: &str = "integration_multi_stage_tables.sql"; + +/// Multi-stage planning wraps CTE references into trivial +/// `(SELECT * FROM cte_n AS cte_n) AS alias` derived tables; the +/// trivial-subquery optimizer must collapse them into direct +/// `cte_n AS alias` references so engines that inline CTE bodies at +/// every reference (Cube Store / DataFusion) don't pay two extra plan +/// nodes per usage. +#[tokio::test(flavor = "multi_thread")] +async fn test_trivial_cte_wrappers_are_collapsed() { + let ctx = create_context(); + + let query = indoc! {r#" + measures: + - orders.rolling_sum_7d + - orders.amount_prev_month + time_dimensions: + - dimension: orders.created_at + granularity: month + dateRange: + - "2024-01-01" + - "2024-03-31" + "#}; + + let sql = ctx.build_sql(query).unwrap(); + + assert!( + sql.contains("cte_0"), + "Expected a multi-stage CTE in the generated SQL:\n{}", + sql + ); + + // `SELECT * FROM ` (a bare table or CTE reference, not a + // parenthesized subquery) is exactly the trivial pass-through shape the + // optimizer must have collapsed. + let trivial_wrapper = Regex::new(r"(?i)\(\s*SELECT\s+\*\s+FROM\s+[A-Za-z_]").unwrap(); + assert!( + !trivial_wrapper.is_match(&sql), + "Generated SQL still contains a trivial pass-through subquery wrapper:\n{}", + sql + ); + + // The collapsed references keep the outer alias. + assert!( + sql.contains("cte_0 AS"), + "Expected a direct aliased CTE reference in the generated SQL:\n{}", + sql + ); + + if let Some(result) = ctx.try_execute_pg(query, SEED).await { + insta::assert_snapshot!(result); + } +} diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/multi_stage/snapshots/cubesqlplanner__tests__integration__multi_stage__plan_optimizers__trivial_cte_wrappers_are_collapsed.snap b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/multi_stage/snapshots/cubesqlplanner__tests__integration__multi_stage__plan_optimizers__trivial_cte_wrappers_are_collapsed.snap new file mode 100644 index 0000000000000..f017fa96d68da --- /dev/null +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/multi_stage/snapshots/cubesqlplanner__tests__integration__multi_stage__plan_optimizers__trivial_cte_wrappers_are_collapsed.snap @@ -0,0 +1,9 @@ +--- +source: cubesqlplanner/cubesqlplanner/src/tests/integration/multi_stage/plan_optimizers.rs +expression: result +--- +orders__created_at_month | orders__rolling_sum_7d | orders__amount_prev_month +-------------------------+------------------------+-------------------------- +2024-01-01 00:00:00 | NULL | NULL +2024-02-01 00:00:00 | 30.00 | 500.00 +2024-03-01 00:00:00 | 50.00 | 750.00 From a80729c42381fe93826c33bc5479792c54d676fb Mon Sep 17 00:00:00 2001 From: Pavel Tiunov Date: Fri, 24 Jul 2026 23:27:35 -0700 Subject: [PATCH 2/2] feat(cube-cli): expose the dev-mode branch name and document dev-mode-only data-model writes (#11351) --- rust/cube-cli/README.md | 2 +- rust/cube-cli/src/commands/data_model.rs | 71 +++++++++++++++++++----- 2 files changed, 57 insertions(+), 16 deletions(-) diff --git a/rust/cube-cli/README.md b/rust/cube-cli/README.md index 67aede1a13849..04e822fcbd989 100644 --- a/rust/cube-cli/README.md +++ b/rust/cube-cli/README.md @@ -132,7 +132,7 @@ Every endpoint of the Console Server public API is covered: | `regions` | list available deployment regions | | `logs` | tail deployment pod logs (`--pod`, `-c/--container`; defaults to the Cube API container) | | `github` (`gh`) | status, installations, repos, branches, connect (import a repo into a deployment + first build) | -| `data-model` (`dm`) | list, get, put, delete, rename files; branches, create-branch, dev-mode, exit-dev-mode, commit, pull | +| `data-model` (`dm`) | list, get, put, delete, rename files; branches, create-branch, dev-mode, exit-dev-mode, commit, pull. File writes only land on a **dev-mode branch**: `dev-mode ` forks a personal `dev-…` branch and prints its name — pass that via `--branch` (or omit `--branch` to use your active dev-mode branch); puts to any other branch are rejected by the API | | `environments` | list, tokens, create-token (incl. `--meta-sync`) | | `variables` | list, set (`KEY=VALUE` upserts) | | `folders` | list, create, update, delete, ancestors | diff --git a/rust/cube-cli/src/commands/data_model.rs b/rust/cube-cli/src/commands/data_model.rs index 297d137534d81..17784ccf60885 100644 --- a/rust/cube-cli/src/commands/data_model.rs +++ b/rust/cube-cli/src/commands/data_model.rs @@ -38,7 +38,7 @@ enum Cmd { #[arg(long)] branch: Option, }, - /// Create or overwrite a file + /// Create or overwrite a file (writes require a dev-mode branch) Put { /// Deployment id deployment: i64, @@ -50,11 +50,12 @@ enum Cmd { /// Inline content (use `-` to read stdin) #[arg(long)] content: Option, - /// Branch name (defaults to the deployment default branch) + /// Dev-mode branch to write to, as returned by `dev-mode` (defaults to + /// your active dev-mode branch) #[arg(long)] branch: Option, }, - /// Delete files + /// Delete files (writes require a dev-mode branch) #[command(alias = "rm")] Delete { /// Deployment id @@ -62,11 +63,12 @@ enum Cmd { /// One or more file paths to delete #[arg(required = true)] paths: Vec, - /// Branch name (defaults to the deployment default branch) + /// Dev-mode branch to write to, as returned by `dev-mode` (defaults to + /// your active dev-mode branch) #[arg(long)] branch: Option, }, - /// Rename (move) a file + /// Rename (move) a file (writes require a dev-mode branch) Rename { /// Deployment id deployment: i64, @@ -74,7 +76,8 @@ enum Cmd { from: String, /// Destination path to: String, - /// Branch name (defaults to the deployment default branch) + /// Dev-mode branch to write to, as returned by `dev-mode` (defaults to + /// your active dev-mode branch) #[arg(long)] branch: Option, }, @@ -93,11 +96,12 @@ enum Cmd { #[arg(long)] dev_mode: bool, }, - /// Enter dev mode / switch to a branch + /// Enter dev mode on a branch (prints the personal dev-mode branch that + /// file writes must target) DevMode { /// Deployment id deployment: i64, - /// Branch to switch to (required by the API) + /// Branch to base dev mode on (required by the API) branch: String, }, /// Exit dev mode @@ -181,6 +185,22 @@ fn flatten(nodes: &[serde_json::Value], out: &mut Vec) { } } +/// Printed after entering dev mode, so the user knows which branch the write +/// commands accept. +fn dev_branch_hint(dev_branch: &str) -> String { + format!( + "Data-model writes target it: pass --branch {dev_branch} \ + (or omit --branch to use your active dev-mode branch)." + ) +} + +/// The source tree reports absolute paths (`/model/cubes/orders.yml`) while the +/// write endpoints accept them with or without the leading slash, so compare +/// paths without it. +fn same_path(a: &str, b: &str) -> bool { + a.trim_start_matches('/') == b.trim_start_matches('/') +} + fn tree_nodes(res: &serde_json::Value) -> Vec { let mut out = Vec::new(); if let Some(arr) = res.get("data").and_then(|d| d.as_array()) { @@ -252,9 +272,9 @@ pub async fn command(args: Args, ctx: &Ctx) -> Result<()> { let mut query: Query = vec![("withContent".into(), "true".into())]; util::push(&mut query, "branchName", &branch); let res = api.get(&base(deployment), &query).await?; - let file = tree_nodes(&res) - .into_iter() - .find(|f| output::field(f, "path") == path && output::field(f, "type") == "file"); + let file = tree_nodes(&res).into_iter().find(|f| { + same_path(&output::field(f, "path"), &path) && output::field(f, "type") == "file" + }); match file { Some(f) => print!("{}", output::field(&f, "content")), None => anyhow::bail!("file not found: {path}"), @@ -303,9 +323,10 @@ pub async fn command(args: Args, ctx: &Ctx) -> Result<()> { to, branch, } => { + // Like put/delete, the rename endpoint expects `files` as an array + // of objects — here `{ path, newPath }`. let mut map = serde_json::Map::new(); - map.insert("from".into(), json!(from)); - map.insert("to".into(), json!(to)); + map.insert("files".into(), json!([{ "path": from, "newPath": to }])); let res = api .post( &format!("{}/rename", base(deployment)), @@ -350,7 +371,17 @@ pub async fn command(args: Args, ctx: &Ctx) -> Result<()> { if ctx.json { output::print_json(&res); } else { - output::success(&format!("Created branch {name}")); + // With --dev-mode the server forks a personal dev-mode branch off + // the new branch; that's the branch file writes must target. + let dev_branch = output::field(&res, "branchName"); + if dev_mode && !dev_branch.is_empty() && dev_branch != name { + output::success(&format!( + "Created branch {name}; entered dev mode on {dev_branch}" + )); + println!("{}", dev_branch_hint(&dev_branch)); + } else { + output::success(&format!("Created branch {name}")); + } } } Cmd::DevMode { deployment, branch } => { @@ -364,7 +395,17 @@ pub async fn command(args: Args, ctx: &Ctx) -> Result<()> { if ctx.json { output::print_json(&res); } else { - output::success(&format!("Entered dev mode on {branch}")); + // Dev mode runs on a personal `dev-…` branch forked from the + // requested one — expose it, since file writes only accept it. + let dev_branch = output::field(&res, "branchName"); + if dev_branch.is_empty() || dev_branch == branch { + output::success(&format!("Entered dev mode on {branch}")); + } else { + output::success(&format!( + "Entered dev mode on {dev_branch} (forked from {branch})" + )); + println!("{}", dev_branch_hint(&dev_branch)); + } } } Cmd::ExitDevMode { deployment } => {