From ae248c446651790483212dd68176a9b78fe7844b Mon Sep 17 00:00:00 2001 From: Alex Qyoun-ae <4062971+MazterQyou@users.noreply.github.com> Date: Wed, 22 Jul 2026 19:38:37 +0400 Subject: [PATCH 1/3] fix(cubesql): Keep PostgreSQL integer division semantics in pushdown SQL (#11319) Signed-off-by: Alex Qyoun-ae <4062971+MazterQyou@users.noreply.github.com> --- .../src/DatabricksQuery.ts | 3 + .../cubejs-duckdb-driver/src/DuckDBQuery.ts | 4 + .../cubejs-pinot-driver/src/PinotQuery.ts | 6 + .../src/adapter/BaseQuery.js | 4 + .../src/adapter/BigqueryQuery.ts | 3 + .../src/adapter/ClickHouseQuery.ts | 4 + .../src/adapter/MysqlQuery.ts | 3 + .../src/adapter/OracleQuery.ts | 3 + .../src/adapter/SnowflakeQuery.ts | 4 + .../cubesql/src/compile/engine/df/wrapper.rs | 172 +++++++++++++++++- rust/cubesql/cubesql/src/compile/mod.rs | 60 ++++++ rust/cubesql/cubesql/src/compile/test/mod.rs | 2 + rust/cubesql/cubesql/src/transport/service.rs | 7 + 13 files changed, 266 insertions(+), 9 deletions(-) diff --git a/packages/cubejs-databricks-jdbc-driver/src/DatabricksQuery.ts b/packages/cubejs-databricks-jdbc-driver/src/DatabricksQuery.ts index 0ee37df1b9bd4..5d616e756eb44 100644 --- a/packages/cubejs-databricks-jdbc-driver/src/DatabricksQuery.ts +++ b/packages/cubejs-databricks-jdbc-driver/src/DatabricksQuery.ts @@ -195,6 +195,9 @@ export class DatabricksQuery extends BaseQuery { "'DDD', '@DOY@'), 'Day', 'EEEE'), 'Dy', 'EEE'), 'DD', 'dd'), '@DOY@', 'DDD'), " + "'AM', 'a'), 'PM', 'a'))"; templates.expressions.timestamp_literal = 'from_utc_timestamp(\'{{ value }}\', \'UTC\')'; + // Spark `/` returns DOUBLE for integer operands; `div` returns the integral + // part of the division as BIGINT (truncation toward zero), matching PostgreSQL + templates.expressions.int_division = '({{ left }} div {{ right }})'; templates.expressions.extract = '{% if date_part|lower == "epoch" %}unix_timestamp({{ expr }}){% else %}EXTRACT({{ date_part }} FROM {{ expr }}){% endif %}'; templates.expressions.interval_single_date_part = 'INTERVAL \'{{ num }}\' {{ date_part }}'; templates.quotes.identifiers = '`'; diff --git a/packages/cubejs-duckdb-driver/src/DuckDBQuery.ts b/packages/cubejs-duckdb-driver/src/DuckDBQuery.ts index 758cea95c5b57..d9c0061a890df 100644 --- a/packages/cubejs-duckdb-driver/src/DuckDBQuery.ts +++ b/packages/cubejs-duckdb-driver/src/DuckDBQuery.ts @@ -69,6 +69,10 @@ export class DuckDBQuery extends BaseQuery { templates.functions.STRING_AGG = 'STRING_AGG({% if distinct %}DISTINCT {% endif %}{{ args[0] }}, COALESCE({{ args[1] }}, \'\'))'; templates.expressions.like = '{{ expr }} {% if negated %}NOT {% endif %}LIKE {{ pattern }}{% if default_escape %} ESCAPE \'\\\'{% endif %}'; templates.expressions.ilike = '{{ expr }} {% if negated %}NOT {% endif %}ILIKE {{ pattern }}{% if default_escape %} ESCAPE \'\\\'{% endif %}'; + // DuckDB `/` performs float division even for integer operands (since v0.8); + // `//` is integer division truncating toward zero (-7 // 2 = -3), matching + // PostgreSQL + templates.expressions.int_division = '({{ left }} // {{ right }})'; return templates; } diff --git a/packages/cubejs-pinot-driver/src/PinotQuery.ts b/packages/cubejs-pinot-driver/src/PinotQuery.ts index 73cb3b96955aa..e1b3a1ef44296 100644 --- a/packages/cubejs-pinot-driver/src/PinotQuery.ts +++ b/packages/cubejs-pinot-driver/src/PinotQuery.ts @@ -235,6 +235,12 @@ export class PinotQuery extends BaseQuery { '{% if limit is not none %}\nLIMIT {{ limit }}{% endif %}' + '{% if offset is not none %}\nOFFSET {{ offset }}{% endif %}'; templates.expressions.extract = 'EXTRACT({{ date_part }} FROM {{ expr }})'; + // Pinot `/` returns DOUBLE for integer operands, and intDiv() is floor + // division (intDiv(-7, 2) = -4). CAST to LONG truncates toward zero + // (CAST(-7 / 2 AS LONG) = -3), matching PostgreSQL integer division. + // Note: the division itself happens in DOUBLE, so results are exact only + // up to 2^53 + templates.expressions.int_division = 'CAST({{ left }} / {{ right }} AS LONG)'; templates.expressions.timestamp_literal = 'fromDateTime(\'{{ value }}\', \'yyyy-MM-dd\'\'T\'\'HH:mm:ss.SSS\'\'Z\'\'\')'; // NOTE: this template contains a comma; two order expressions are being generated templates.expressions.sort = '{{ expr }} IS NULL {% if nulls_first %}DESC{% else %}ASC{% endif %}, {{ expr }} {% if asc %}ASC{% else %}DESC{% endif %}'; diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js index fbc4aae41d0f9..7f44fe3f44aab 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js +++ b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js @@ -4609,6 +4609,10 @@ export class BaseQuery { case: 'CASE{% if expr %} {{ expr }}{% endif %}{% for when, then in when_then %} WHEN {{ when }} THEN {{ then }}{% endfor %}{% if else_expr %} ELSE {{ else_expr }}{% endif %} END', is_null: '({{ expr }} IS {% if negate %}NOT {% endif %}NULL)', binary: '({{ left }} {{ op }} {{ right }})', + // Integer division with PostgreSQL semantics: truncation toward zero. + // Plain `/` is correct for dialects where int / int is integer division; + // dialects with decimal or float `/` must override this template + int_division: '({{ left }} / {{ right }})', sort: '{{ expr }} {% if asc %}ASC{% else %}DESC{% endif %} NULLS {% if nulls_first %}FIRST{% else %}LAST{% endif %}', order_by: '{% if index %} {{ index }} {% else %} {{ expr }} {% endif %} {% if asc %}ASC{% else %}DESC{% endif %}{% if nulls_first %} NULLS FIRST{% endif %}', cast: 'CAST({{ expr }} AS {{ data_type }})', diff --git a/packages/cubejs-schema-compiler/src/adapter/BigqueryQuery.ts b/packages/cubejs-schema-compiler/src/adapter/BigqueryQuery.ts index bae84bd3c51ab..d9b45a063e8f8 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BigqueryQuery.ts +++ b/packages/cubejs-schema-compiler/src/adapter/BigqueryQuery.ts @@ -337,6 +337,9 @@ export class BigqueryQuery extends BaseQuery { delete templates.functions.PERCENTILECONT; templates.expressions.binary = '{% if op == \'%\' %}MOD({{ left }}, {{ right }}){% else %}({{ left }} {{ op }} {{ right }}){% endif %}'; templates.expressions.interval = 'INTERVAL {{ interval }}'; + // BigQuery `/` on INT64 operands returns FLOAT64; DIV() is integer division + // truncating toward zero, matching PostgreSQL (DIV(12, -7) = -1) + templates.expressions.int_division = 'DIV({{ left }}, {{ right }})'; templates.expressions.extract = 'EXTRACT({% if date_part == \'DOW\' %}DAYOFWEEK{% elif date_part == \'DOY\' %}DAYOFYEAR{% else %}{{ date_part }}{% endif %} FROM {{ expr }})'; templates.expressions.timestamp_literal = 'TIMESTAMP(\'{{ value }}\')'; templates.expressions.rolling_window_expr_timestamp_cast = 'TIMESTAMP({{ value }})'; diff --git a/packages/cubejs-schema-compiler/src/adapter/ClickHouseQuery.ts b/packages/cubejs-schema-compiler/src/adapter/ClickHouseQuery.ts index 1f7a4113e6d6a..9c2187bc60878 100644 --- a/packages/cubejs-schema-compiler/src/adapter/ClickHouseQuery.ts +++ b/packages/cubejs-schema-compiler/src/adapter/ClickHouseQuery.ts @@ -279,6 +279,10 @@ export class ClickHouseQuery extends BaseQuery { delete templates.types.interval; delete templates.types.binary; templates.expressions.is_not_distinct_from = 'isNotDistinctFrom({{ left }}, {{ right }})'; + // ClickHouse `/` always returns Float64; intDiv is integer division truncating + // toward zero, matching PostgreSQL (intDiv(-7, 2) = -3 despite docs saying + // "rounded down") + templates.expressions.int_division = 'intDiv({{ left }}, {{ right }})'; templates.statements.time_series_select = 'SELECT parseDateTimeBestEffort(dates.f) date_from, parseDateTimeBestEffort(dates.t) date_to \n' + 'FROM (\n' + diff --git a/packages/cubejs-schema-compiler/src/adapter/MysqlQuery.ts b/packages/cubejs-schema-compiler/src/adapter/MysqlQuery.ts index ac45aa5fdd3cc..e3cbd2e4a3aa8 100644 --- a/packages/cubejs-schema-compiler/src/adapter/MysqlQuery.ts +++ b/packages/cubejs-schema-compiler/src/adapter/MysqlQuery.ts @@ -193,6 +193,9 @@ export class MysqlQuery extends BaseQuery { templates.quotes.escape = '\\`'; // NOTE: this template contains a comma; two order expressions are being generated templates.expressions.sort = '{{ expr }} IS NULL {% if nulls_first %}DESC{% else %}ASC{% endif %}, {{ expr }} {% if asc %}ASC{% else %}DESC{% endif %}'; + // MySQL `/` returns DECIMAL even for integer operands; DIV discards the + // fractional part (truncation toward zero), matching PostgreSQL (-5 DIV 2 = -2) + templates.expressions.int_division = '({{ left }} DIV {{ right }})'; delete templates.expressions.ilike; templates.types.string = 'CHAR'; templates.types.boolean = 'TINYINT'; diff --git a/packages/cubejs-schema-compiler/src/adapter/OracleQuery.ts b/packages/cubejs-schema-compiler/src/adapter/OracleQuery.ts index 789efef445a80..b9ab41d0dc9cb 100644 --- a/packages/cubejs-schema-compiler/src/adapter/OracleQuery.ts +++ b/packages/cubejs-schema-compiler/src/adapter/OracleQuery.ts @@ -218,6 +218,9 @@ export class OracleQuery extends BaseQuery { templates.functions.UTCTIMESTAMP = 'SYS_EXTRACT_UTC(SYSTIMESTAMP)'; // Oracle forbids `AS` before a table/subquery alias. templates.expressions.query_aliased = '{{ query }} {{ quoted_alias }}'; + // Oracle `/` on NUMBER keeps the fractional part; TRUNC drops decimal digits + // (truncation toward zero), matching PostgreSQL integer division + templates.expressions.int_division = 'TRUNC({{ left }} / {{ right }})'; // Oracle does not support positional GROUP BY — group by expressions. templates.statements.group_by_exprs = '{{ group_by | map(attribute=\'expr\') | join(\', \') }}'; // No `AS` before the FROM subquery alias, and Oracle row-limiting syntax. diff --git a/packages/cubejs-schema-compiler/src/adapter/SnowflakeQuery.ts b/packages/cubejs-schema-compiler/src/adapter/SnowflakeQuery.ts index 746bf0fc3cbf7..88ccca3a6b391 100644 --- a/packages/cubejs-schema-compiler/src/adapter/SnowflakeQuery.ts +++ b/packages/cubejs-schema-compiler/src/adapter/SnowflakeQuery.ts @@ -115,6 +115,10 @@ export class SnowflakeQuery extends BaseQuery { templates.functions.BTRIM = 'TRIM({{ args_concat }})'; templates.functions.STRING_AGG = 'LISTAGG({% if distinct %}DISTINCT {% endif %}{{ args_concat }})'; templates.expressions.extract = 'EXTRACT({{ date_part }} FROM {{ expr }})'; + // Snowflake `/` is decimal division even for integer operands (output scale + // is dividend scale + 6), while this template must keep PostgreSQL integer + // division semantics. TRUNC rounds toward zero, matching PostgreSQL. + templates.expressions.int_division = 'CAST(TRUNC({{ left }} / {{ right }}) AS BIGINT)'; // Snowflake can't EXTRACT(EPOCH FROM ), so the epoch of a timestamp // difference (left - right) is rendered as fractional seconds between them. // TIMESTAMPDIFF is measured once at microsecond granularity (no per-second diff --git a/rust/cubesql/cubesql/src/compile/engine/df/wrapper.rs b/rust/cubesql/cubesql/src/compile/engine/df/wrapper.rs index 998d75d68e5e6..8a91bb781723b 100644 --- a/rust/cubesql/cubesql/src/compile/engine/df/wrapper.rs +++ b/rust/cubesql/cubesql/src/compile/engine/df/wrapper.rs @@ -26,11 +26,17 @@ use cubeclient::models::{V1LoadRequestQuery, V1LoadRequestQueryJoinSubquery}; use datafusion::logical_plan::{ExprVisitable, ExpressionVisitor, Recursion}; use datafusion::{ error::{DataFusionError, Result}, + logical_expr::{ReturnTypeFunction, ScalarFunctionImplementation}, logical_plan::{ - plan::Extension, replace_col, Column, DFSchema, DFSchemaRef, Expr, GroupingSet, JoinType, - LogicalPlan, Operator, UserDefinedLogicalNode, + plan::Extension, replace_col, Column, DFSchema, DFSchemaRef, Expr, ExprRewritable, + ExprRewriter, ExprSchemable, GroupingSet, JoinType, LogicalPlan, Operator, + UserDefinedLogicalNode, + }, + physical_plan::{ + aggregates::AggregateFunction, + functions::{BuiltinScalarFunction, Signature, Volatility}, + udf::ScalarUDF, }, - physical_plan::{aggregates::AggregateFunction, functions::BuiltinScalarFunction}, scalar::ScalarValue, }; use futures::FutureExt; @@ -673,6 +679,82 @@ pub struct SqlGenerationResult { static DATE_PART_REGEX: LazyLock = LazyLock::new(|| Regex::new("^[A-Za-z_ ]+$").unwrap()); +/// Marker function for integer division. DataFusion types `int / int` as integer +/// division (PostgreSQL semantics: truncation toward zero), but `/` in some dialects +/// (Snowflake, BigQuery, MySQL, ...) performs decimal or float division. Integer +/// divisions are rewritten to this marker during SQL generation, and the marker is +/// rendered with the `expressions/int_division` template, so each dialect can map it +/// to a construct that keeps PostgreSQL semantics. The marker never reaches execution. +const INT_DIVISION_MARKER: &str = "__int_division"; + +static INT_DIVISION_UDF: LazyLock> = LazyLock::new(|| { + let fun: ScalarFunctionImplementation = Arc::new(|_| { + Err(DataFusionError::Internal(format!( + "{INT_DIVISION_MARKER} marker is only used for SQL generation and can't be evaluated" + ))) + }); + let return_type: ReturnTypeFunction = Arc::new(|types| Ok(Arc::new(types[0].clone()))); + Arc::new(ScalarUDF::new( + INT_DIVISION_MARKER, + &Signature::any(2, Volatility::Immutable), + &return_type, + &fun, + )) +}); + +fn is_integer_type(data_type: &DataType) -> bool { + matches!( + data_type, + DataType::Int8 + | DataType::Int16 + | DataType::Int32 + | DataType::Int64 + | DataType::UInt8 + | DataType::UInt16 + | DataType::UInt32 + | DataType::UInt64 + ) +} + +/// Replaces `left / right` with the `__int_division` marker when both operands are +/// integers per `schema`. Subtrees whose types can't be resolved are left unchanged. +struct IntDivisionRewriter<'a> { + schema: &'a DFSchema, +} + +impl ExprRewriter for IntDivisionRewriter<'_> { + fn mutate(&mut self, expr: Expr) -> Result { + match expr { + Expr::BinaryExpr { + left, + op: Operator::Divide, + right, + } => { + let operand_is_integer = |operand: &Expr| { + operand + .get_type(self.schema) + .map(|t| is_integer_type(&t)) + .unwrap_or(false) + }; + let is_int_division = operand_is_integer(&left) && operand_is_integer(&right); + if is_int_division { + Ok(Expr::ScalarUDF { + fun: INT_DIVISION_UDF.clone(), + args: vec![*left, *right], + }) + } else { + Ok(Expr::BinaryExpr { + left, + op: Operator::Divide, + right, + }) + } + } + _ => Ok(expr), + } + } +} + macro_rules! generate_sql_for_timestamp { (@generic $literal:ident, $value:ident, $value_block:expr, $sql_generator:expr, $sql_query:expr) => { if let Some($value) = $value { @@ -1515,6 +1597,34 @@ impl WrappedSelectNode { Ok((patches, other, sql_query)) } + /// Rewrites integer divisions in `exprs` to the `__int_division` marker (see + /// [`INT_DIVISION_UDF`]). When a rewrite changes an expression, it is wrapped in an + /// alias with the original expression name, so generated column aliases stay stable. + fn rewrite_int_divisions(exprs: Vec, input_schema: &DFSchema) -> Vec { + exprs + .into_iter() + .map(|expr| { + let Ok(rewritten) = expr.clone().rewrite(&mut IntDivisionRewriter { + schema: input_schema, + }) else { + return expr; + }; + if rewritten == expr { + return expr; + } + match &rewritten { + // Sort and alias expressions can't be wrapped in an alias, and + // their names are not used for generated column aliases + Expr::Sort { .. } | Expr::Alias(_, _) => rewritten, + _ => match expr.name(input_schema) { + Ok(name) => Expr::Alias(Box::new(rewritten), name), + Err(_) => rewritten, + }, + } + }) + .collect() + } + async fn generate_columns( &self, meta: &MetaContext, @@ -1556,9 +1666,22 @@ impl WrappedSelectNode { )) })? .clone(); + + // Integer division type detection needs a schema that resolves input columns + // of this select: `from` combined with any join inputs + let mut input_schema = self.from.schema().as_ref().clone(); + for (join_plan, _, _) in &self.joins { + input_schema = input_schema.join(join_plan.schema()).map_err(|e| { + CubeError::internal(format!( + "Can't join schemas for wrapped select input: {}", + e + )) + })?; + } + let (projection, sql) = Self::generate_column_expr( schema.clone(), - self.projection_expr.iter().cloned(), + Self::rewrite_int_divisions(self.projection_expr.clone(), &input_schema), sql, generator.clone(), column_remapping, @@ -1571,7 +1694,7 @@ impl WrappedSelectNode { let flat_group_expr = extract_exprlist_from_groupping_set(&self.group_expr); let (group_by, sql) = Self::generate_column_expr( schema.clone(), - flat_group_expr.clone(), + Self::rewrite_int_divisions(flat_group_expr.clone(), &input_schema), sql, generator.clone(), column_remapping, @@ -1598,7 +1721,7 @@ impl WrappedSelectNode { let (aggregate, sql) = Self::generate_column_expr( schema.clone(), - aggr_expr.clone(), + Self::rewrite_int_divisions(aggr_expr.clone(), &input_schema), sql, generator.clone(), column_remapping, @@ -1611,7 +1734,7 @@ impl WrappedSelectNode { let (filter, sql) = Self::generate_column_expr( schema.clone(), - self.filter_expr.iter().cloned(), + Self::rewrite_int_divisions(self.filter_expr.clone(), &input_schema), sql, generator.clone(), column_remapping, @@ -1624,7 +1747,7 @@ impl WrappedSelectNode { let (window, sql) = Self::generate_column_expr( schema.clone(), - self.window_expr.iter().cloned(), + Self::rewrite_int_divisions(self.window_expr.clone(), &input_schema), sql, generator.clone(), column_remapping, @@ -1661,7 +1784,7 @@ impl WrappedSelectNode { let (order, sql) = Self::generate_column_expr( schema.clone(), - order_expr, + Self::rewrite_int_divisions(order_expr, &input_schema), sql, generator.clone(), column_remapping, @@ -2969,6 +3092,37 @@ impl WrappedSelectNode { "generate_sql_for_scalar_udf called with non-ScalarUDF expr".to_string(), )); }; + if fun.name == INT_DIVISION_MARKER { + let [left, right]: [Expr; 2] = args.try_into().map_err(|args| { + DataFusionError::Internal(format!( + "{INT_DIVISION_MARKER} marker expects exactly 2 arguments, got {args:?}" + )) + })?; + let (left, sql_query) = Self::generate_sql_for_expr( + sql_query, + sql_generator.clone(), + left, + push_to_cube_context, + subqueries, + )?; + let (right, sql_query) = Self::generate_sql_for_expr( + sql_query, + sql_generator.clone(), + right, + push_to_cube_context, + subqueries, + )?; + let resulting_sql = sql_generator + .get_sql_templates() + .int_division_expr(left, right) + .map_err(|e| { + DataFusionError::Internal(format!( + "Can't generate SQL for integer division: {}", + e + )) + })?; + return Ok((resulting_sql, sql_query)); + } let date_part_err = |dp| { DataFusionError::Internal(format!( "Can't generate SQL for scalar function: date part '{}' is not supported", diff --git a/rust/cubesql/cubesql/src/compile/mod.rs b/rust/cubesql/cubesql/src/compile/mod.rs index cae13145d01e2..694a9ffcf48eb 100644 --- a/rust/cubesql/cubesql/src/compile/mod.rs +++ b/rust/cubesql/cubesql/src/compile/mod.rs @@ -15430,6 +15430,66 @@ ORDER BY "source"."str0" ASC assert!(sql.contains("INTERVAL '7 DAY'")); } + #[tokio::test] + async fn test_int_division_template() { + init_testing_logger(); + + let int_division_templates = vec![( + "expressions/int_division".to_string(), + "CAST(TRUNC({{ left }} / {{ right }}) AS BIGINT)".to_string(), + )]; + + // Integer / integer division must be rendered through the + // expressions/int_division template to keep PostgreSQL integer division + // semantics on dialects where `/` is decimal or float division + let query_plan = convert_select_to_query_plan_customized( + " + SELECT customer_gender, COUNT(*) / NULLIF(COUNT(DISTINCT notes), 0) AS ratio + FROM KibanaSampleDataEcommerce + WHERE LOWER(customer_gender) = 'test' + GROUP BY 1 + ORDER BY 2 DESC + LIMIT 100 + " + .to_string(), + DatabaseProtocol::PostgreSQL, + int_division_templates.clone(), + ) + .await; + + let logical_plan = query_plan.as_logical_plan(); + let sql = logical_plan.find_cube_scan_wrapped_sql().wrapped_sql.sql; + assert!( + sql.contains("CAST(TRUNC("), + "{}", + "int/int division must use expressions/int_division template, got: {sql}" + ); + + // Float division must stay on the plain binary expression template + let query_plan = convert_select_to_query_plan_customized( + " + SELECT customer_gender, SUM(taxful_total_price) / NULLIF(COUNT(DISTINCT notes), 0) AS ratio + FROM KibanaSampleDataEcommerce + WHERE LOWER(customer_gender) = 'test' + GROUP BY 1 + ORDER BY 2 DESC + LIMIT 100 + " + .to_string(), + DatabaseProtocol::PostgreSQL, + int_division_templates, + ) + .await; + + let logical_plan = query_plan.as_logical_plan(); + let sql = logical_plan.find_cube_scan_wrapped_sql().wrapped_sql.sql; + assert!( + !sql.contains("CAST(TRUNC("), + "{}", + "float division must not use expressions/int_division template, got: {sql}" + ); + } + #[tokio::test] async fn test_string_multiply_interval() -> Result<(), CubeError> { init_testing_logger(); diff --git a/rust/cubesql/cubesql/src/compile/test/mod.rs b/rust/cubesql/cubesql/src/compile/test/mod.rs index 3a934c1da5f1a..637ad40a05cbd 100644 --- a/rust/cubesql/cubesql/src/compile/test/mod.rs +++ b/rust/cubesql/cubesql/src/compile/test/mod.rs @@ -672,6 +672,7 @@ pub fn sql_generator( SqlTemplates::new( vec![ ("functions/COALESCE".to_string(), "COALESCE({{ args_concat }})".to_string()), + ("functions/NULLIF".to_string(), "NULLIF({{ args_concat }})".to_string()), ("functions/SUM".to_string(), "SUM({{ args_concat }})".to_string()), ("functions/MIN".to_string(), "MIN({{ args_concat }})".to_string()), ("functions/MAX".to_string(), "MAX({{ args_concat }})".to_string()), @@ -733,6 +734,7 @@ OFFSET {{ offset }}{% endif %}"#.to_string(), "{{expr}} {{quoted_alias}}".to_string(), ), ("expressions/binary".to_string(), "({{ left }} {{ op }} {{ right }})".to_string()), + ("expressions/int_division".to_string(), "({{ left }} / {{ right }})".to_string()), ("expressions/is_null".to_string(), "({{ expr }} IS {% if negate %}NOT {% endif %}NULL)".to_string()), ("expressions/case".to_string(), "CASE{% if expr %} {{ expr }}{% endif %}{% for when, then in when_then %} WHEN {{ when }} THEN {{ then }}{% endfor %}{% if else_expr %} ELSE {{ else_expr }}{% endif %} END".to_string()), ("expressions/sort".to_string(), "{{ expr }} {% if asc %}ASC{% else %}DESC{% endif %}{% if nulls_first %} NULLS FIRST {% endif %}".to_string()), diff --git a/rust/cubesql/cubesql/src/transport/service.rs b/rust/cubesql/cubesql/src/transport/service.rs index 1de989e4b2237..85d0c48aae7e8 100644 --- a/rust/cubesql/cubesql/src/transport/service.rs +++ b/rust/cubesql/cubesql/src/transport/service.rs @@ -765,6 +765,13 @@ impl SqlTemplates { ) } + pub fn int_division_expr(&self, left: String, right: String) -> Result { + self.render_template( + "expressions/int_division", + context! { left => left, right => right }, + ) + } + pub fn is_null_expr(&self, expr: String, negate: bool) -> Result { self.render_template( "expressions/is_null", From eb87c43e42a9b600748268e82d8b4d78a3082ce2 Mon Sep 17 00:00:00 2001 From: Alex Qyoun-ae <4062971+MazterQyou@users.noreply.github.com> Date: Wed, 22 Jul 2026 21:17:48 +0400 Subject: [PATCH 2/3] fix(cubesql): Escape PatchMeasure filters for member expression evaluation (#11329) Signed-off-by: Alex Qyoun-ae <4062971+MazterQyou@users.noreply.github.com> --- .../cubesql/src/compile/engine/df/wrapper.rs | 1 + .../cubesql/src/compile/test/test_wrapper.rs | 55 +++++++++++++++++++ 2 files changed, 56 insertions(+) diff --git a/rust/cubesql/cubesql/src/compile/engine/df/wrapper.rs b/rust/cubesql/cubesql/src/compile/engine/df/wrapper.rs index 8a91bb781723b..a61b1a9d1fd2f 100644 --- a/rust/cubesql/cubesql/src/compile/engine/df/wrapper.rs +++ b/rust/cubesql/cubesql/src/compile/engine/df/wrapper.rs @@ -1483,6 +1483,7 @@ impl WrappedSelectNode { Some(push_to_cube_context), subqueries, )?; + let filter = Self::escape_interpolation_quotes(filter, true); let used_cubes = Self::prepare_used_cubes(&used_members); diff --git a/rust/cubesql/cubesql/src/compile/test/test_wrapper.rs b/rust/cubesql/cubesql/src/compile/test/test_wrapper.rs index 56e04ff534e0d..c0971d3c3ea24 100644 --- a/rust/cubesql/cubesql/src/compile/test/test_wrapper.rs +++ b/rust/cubesql/cubesql/src/compile/test/test_wrapper.rs @@ -2555,6 +2555,61 @@ GROUP BY ); } +/// Backslashes in patch measure filter SQL must be escaped for JS template literal +/// evaluation on the Cube side, same as SqlFunction member expressions +#[tokio::test] +async fn test_ad_hoc_measure_filter_like_escape() { + if !Rewriter::sql_push_down_enabled() { + return; + } + init_testing_logger(); + + let query_plan = convert_select_to_query_plan_customized( + // language=PostgreSQL + r#"SELECT + SUM( + CASE (dim_str0 ILIKE '%foo%') + WHEN TRUE + THEN maxPrice + ELSE NULL + END + ) +FROM MultiTypeCube +;"# + .to_string(), + DatabaseProtocol::PostgreSQL, + vec![( + "expressions/ilike".to_string(), + "{{ expr }} {% if negated %}NOT {% endif %}ILIKE {{ pattern }}\ + {% if default_escape %} ESCAPE '\\\\'{% endif %}" + .to_string(), + )], + ) + .await; + + let physical_plan = query_plan.as_physical_plan().await.unwrap(); + println!( + "Physical plan: {}", + displayable(physical_plan.as_ref()).indent() + ); + + let request = query_plan + .as_logical_plan() + .find_cube_scan_wrapped_sql() + .request; + let measures = request.measures.unwrap(); + assert_eq!(measures.len(), 1); + let measure: serde_json::Value = serde_json::from_str(&measures[0]).unwrap(); + let filter_sql = measure["expr"]["addFilters"][0]["sql"].as_str().unwrap(); + // Member expression SQL is evaluated as a JS template literal on the Cube side, + // which collapses `\\` back to `\`, so the ESCAPE clause must carry doubled backslashes + assert!( + filter_sql.contains(r#"ILIKE $0$ ESCAPE '\\\\'"#), + "expected escaped ESCAPE clause in patch measure filter, got: {}", + filter_sql + ); +} + #[tokio::test] async fn test_wrapper_between() { if !Rewriter::sql_push_down_enabled() { From c9bf77684d743344134a7af0c3f42e52e74c54b1 Mon Sep 17 00:00:00 2001 From: Alex Qyoun-ae <4062971+MazterQyou@users.noreply.github.com> Date: Wed, 22 Jul 2026 23:13:00 +0400 Subject: [PATCH 3/3] feat(cubesql): Support parsing date-only timestamp strings (#11316) Signed-off-by: Alex Qyoun-ae <4062971+MazterQyou@users.noreply.github.com> --- packages/cubejs-backend-native/Cargo.lock | 16 +++--- .../cubejs-druid-driver/src/DruidQuery.ts | 5 ++ .../src/FireboltQuery.ts | 5 ++ packages/cubejs-ksql-driver/src/KsqlQuery.ts | 6 +++ .../src/adapter/BaseQuery.js | 5 +- .../src/adapter/MssqlQuery.ts | 5 ++ .../src/adapter/MysqlQuery.ts | 6 +++ .../src/adapter/OracleQuery.ts | 4 ++ rust/cubesql/Cargo.lock | 16 +++--- rust/cubesql/cubesql/Cargo.toml | 2 +- rust/cubesql/cubesql/src/compile/mod.rs | 53 +++++++++++++++++++ 11 files changed, 105 insertions(+), 18 deletions(-) diff --git a/packages/cubejs-backend-native/Cargo.lock b/packages/cubejs-backend-native/Cargo.lock index b868307a87269..fcd1c702353b5 100644 --- a/packages/cubejs-backend-native/Cargo.lock +++ b/packages/cubejs-backend-native/Cargo.lock @@ -103,7 +103,7 @@ checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" [[package]] name = "arrow" version = "13.0.0" -source = "git+https://github.com/cube-js/arrow-rs.git?rev=de57e9799dcfde7bdd02885e63b65e3499f192e5#de57e9799dcfde7bdd02885e63b65e3499f192e5" +source = "git+https://github.com/cube-js/arrow-rs.git?rev=837ddde06037034056b9faced3d8ea7073529cfc#837ddde06037034056b9faced3d8ea7073529cfc" dependencies = [ "bitflags 1.3.2", "chrono", @@ -897,7 +897,7 @@ dependencies = [ [[package]] name = "cube-ext" version = "1.0.0" -source = "git+https://github.com/cube-js/arrow-datafusion.git?rev=b01223b6454b0d72154d15a88aec76efc9da726d#b01223b6454b0d72154d15a88aec76efc9da726d" +source = "git+https://github.com/cube-js/arrow-datafusion.git?rev=d203d8771a9673fb676b44dce1f99e91634b4080#d203d8771a9673fb676b44dce1f99e91634b4080" dependencies = [ "arrow 13.0.0", "chrono", @@ -1073,7 +1073,7 @@ checksum = "e8566979429cf69b49a5c740c60791108e86440e8be149bbea4fe54d2c32d6e2" [[package]] name = "datafusion" version = "7.0.0" -source = "git+https://github.com/cube-js/arrow-datafusion.git?rev=b01223b6454b0d72154d15a88aec76efc9da726d#b01223b6454b0d72154d15a88aec76efc9da726d" +source = "git+https://github.com/cube-js/arrow-datafusion.git?rev=d203d8771a9673fb676b44dce1f99e91634b4080#d203d8771a9673fb676b44dce1f99e91634b4080" dependencies = [ "ahash 0.7.8", "arrow 13.0.0", @@ -1106,7 +1106,7 @@ dependencies = [ [[package]] name = "datafusion-common" version = "7.0.0" -source = "git+https://github.com/cube-js/arrow-datafusion.git?rev=b01223b6454b0d72154d15a88aec76efc9da726d#b01223b6454b0d72154d15a88aec76efc9da726d" +source = "git+https://github.com/cube-js/arrow-datafusion.git?rev=d203d8771a9673fb676b44dce1f99e91634b4080#d203d8771a9673fb676b44dce1f99e91634b4080" dependencies = [ "arrow 13.0.0", "ordered-float 2.10.1", @@ -1117,7 +1117,7 @@ dependencies = [ [[package]] name = "datafusion-data-access" version = "1.0.0" -source = "git+https://github.com/cube-js/arrow-datafusion.git?rev=b01223b6454b0d72154d15a88aec76efc9da726d#b01223b6454b0d72154d15a88aec76efc9da726d" +source = "git+https://github.com/cube-js/arrow-datafusion.git?rev=d203d8771a9673fb676b44dce1f99e91634b4080#d203d8771a9673fb676b44dce1f99e91634b4080" dependencies = [ "async-trait", "chrono", @@ -1130,7 +1130,7 @@ dependencies = [ [[package]] name = "datafusion-expr" version = "7.0.0" -source = "git+https://github.com/cube-js/arrow-datafusion.git?rev=b01223b6454b0d72154d15a88aec76efc9da726d#b01223b6454b0d72154d15a88aec76efc9da726d" +source = "git+https://github.com/cube-js/arrow-datafusion.git?rev=d203d8771a9673fb676b44dce1f99e91634b4080#d203d8771a9673fb676b44dce1f99e91634b4080" dependencies = [ "ahash 0.7.8", "arrow 13.0.0", @@ -1141,7 +1141,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" version = "7.0.0" -source = "git+https://github.com/cube-js/arrow-datafusion.git?rev=b01223b6454b0d72154d15a88aec76efc9da726d#b01223b6454b0d72154d15a88aec76efc9da726d" +source = "git+https://github.com/cube-js/arrow-datafusion.git?rev=d203d8771a9673fb676b44dce1f99e91634b4080#d203d8771a9673fb676b44dce1f99e91634b4080" dependencies = [ "ahash 0.7.8", "arrow 13.0.0", @@ -2525,7 +2525,7 @@ dependencies = [ [[package]] name = "parquet" version = "13.0.0" -source = "git+https://github.com/cube-js/arrow-rs.git?rev=de57e9799dcfde7bdd02885e63b65e3499f192e5#de57e9799dcfde7bdd02885e63b65e3499f192e5" +source = "git+https://github.com/cube-js/arrow-rs.git?rev=837ddde06037034056b9faced3d8ea7073529cfc#837ddde06037034056b9faced3d8ea7073529cfc" dependencies = [ "arrow 13.0.0", "base64 0.13.1", diff --git a/packages/cubejs-druid-driver/src/DruidQuery.ts b/packages/cubejs-druid-driver/src/DruidQuery.ts index 8b3d84a1c8af3..69c1d6d289d0c 100644 --- a/packages/cubejs-druid-driver/src/DruidQuery.ts +++ b/packages/cubejs-druid-driver/src/DruidQuery.ts @@ -57,6 +57,11 @@ export class DruidQuery extends BaseQuery { // Druid doesn't support ILIKE, so case-insensitive matching is emulated with LOWER(...) LIKE CONCAT(...) templates.expressions.ilike = 'LOWER({{ expr }}) {% if negated %}NOT {% endif %}LIKE LOWER({{ pattern }})'; + // Timestamp constants arrive as ISO-8601 UTC strings ('2021-01-01T00:00:00.000Z'); + // TIME_PARSE without a pattern parses ISO-8601, which is also Druid's native + // timestamp format. The base template renders the value bare, which is invalid + // syntax + templates.expressions.timestamp_literal = 'TIME_PARSE(\'{{ value }}\')'; delete templates.expressions.like_escape; templates.filters.like_pattern = 'CONCAT({% if start_wild %}\'%\'{% else %}\'\'{% endif %}, LOWER({{ value }}), {% if end_wild %}\'%\'{% else %}\'\'{% endif %})'; templates.tesseract.ilike = 'LOWER({{ expr }}) {% if negated %}NOT {% endif %}LIKE {{ pattern }}'; diff --git a/packages/cubejs-firebolt-driver/src/FireboltQuery.ts b/packages/cubejs-firebolt-driver/src/FireboltQuery.ts index a168e8a5f459a..2b42e699afa64 100644 --- a/packages/cubejs-firebolt-driver/src/FireboltQuery.ts +++ b/packages/cubejs-firebolt-driver/src/FireboltQuery.ts @@ -51,6 +51,11 @@ export class FireboltQuery extends BaseQuery { public sqlTemplates() { const templates = super.sqlTemplates(); + // Timestamp constants arrive as ISO-8601 UTC strings ('2021-01-01T00:00:00.000Z'); + // the TIMESTAMPTZ literal accepts a 'T' separator and the 'Z' UTC designator per + // Firebolt's documented ISO-8601/RFC-3339 grammar. The base template renders the + // value bare, which is invalid syntax + templates.expressions.timestamp_literal = 'TIMESTAMPTZ \'{{ value }}\''; templates.tesseract.bool_param_cast = 'CAST({{ expr }} AS BOOLEAN)'; return templates; } diff --git a/packages/cubejs-ksql-driver/src/KsqlQuery.ts b/packages/cubejs-ksql-driver/src/KsqlQuery.ts index 765d7824b6714..c67fca19847d9 100644 --- a/packages/cubejs-ksql-driver/src/KsqlQuery.ts +++ b/packages/cubejs-ksql-driver/src/KsqlQuery.ts @@ -65,6 +65,12 @@ export class KsqlQuery extends BaseQuery { // ksqlDB has no `||` string operator — concatenation and LIKE-pattern // wildcards must use CONCAT (mirrors concatStringsSql / KsqlFilter). templates.expressions.concat_strings = 'CONCAT({{ strings | join(\', \') }})'; + // Timestamp constants arrive as ISO-8601 UTC strings ('2021-01-01T00:00:00.000Z'). + // ksqlDB silently returns NULL when casting strings with a trailing 'Z' + // (confluentinc/ksql#9094), so the markers are stripped and the UTC zone is passed + // to PARSE_TIMESTAMP explicitly. The base template renders the value bare, which + // is invalid syntax + templates.expressions.timestamp_literal = 'PARSE_TIMESTAMP(\'{{ value | replace("T", " ") | replace("Z", "") }}\', \'yyyy-MM-dd HH:mm:ss.SSS\', \'UTC\')'; templates.filters.like_pattern = '{% if start_wild or end_wild %}CONCAT(' + '{% if start_wild %}\'%\', {% endif %}{{ value }}{% if end_wild %}, \'%\'{% endif %})' + diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js index 7f44fe3f44aab..b6f9dbd3bd837 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js +++ b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js @@ -4637,7 +4637,10 @@ export class BaseQuery { wrap_segment_select: '{{ expr }}', wrap_segment_filter: '{{ expr }}', rolling_window_expr_timestamp_cast: '{{ value }}', - timestamp_literal: '{{ value }}', + // Timestamp constants arrive as ISO-8601 UTC strings ('2021-01-01T00:00:00.000Z'). + // ANSI CAST of the quoted value is the most portable default; dialects override + // with their exact parsing construct. A bare unquoted value is never valid SQL + timestamp_literal: 'CAST(\'{{ value }}\' AS TIMESTAMP)', between: '{{ expr }} {% if negated %}NOT {% endif %}BETWEEN {{ low }} AND {{ high }}', }, tesseract: { diff --git a/packages/cubejs-schema-compiler/src/adapter/MssqlQuery.ts b/packages/cubejs-schema-compiler/src/adapter/MssqlQuery.ts index 48915ddf41780..e91da7efa9ce4 100644 --- a/packages/cubejs-schema-compiler/src/adapter/MssqlQuery.ts +++ b/packages/cubejs-schema-compiler/src/adapter/MssqlQuery.ts @@ -281,6 +281,11 @@ export class MssqlQuery extends BaseQuery { templates.expressions.concat_strings = '{{ strings | join(\' + \' ) }}'; // NOTE: this template contains a comma; two order expressions are being generated templates.expressions.sort = '{{ expr }} IS NULL {% if nulls_first %}DESC{% else %}ASC{% endif %}, {{ expr }} {% if asc %}ASC{% else %}DESC{% endif %}'; + // Timestamp constants arrive as ISO-8601 UTC strings ('2021-01-01T00:00:00.000Z'); + // CONVERT style 127 is defined as exactly this format (yyyy-mm-ddThh:mi:ss.mmmZ, + // "ISO8601 with time zone Z"). The base template renders the value bare, which is + // invalid T-SQL syntax + templates.expressions.timestamp_literal = 'CONVERT(DATETIME2, \'{{ value }}\', 127)'; templates.types.string = 'VARCHAR'; templates.types.boolean = 'BIT'; templates.types.integer = 'INT'; diff --git a/packages/cubejs-schema-compiler/src/adapter/MysqlQuery.ts b/packages/cubejs-schema-compiler/src/adapter/MysqlQuery.ts index e3cbd2e4a3aa8..a19744df4b80b 100644 --- a/packages/cubejs-schema-compiler/src/adapter/MysqlQuery.ts +++ b/packages/cubejs-schema-compiler/src/adapter/MysqlQuery.ts @@ -196,6 +196,12 @@ export class MysqlQuery extends BaseQuery { // MySQL `/` returns DECIMAL even for integer operands; DIV discards the // fractional part (truncation toward zero), matching PostgreSQL (-5 DIV 2 = -2) templates.expressions.int_division = '({{ left }} DIV {{ right }})'; + // Timestamp constants arrive as ISO-8601 UTC strings ('2021-01-01T00:00:00.000Z'). + // MySQL parses the 'T'/'Z' markers only with a "Truncated incorrect datetime value" + // warning and ignores the zone, so both markers are stripped instead. The driver + // pins the session time zone to UTC, which keeps the naive literal on the same + // instant. The base template renders the value bare, which is invalid MySQL syntax + templates.expressions.timestamp_literal = 'TIMESTAMP(\'{{ value | replace("T", " ") | replace("Z", "") }}\')'; delete templates.expressions.ilike; templates.types.string = 'CHAR'; templates.types.boolean = 'TINYINT'; diff --git a/packages/cubejs-schema-compiler/src/adapter/OracleQuery.ts b/packages/cubejs-schema-compiler/src/adapter/OracleQuery.ts index b9ab41d0dc9cb..787fd5190935d 100644 --- a/packages/cubejs-schema-compiler/src/adapter/OracleQuery.ts +++ b/packages/cubejs-schema-compiler/src/adapter/OracleQuery.ts @@ -221,6 +221,10 @@ export class OracleQuery extends BaseQuery { // Oracle `/` on NUMBER keeps the fractional part; TRUNC drops decimal digits // (truncation toward zero), matching PostgreSQL integer division templates.expressions.int_division = 'TRUNC({{ left }} / {{ right }})'; + // Timestamp constants arrive as ISO-8601 UTC strings ('2021-01-01T00:00:00.000Z'); + // the 'T'/'Z' markers are consumed as literal chunks in the format mask. The base + // template renders the value bare, which is invalid Oracle syntax + templates.expressions.timestamp_literal = 'TO_TIMESTAMP(\'{{ value }}\', \'YYYY-MM-DD"T"HH24:MI:SS.FF3"Z"\')'; // Oracle does not support positional GROUP BY — group by expressions. templates.statements.group_by_exprs = '{{ group_by | map(attribute=\'expr\') | join(\', \') }}'; // No `AS` before the FROM subquery alias, and Oracle row-limiting syntax. diff --git a/rust/cubesql/Cargo.lock b/rust/cubesql/Cargo.lock index ca57b5f4612fb..f703f45b452d7 100644 --- a/rust/cubesql/Cargo.lock +++ b/rust/cubesql/Cargo.lock @@ -112,7 +112,7 @@ checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6" [[package]] name = "arrow" version = "13.0.0" -source = "git+https://github.com/cube-js/arrow-rs.git?rev=de57e9799dcfde7bdd02885e63b65e3499f192e5#de57e9799dcfde7bdd02885e63b65e3499f192e5" +source = "git+https://github.com/cube-js/arrow-rs.git?rev=837ddde06037034056b9faced3d8ea7073529cfc#837ddde06037034056b9faced3d8ea7073529cfc" dependencies = [ "bitflags 1.3.2", "chrono", @@ -698,7 +698,7 @@ dependencies = [ [[package]] name = "cube-ext" version = "1.0.0" -source = "git+https://github.com/cube-js/arrow-datafusion.git?rev=b01223b6454b0d72154d15a88aec76efc9da726d#b01223b6454b0d72154d15a88aec76efc9da726d" +source = "git+https://github.com/cube-js/arrow-datafusion.git?rev=d203d8771a9673fb676b44dce1f99e91634b4080#d203d8771a9673fb676b44dce1f99e91634b4080" dependencies = [ "arrow", "chrono", @@ -822,7 +822,7 @@ dependencies = [ [[package]] name = "datafusion" version = "7.0.0" -source = "git+https://github.com/cube-js/arrow-datafusion.git?rev=b01223b6454b0d72154d15a88aec76efc9da726d#b01223b6454b0d72154d15a88aec76efc9da726d" +source = "git+https://github.com/cube-js/arrow-datafusion.git?rev=d203d8771a9673fb676b44dce1f99e91634b4080#d203d8771a9673fb676b44dce1f99e91634b4080" dependencies = [ "ahash 0.7.8", "arrow", @@ -855,7 +855,7 @@ dependencies = [ [[package]] name = "datafusion-common" version = "7.0.0" -source = "git+https://github.com/cube-js/arrow-datafusion.git?rev=b01223b6454b0d72154d15a88aec76efc9da726d#b01223b6454b0d72154d15a88aec76efc9da726d" +source = "git+https://github.com/cube-js/arrow-datafusion.git?rev=d203d8771a9673fb676b44dce1f99e91634b4080#d203d8771a9673fb676b44dce1f99e91634b4080" dependencies = [ "arrow", "ordered-float 2.10.0", @@ -866,7 +866,7 @@ dependencies = [ [[package]] name = "datafusion-data-access" version = "1.0.0" -source = "git+https://github.com/cube-js/arrow-datafusion.git?rev=b01223b6454b0d72154d15a88aec76efc9da726d#b01223b6454b0d72154d15a88aec76efc9da726d" +source = "git+https://github.com/cube-js/arrow-datafusion.git?rev=d203d8771a9673fb676b44dce1f99e91634b4080#d203d8771a9673fb676b44dce1f99e91634b4080" dependencies = [ "async-trait", "chrono", @@ -879,7 +879,7 @@ dependencies = [ [[package]] name = "datafusion-expr" version = "7.0.0" -source = "git+https://github.com/cube-js/arrow-datafusion.git?rev=b01223b6454b0d72154d15a88aec76efc9da726d#b01223b6454b0d72154d15a88aec76efc9da726d" +source = "git+https://github.com/cube-js/arrow-datafusion.git?rev=d203d8771a9673fb676b44dce1f99e91634b4080#d203d8771a9673fb676b44dce1f99e91634b4080" dependencies = [ "ahash 0.7.8", "arrow", @@ -890,7 +890,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" version = "7.0.0" -source = "git+https://github.com/cube-js/arrow-datafusion.git?rev=b01223b6454b0d72154d15a88aec76efc9da726d#b01223b6454b0d72154d15a88aec76efc9da726d" +source = "git+https://github.com/cube-js/arrow-datafusion.git?rev=d203d8771a9673fb676b44dce1f99e91634b4080#d203d8771a9673fb676b44dce1f99e91634b4080" dependencies = [ "ahash 0.7.8", "arrow", @@ -2104,7 +2104,7 @@ dependencies = [ [[package]] name = "parquet" version = "13.0.0" -source = "git+https://github.com/cube-js/arrow-rs.git?rev=de57e9799dcfde7bdd02885e63b65e3499f192e5#de57e9799dcfde7bdd02885e63b65e3499f192e5" +source = "git+https://github.com/cube-js/arrow-rs.git?rev=837ddde06037034056b9faced3d8ea7073529cfc#837ddde06037034056b9faced3d8ea7073529cfc" dependencies = [ "arrow", "base64 0.13.0", diff --git a/rust/cubesql/cubesql/Cargo.toml b/rust/cubesql/cubesql/Cargo.toml index e25b02357986c..6dedf834d8ab5 100644 --- a/rust/cubesql/cubesql/Cargo.toml +++ b/rust/cubesql/cubesql/Cargo.toml @@ -10,7 +10,7 @@ homepage = "https://cube.dev" [dependencies] arc-swap = "1" -datafusion = { git = 'https://github.com/cube-js/arrow-datafusion.git', rev = "b01223b6454b0d72154d15a88aec76efc9da726d", default-features = false, features = [ +datafusion = { git = 'https://github.com/cube-js/arrow-datafusion.git', rev = "d203d8771a9673fb676b44dce1f99e91634b4080", default-features = false, features = [ "regex_expressions", "unicode_expressions", ] } diff --git a/rust/cubesql/cubesql/src/compile/mod.rs b/rust/cubesql/cubesql/src/compile/mod.rs index 694a9ffcf48eb..669e2f4565e33 100644 --- a/rust/cubesql/cubesql/src/compile/mod.rs +++ b/rust/cubesql/cubesql/src/compile/mod.rs @@ -2928,6 +2928,19 @@ limit ])), }]) ), + // DATE_ADD with a date-only string argument + ( + "COUNT(*), DATE(order_date) AS __timestamp".to_string(), + "order_date >= DATE_ADD('2021-09-30', INTERVAL '-30' DAY) AND order_date < '2021-09-07'".to_string(), + Some(vec![V1LoadRequestQueryTimeDimension { + dimension: "KibanaSampleDataEcommerce.order_date".to_string(), + granularity: Some("day".to_string()), + date_range: Some(json!(vec![ + "2021-08-31T00:00:00.000Z".to_string(), + "2021-09-06T23:59:59.999Z".to_string() + ])), + }]) + ), // Column precedence vs projection alias ( "COUNT(*), DATE(order_date) AS order_date".to_string(), @@ -15490,6 +15503,46 @@ ORDER BY "source"."str0" ASC ); } + #[tokio::test] + async fn test_timestamp_literal_from_date_only_string() { + if !Rewriter::sql_push_down_enabled() { + return; + } + init_testing_logger(); + + // MySQL-style timestamp_literal template: ISO-8601 'T'/'Z' markers are + // stripped with the replace filter. A date-only string cast to timestamp + // constant-folds, so the resulting constant must be rendered through the + // timestamp_literal template instead of being emitted bare + let templates = vec![( + "expressions/timestamp_literal".to_string(), + "TIMESTAMP('{{ value | replace(\"T\", \" \") | replace(\"Z\", \"\") }}')".to_string(), + )]; + + let query_plan = convert_select_to_query_plan_customized( + r#" + SELECT customer_gender + FROM KibanaSampleDataEcommerce + WHERE DATE_ADD(order_date, INTERVAL '2 DAY') < CAST('2021-01-01' AS TIMESTAMP) + GROUP BY 1 + "# + .to_string(), + DatabaseProtocol::PostgreSQL, + templates, + ) + .await; + + let request = query_plan + .as_logical_plan() + .find_cube_scan_wrapped_sql() + .request; + let request_json = serde_json::to_string(&request).unwrap(); + assert!( + request_json.contains("TIMESTAMP('2021-01-01 00:00:00.000')"), + "{}", "timestamp literal must render through the timestamp_literal template, got: {request_json}" + ); + } + #[tokio::test] async fn test_string_multiply_interval() -> Result<(), CubeError> { init_testing_logger();