From 91370a09c2d2077492ee1eb766e2023c8f013b38 Mon Sep 17 00:00:00 2001 From: Shehab Amin <11789402+shehabgamin@users.noreply.github.com> Date: Thu, 25 Jun 2026 23:35:02 -0700 Subject: [PATCH 001/131] refactor: pivot (#2138) --- .../sail-plan/src/resolver/query/aggregate.rs | 27 ++- .../sail-plan/src/resolver/query/pivoting.rs | 178 ++++++++++++--- .../tests/spark/dataframe/test_pivot.py | 205 ++++++++++++++++-- 3 files changed, 353 insertions(+), 57 deletions(-) diff --git a/crates/sail-plan/src/resolver/query/aggregate.rs b/crates/sail-plan/src/resolver/query/aggregate.rs index c321701bd..def69f211 100644 --- a/crates/sail-plan/src/resolver/query/aggregate.rs +++ b/crates/sail-plan/src/resolver/query/aggregate.rs @@ -2,7 +2,7 @@ use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode, Tre use datafusion_common::{ Column, DFSchemaRef, DataFusionError, Result as DataFusionResult, ScalarValue, }; -use datafusion_expr::utils::{expr_as_column_expr, find_aggregate_exprs}; +use datafusion_expr::utils::find_aggregate_exprs; use datafusion_expr::{ bitwise_and, bitwise_shift_right, cast, Aggregate, Expr, LogicalPlan, LogicalPlanBuilder, Volatility, @@ -598,7 +598,10 @@ impl PlanResolver<'_> { Ok(expr .transform_down(|e| { if base.contains(&e) { - Ok(Transformed::yes(expr_as_column_expr(&e, plan)?)) + Ok(Transformed::yes( + Self::expr_as_column_expr(&e, plan) + .map_err(|e| DataFusionError::External(Box::new(e)))?, + )) } else { Ok(Transformed::no(e)) } @@ -606,6 +609,26 @@ impl PlanResolver<'_> { .data()?) } + // Modification of DataFusion's `expr_as_column_expr` + fn expr_as_column_expr(expr: &Expr, plan: &LogicalPlan) -> PlanResult { + match expr { + Expr::Column(column) => { + let result = plan + .schema() + .qualified_field_from_column(column) + .or_else(|_| { + let column = Column::new_unqualified(column.name.clone()); + plan.schema().qualified_field_from_column(&column) + })?; + let (qualifier, field) = result; + Ok(Expr::from(Column::from((qualifier, field)))) + } + _ => Ok(Expr::Column(Column::from_name( + expr.schema_name().to_string(), + ))), + } + } + /// Spark CheckAnalysis: GroupedAgg Pandas/Arrow UDFs cannot be mixed with regular /// (non-UDF) aggregate functions in the same `.agg()` call. fn check_no_mixed_grouped_agg_udf(projections: &[NamedExpr]) -> PlanResult<()> { diff --git a/crates/sail-plan/src/resolver/query/pivoting.rs b/crates/sail-plan/src/resolver/query/pivoting.rs index fdded7a06..0b70ae8fb 100644 --- a/crates/sail-plan/src/resolver/query/pivoting.rs +++ b/crates/sail-plan/src/resolver/query/pivoting.rs @@ -2,9 +2,9 @@ use std::collections::{HashMap, HashSet}; use std::sync::Arc; use arrow::datatypes::DataType; -use datafusion_common::tree_node::{Transformed, TreeNode}; +use datafusion_common::tree_node::{Transformed, TreeNode, TreeNodeRecursion}; use datafusion_common::{Column, DFSchema, DFSchemaRef, ScalarValue}; -use datafusion_expr::utils::find_aggregate_exprs; +use datafusion_expr::expr::NullTreatment; use datafusion_expr::{ col, expr, lit, ExprSchemable, LogicalPlan, LogicalPlanBuilder, Projection, ScalarUDF, }; @@ -40,28 +40,26 @@ impl PlanResolver<'_> { let input = self.resolve_query_plan(*input, state).await?; let schema = input.schema().clone(); - // Only a single pivot column is supported. let mut pivot_columns = self.resolve_expressions(columns, &schema, state).await?; - if pivot_columns.len() != 1 { - return Err(PlanError::todo("pivot with multiple pivot columns")); - } - let pivot_column = pivot_columns.swap_remove(0); + let column_arity = pivot_columns.len(); + let pivot_column = match column_arity { + 0 => { + return Err(PlanError::AnalysisError( + "pivot column required".to_string(), + )) + } + 1 => pivot_columns.swap_remove(0), + _ => make_pivot_struct(pivot_columns), + }; let aggregates = self .resolve_named_expressions(aggregate, &schema, state) .await?; - // Spark requires every pivot expression to contain an aggregate function. Reject a bare - // non-aggregate expression up front with a clear error instead of letting it fail - // cryptically (or pass through unfiltered) during the aggregate rewrite. + // Spark allows aggregate expressions and pure literals, but rejects columns outside an + // aggregate function. for agg in &aggregates { - if find_aggregate_exprs(std::slice::from_ref(&agg.expr)).is_empty() { - return Err(PlanError::AnalysisError(format!( - "Aggregate expression required for pivot, but '{}' did not appear in any \ - aggregate function.", - agg.name.join(".") - ))); - } + check_valid_pivot_aggregate(&agg.expr, state)?; } // For SQL pivots the grouping list is empty; the implicit grouping columns are @@ -79,7 +77,8 @@ impl PlanResolver<'_> { // given explicitly (`... IN (v1, v2)`); otherwise infer the distinct values // from the data (matching Spark's `pivot(column)`). let pivot_values: Vec<(ScalarValue, Option)> = if values.is_empty() { - self.infer_pivot_values(&input, &pivot_column) + let pivot_column_name = pivot_column_display_name(&pivot_column, state)?; + self.infer_pivot_values(&input, &pivot_column, &pivot_column_name) .await? .into_iter() .map(|scalar| (scalar, None)) @@ -96,17 +95,21 @@ impl PlanResolver<'_> { values: exprs, alias, } = value; - if exprs.len() != 1 { - return Err(PlanError::todo( - "pivot value with multiple expressions (struct pivot)", - )); + if exprs.len() != column_arity { + return Err(PlanError::AnalysisError(format!( + "pivot value has {} expressions, but pivot column has {}", + exprs.len(), + column_arity + ))); } - let Some(expr) = exprs.into_iter().next() else { - return Err(PlanError::AnalysisError( - "pivot value must have a value".to_string(), - )); + let mut exprs = self + .resolve_expressions(exprs, &empty_schema, state) + .await?; + let expr = if column_arity == 1 { + exprs.swap_remove(0) + } else { + make_pivot_struct(exprs) }; - let expr = self.resolve_expression(expr, &empty_schema, state).await?; let scalar = evaluator .evaluate(&expr) .map_err(|e| PlanError::invalid(e.to_string()))?; @@ -115,6 +118,19 @@ impl PlanResolver<'_> { resolved }; + // Spark casts each pivot value to the pivot column's type before comparing + // (`Cast(value, pivotColumn.dataType)`), so an `int` literal matches a `bigint` column and + // a struct value matches the struct column's field types. The output name still uses the + // raw value (Spark casts the original value to string), so it is taken from `scalar` below + // before the cast. + let column_type = pivot_column.get_type(&schema)?; + let spark_uses_pivot_first = aggregates + .iter() + .map(|agg| agg.expr.get_type(&schema)) + .collect::, _>>()? + .iter() + .all(spark_pivot_first_supports_data_type); + let force_first_last_ignore_nulls = !spark_uses_pivot_first; let single_aggregate = aggregates.len() == 1; let mut projections = grouping.clone(); for (scalar, alias) in pivot_values { @@ -130,10 +146,22 @@ impl PlanResolver<'_> { let predicate = if is_null { pivot_column.clone().is_null() } else { - pivot_column.clone().eq(lit(scalar)) + let scalar = scalar + .cast_to(&column_type) + .map_err(|e| PlanError::invalid(e.to_string()))?; + // Spark's PivotFirst fast path creates a NaN column but does not match NaN rows into it. + if spark_uses_pivot_first && scalar_is_nan(&scalar) { + lit(false) + } else { + pivot_column.clone().eq(lit(scalar)) + } }; for agg in &aggregates { - let expr = inject_pivot_filter(agg.expr.clone(), &predicate)?; + let expr = inject_pivot_filter( + agg.expr.clone(), + &predicate, + force_first_last_ignore_nulls, + )?; let name = if single_aggregate { value_name.clone() } else { @@ -156,6 +184,7 @@ impl PlanResolver<'_> { &self, input: &LogicalPlan, pivot_column: &expr::Expr, + pivot_column_name: &str, ) -> PlanResult> { // Spark rejects pivots whose distinct-value count exceeds the configured maximum // (`spark.sql.pivotMaxValues`, default 10000), defined as `DATAFRAME_PIVOT_MAX_VALUES` @@ -170,7 +199,7 @@ impl PlanResolver<'_> { // values into the planner process; the extra row still lets us detect overflow. let plan = LogicalPlanBuilder::from(input.clone()) .aggregate(vec![pivot_column.clone()], Vec::::new())? - .limit(0, Some(max_pivot_values + 1))? + .limit(0, Some(max_pivot_values.saturating_add(1)))? .build()?; let batches = self.ctx.execute_logical_plan(plan).await?.collect().await?; let mut values = Vec::new(); @@ -184,8 +213,10 @@ impl PlanResolver<'_> { // an exact count (which would be capped and misleading) — mirroring Spark's message. if values.len() > max_pivot_values { return Err(PlanError::AnalysisError(format!( - "The pivot column has more than {max_pivot_values} distinct values; this could \ - indicate an error. Specify the pivot values explicitly if this was intended." + "The pivot column {pivot_column_name} has more than {max_pivot_values} distinct \ + values, this could indicate an error. If this was intended, set \ + spark.sql.pivotMaxValues to at least the number of distinct values of the pivot \ + column." ))); } // `partial_cmp` returns `None` only for incomparable values such as float `NaN`. @@ -404,10 +435,86 @@ fn pivot_value_name(scalar: &ScalarValue) -> PlanResult { .map_err(|e| PlanError::invalid(e.to_string())) } +fn make_pivot_struct(exprs: Vec) -> expr::Expr { + ScalarUDF::from(StructFunction::new( + (0..exprs.len()).map(|i| format!("col{}", i + 1)).collect(), + )) + .call(exprs) +} + +fn pivot_column_display_name(expr: &expr::Expr, state: &PlanResolverState) -> PlanResult { + if let expr::Expr::Column(column) = expr { + Ok(state.get_field_info(column.name())?.name().to_string()) + } else { + Ok(expr.to_string()) + } +} + +fn check_valid_pivot_aggregate(expr: &expr::Expr, state: &PlanResolverState) -> PlanResult<()> { + let mut bare_column = None; + + expr.apply(|e| match e { + expr::Expr::AggregateFunction(_) => Ok(TreeNodeRecursion::Jump), + expr::Expr::Column(column) => { + bare_column = Some(column.clone()); + Ok(TreeNodeRecursion::Stop) + } + _ => Ok(TreeNodeRecursion::Continue), + })?; + + if let Some(column) = bare_column { + let name = state + .get_field_info(column.name()) + .map(|info| info.name().to_string()) + .unwrap_or_else(|_| column.name().to_string()); + return Err(PlanError::AnalysisError(format!( + "Aggregate expression required for pivot, but '{name}' did not appear in any \ + aggregate function." + ))); + } + + Ok(()) +} + +fn spark_pivot_first_supports_data_type(data_type: &DataType) -> bool { + matches!( + data_type, + DataType::Boolean + | DataType::Int8 + | DataType::Int16 + | DataType::Int32 + | DataType::Int64 + | DataType::UInt8 + | DataType::UInt16 + | DataType::UInt32 + | DataType::UInt64 + | DataType::Float16 + | DataType::Float32 + | DataType::Float64 + | DataType::Decimal32(_, _) + | DataType::Decimal64(_, _) + | DataType::Decimal128(_, _) + | DataType::Decimal256(_, _) + ) +} + +fn scalar_is_nan(scalar: &ScalarValue) -> bool { + match scalar { + ScalarValue::Float16(Some(value)) => value.is_nan(), + ScalarValue::Float32(Some(value)) => value.is_nan(), + ScalarValue::Float64(Some(value)) => value.is_nan(), + _ => false, + } +} + /// Add `predicate` as a FILTER to every aggregate function inside `expr`, /// AND-combining with any existing filter. Implements Spark's pivot semantics: /// each aggregate is computed only over rows matching the pivot value. -fn inject_pivot_filter(expr: expr::Expr, predicate: &expr::Expr) -> PlanResult { +fn inject_pivot_filter( + expr: expr::Expr, + predicate: &expr::Expr, + force_first_last_ignore_nulls: bool, +) -> PlanResult { Ok(expr .transform(|e| match e { expr::Expr::AggregateFunction(mut func) => { @@ -416,6 +523,11 @@ fn inject_pivot_filter(expr: expr::Expr, predicate: &expr::Expr) -> PlanResult predicate.clone(), }; func.params.filter = Some(Box::new(filter)); + if force_first_last_ignore_nulls + && matches!(func.func.name(), "first_value" | "last_value") + { + func.params.null_treatment = Some(NullTreatment::IgnoreNulls); + } Ok(Transformed::yes(expr::Expr::AggregateFunction(func))) } other => Ok(Transformed::no(other)), diff --git a/python/pysail/tests/spark/dataframe/test_pivot.py b/python/pysail/tests/spark/dataframe/test_pivot.py index ecc13d839..220431437 100644 --- a/python/pysail/tests/spark/dataframe/test_pivot.py +++ b/python/pysail/tests/spark/dataframe/test_pivot.py @@ -31,15 +31,29 @@ def test_pivot_infers_values_sorted(spark): assert_frame_equal(actual, expected) +def test_pivot_infers_numeric_values_sorted_numerically(spark): + actual = ( + spark.createDataFrame( + [("a", 10, 100), ("a", 2, 20), ("b", 2, 5)], + schema="g STRING, k INT, v INT", + ) + .groupBy("g") + .pivot("k") + .sum("v") + .orderBy("g") + .toPandas() + ) + assert list(actual.columns) == ["g", "2", "10"] + expected = pd.DataFrame({"g": ["a", "b"], "2": [20, 5], "10": [100.0, float("nan")]}) + assert_frame_equal(actual, expected) + + def test_pivot_infers_float_values_with_nan_last(spark): # Edge case: pivoting on a floating-point column whose inferred values include NaN. # NaN is incomparable under `partial_cmp`, so the sort falls back to a deterministic # tiebreaker instead of treating NaN as equal to its neighbour (which left the column # order dependent on the non-deterministic aggregate row order). This pins that guarantee: # finite values come first in ascending order, with their `.0` preserved, and NaN sorts last. - # - # One unrelated Spark-parity gap is intentionally NOT asserted here: the `NaN` pivot column - # counts the `k=NaN` row whereas Spark yields NULL (see the dedicated xfail below). actual = ( spark.createDataFrame( [ @@ -57,6 +71,7 @@ def test_pivot_infers_float_values_with_nan_last(spark): .toPandas() ) assert list(actual.columns) == ["g", "1.0", "2.0", "NaN"] + assert actual["NaN"].isna().all() # Sanity: the first finite column (k=1.0) holds its sums for both groups. assert actual.set_index("g")["1.0"].to_dict() == {"a": 10, "b": 5} @@ -97,6 +112,103 @@ def test_pivot_allows_duplicate_aliases(spark): assert actual.values.tolist() == [[2012, 20000, 10000], [2013, 30000, 48000]] +def test_pivot_allows_literal_aggregate_expression(spark): + actual = spark.sql(""" + SELECT * FROM ( + SELECT g, k FROM VALUES + ('a', 'x'), + ('a', 'y'), + ('b', 'x') + AS t(g, k) + ) PIVOT (1 FOR (k) IN ('x', 'y')) + ORDER BY g + """).toPandas() + assert list(actual.columns) == ["g", "x", "y"] + assert actual.values.tolist() == [["a", 1, 1], ["b", 1, 1]] + + +def test_pivot_rejects_bare_column_outside_aggregate(spark): + with pytest.raises( + Exception, + match=r"Aggregate expression required for pivot, but 'v' did not appear in any aggregate function", + ): + spark.sql(""" + SELECT * FROM ( + SELECT g, k, v FROM VALUES + ('a', 'x', 10) + AS t(g, k, v) + ) PIVOT (count(*) + v FOR (k) IN ('x')) + """).collect() + + +def test_pivot_first_last_string_ignores_nulls_like_spark_general_path(spark): + actual = spark.sql(""" + SELECT * FROM ( + SELECT g, k, s FROM VALUES + ('a', 'x', CAST(NULL AS STRING)), + ('a', 'x', 'alpha'), + ('b', 'x', CAST(NULL AS STRING)), + ('b', 'x', 'bravo') + AS t(g, k, s) + ) PIVOT ( + first(s) AS first_s, + last(s) AS last_s + FOR (k) IN ('x') + ) + ORDER BY g + """).toPandas() + assert list(actual.columns) == ["g", "x_first_s", "x_last_s"] + assert actual.values.tolist() == [["a", "alpha", "alpha"], ["b", "bravo", "bravo"]] + + +def test_pivot_multi_column_struct(spark): + actual = spark.sql(""" + SELECT * FROM ( + SELECT year, course, training, earnings FROM VALUES + (2012, 'Java', 'Dummies', 20000), + (2012, 'dotNET', 'Experts', 10000), + (2013, 'Java', 'Dummies', 30000), + (2013, 'dotNET', 'Experts', 48000) + AS s(year, course, training, earnings) + ) PIVOT (sum(earnings) FOR (course, training) + IN (('Java', 'Dummies'), ('dotNET', 'Experts'))) + ORDER BY year + """).toPandas() + assert list(actual.columns) == ["year", "{Java, Dummies}", "{dotNET, Experts}"] + assert actual.values.tolist() == [[2012, 20000, 10000], [2013, 30000, 48000]] + + +def test_pivot_multi_column_struct_alias(spark): + actual = spark.sql(""" + SELECT * FROM ( + SELECT year, course, training, earnings FROM VALUES + (2012, 'Java', 'Dummies', 20000), + (2012, 'dotNET', 'Experts', 10000), + (2013, 'Java', 'Dummies', 30000), + (2013, 'dotNET', 'Experts', 48000) + AS s(year, course, training, earnings) + ) PIVOT (sum(earnings) FOR (course, training) + IN (('Java', 'Dummies') AS java_dummies)) + ORDER BY year + """).toPandas() + assert list(actual.columns) == ["year", "java_dummies"] + assert actual.values.tolist() == [[2012, 20000], [2013, 30000]] + + +def test_pivot_multi_column_value_arity_mismatch_errors(spark): + with pytest.raises( + Exception, + match="pivot value has 1 expressions, but pivot column has 2", + ): + spark.sql(""" + SELECT * FROM ( + SELECT year, course, training, earnings FROM VALUES + (2012, 'Java', 'Dummies', 20000) + AS s(year, course, training, earnings) + ) PIVOT (sum(earnings) FOR (course, training) IN ('Java')) + """).collect() + + # SQL for a pivot whose value alias collides with a grouping column, producing two # output columns both named `year` (Spark allows this). Shared by the pair of tests below. _PIVOT_GROUPING_COLLISION_SQL = """ @@ -119,17 +231,9 @@ def test_pivot_value_colliding_with_grouping_column(spark): assert rows == [(2012, 20000), (2013, 30000)] -@pytest.mark.xfail( - strict=False, - reason="Known flaky bug: with duplicate output column names, the Arrow/toPandas path " - 'intermittently fails with an internal field-id error (No field named s."#0") while ' - "collect() always succeeds. Paired with the collect test above to isolate the failure " - "to toPandas. Separate core field-id resolution bug, not specific to pivot.", -) def test_pivot_value_colliding_with_grouping_column_to_pandas(spark): # Same query as the collect-based test above, but materialized via toPandas. This is the - # path that intermittently breaks; the pairing pinpoints toPandas (Arrow conversion) as - # the culprit rather than the pivot resolution or execution itself. + # path that exercises duplicate output names through Arrow conversion actual = spark.sql(_PIVOT_GROUPING_COLLISION_SQL).toPandas() assert list(actual.columns) == ["year", "year"] assert sorted(actual.values.tolist()) == [[2012, 20000], [2013, 30000]] @@ -140,7 +244,10 @@ def test_pivot_inference_rejects_too_many_distinct_values(spark): # errors beyond it. The distinct-value scan is bounded by a LIMIT of one past the cap, # so a high-cardinality pivot column never collects an unbounded result into the planner. df = spark.range(10001).selectExpr("CAST(0 AS INT) AS g", "id AS k", "CAST(1 AS INT) AS v") - with pytest.raises(Exception, match="10000"): + with pytest.raises( + Exception, + match=r"The pivot column k has more than 10000 distinct values.*spark\.sql\.pivotMaxValues", + ): df.groupBy("g").pivot("k").sum("v").collect() @@ -154,7 +261,10 @@ def test_pivot_max_values_config_is_honored(spark): [("a", 1, 10), ("a", 2, 20), ("a", 3, 30)], schema="g STRING, k INT, v INT", ) - with pytest.raises(Exception, match="more than 2 distinct values"): + with pytest.raises( + Exception, + match=r"The pivot column k has more than 2 distinct values.*spark\.sql\.pivotMaxValues", + ): df.groupBy("g").pivot("k").sum("v").collect() finally: spark.conf.set("spark.sql.pivotMaxValues", original) @@ -195,14 +305,46 @@ def test_pivot_decimal_value_naming_matches_spark(spark): assert actual.values.tolist() == [["a", 10, 20], ["b", 5, 8]] -@pytest.mark.xfail( - strict=True, - reason="A NaN pivot value matches the NaN row in Sail (the pivot predicate `k = NaN` " - "evaluates true, consistent with Sail's `=` operator), so the NaN column is populated. " - "Spark leaves it NULL: although Spark SQL's `NaN = NaN` is also true, its pivot uses raw " - "IEEE `==` in codegen (NaN != NaN) and so contradicts its own `=`. Matching that would make " - "Sail's pivot inconsistent with its own equality, so this extreme edge is left as-is.", -) +def test_pivot_value_cast_to_int_column(spark): + actual = spark.sql(""" + SELECT * FROM ( + SELECT g, k, v FROM VALUES + ('a', 1, 10), ('a', 2, 20), ('b', 1, 5) + AS t(g, k, v) + ) PIVOT (sum(v) FOR (k) IN (1.2)) + ORDER BY g + """).toPandas() + assert list(actual.columns) == ["g", "1.2"] + assert actual.values.tolist() == [["a", 10], ["b", 5]] + + +def test_pivot_value_cast_to_string_column(spark): + actual = spark.sql(""" + SELECT * FROM ( + SELECT g, k, v FROM VALUES + ('a', '1', 10), ('a', '01', 99), + ('a', '2', 20), ('a', '02', 99), + ('b', '1', 5), ('b', '01', 99), + ('b', '2', 8), ('b', '02', 99) + AS t(g, k, v) + ) PIVOT (sum(v) FOR (k) IN (1, 2)) + ORDER BY g + """).toPandas() + assert list(actual.columns) == ["g", "1", "2"] + assert actual.values.tolist() == [["a", 10, 20], ["b", 5, 8]] + + +def test_pivot_rejects_non_castable_value(spark): + with pytest.raises(Exception, match=r"(?i)cast"): + spark.sql(""" + SELECT * FROM ( + SELECT g, k, v FROM VALUES + ('a', 1, 10) + AS t(g, k, v) + ) PIVOT (sum(v) FOR (k) IN ('not-an-int')) + """).collect() + + def test_pivot_nan_value_yields_null_column_like_spark(spark): # In Spark the NaN row matches no pivot value, so the `NaN` column is entirely NULL. actual = ( @@ -290,3 +432,22 @@ def test_pivot_inference_includes_null_pivot_value(spark): ) assert df.columns == ["g", "null", "1"] assert [tuple(r) for r in df.collect()] == [("a", 5, 10), ("b", None, 7)] + + +def test_pivot_nan_value_matches_in_general_path(spark): + import pyspark.sql.functions as F # noqa: N812 + + # Non-numeric aggregate -> Spark's general (If/EqualNullSafe) pivot path, where NaN DOES + # match (unlike the PivotFirst path). Pins the `spark_uses_pivot_first` gate's other branch. + actual = ( + spark.createDataFrame( + [("a", 1.0, "x"), ("a", float("nan"), "z"), ("b", 1.0, "y")], + schema="g STRING, k DOUBLE, s STRING", + ) + .groupBy("g") + .pivot("k") + .agg(F.first("s")) + .orderBy("g") + .toPandas() + ) + assert actual.set_index("g")["NaN"].to_dict() == {"a": "z", "b": None} From dff1432049e0ec57e1cce20e1cf79e698ce912e1 Mon Sep 17 00:00:00 2001 From: Heran Lin Date: Sat, 27 Jun 2026 21:06:20 +0800 Subject: [PATCH 002/131] feat: improve catalog providers (#2141) Co-authored-by: Shehab <11789402+shehabgamin@users.noreply.github.com> --- Cargo.lock | 8 +- crates/sail-catalog-iceberg/Cargo.toml | 1 + crates/sail-catalog-iceberg/src/lib.rs | 112 ++- .../src/models/assert_ref_snapshot_id.rs | 4 +- .../src/models/commit_view_request.rs | 2 +- crates/sail-catalog-iceberg/src/models/mod.rs | 2 + .../src/models/table_requirement.rs | 34 +- .../src/models/table_update.rs | 128 ++- .../src/models/view_requirement.rs | 38 + .../src/models/view_update.rs | 36 +- crates/sail-catalog-iceberg/src/provider.rs | 850 +++++++++--------- .../tests/rest_integration_test.rs | 14 +- crates/sail-catalog-onelake/Cargo.toml | 7 +- .../sail-catalog-onelake/src/credentials.rs | 63 ++ crates/sail-catalog-onelake/src/lib.rs | 4 +- crates/sail-catalog-onelake/src/provider.rs | 761 +++++++--------- crates/sail-catalog-unity/build.rs | 85 ++ crates/sail-catalog-unity/src/config.rs | 10 - crates/sail-catalog-unity/src/credential.rs | 19 + crates/sail-catalog-unity/src/lib.rs | 3 +- crates/sail-catalog-unity/src/provider.rs | 434 ++++++--- .../tests/rest_integration_test.rs | 12 +- crates/sail-catalog/src/credentials.rs | 36 + crates/sail-catalog/src/lib.rs | 1 + crates/sail-catalog/src/manager/table.rs | 2 +- crates/sail-common/src/config/application.rs | 10 + crates/sail-common/src/http.rs | 7 + crates/sail-common/src/lib.rs | 1 + crates/sail-iceberg/src/table/mod.rs | 6 +- crates/sail-iceberg/src/table_format.rs | 23 +- crates/sail-session/src/catalog.rs | 96 +- .../recipes/iceberg-openapi-generation.md | 21 +- docs/guide/catalog/onelake.md | 70 +- docs/guide/storage/azure.md | 2 +- .../pysail/examples/spark/onelake_catalog.py | 2 +- 35 files changed, 1784 insertions(+), 1120 deletions(-) create mode 100644 crates/sail-catalog-iceberg/src/models/view_requirement.rs create mode 100644 crates/sail-catalog-onelake/src/credentials.rs create mode 100644 crates/sail-catalog/src/credentials.rs create mode 100644 crates/sail-common/src/http.rs diff --git a/Cargo.lock b/Cargo.lock index 300db313c..c8ce57dad 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7445,6 +7445,7 @@ version = "0.6.4" dependencies = [ "arrow", "async-trait", + "log", "percent-encoding", "reqwest", "sail-catalog", @@ -7475,13 +7476,14 @@ dependencies = [ name = "sail-catalog-onelake" version = "0.6.4" dependencies = [ - "arrow", "async-trait", - "reqwest", + "object_store", "sail-catalog", + "sail-catalog-iceberg", + "sail-catalog-unity", "sail-common-datafusion", - "serde", "tokio", + "url", ] [[package]] diff --git a/crates/sail-catalog-iceberg/Cargo.toml b/crates/sail-catalog-iceberg/Cargo.toml index 2d9d39e77..14c3f613d 100644 --- a/crates/sail-catalog-iceberg/Cargo.toml +++ b/crates/sail-catalog-iceberg/Cargo.toml @@ -33,6 +33,7 @@ serde_json = { workspace = true } serde_with = { workspace = true } uuid = { workspace = true } tokio = { workspace = true } +log = { workspace = true } [dev-dependencies] tokio = { workspace = true } diff --git a/crates/sail-catalog-iceberg/src/lib.rs b/crates/sail-catalog-iceberg/src/lib.rs index 740411d48..60f35fe18 100644 --- a/crates/sail-catalog-iceberg/src/lib.rs +++ b/crates/sail-catalog-iceberg/src/lib.rs @@ -19,8 +19,8 @@ mod models; mod provider; pub use provider::{ - IcebergRestCatalogProvider, REST_CATALOG_PROP_PREFIX, REST_CATALOG_PROP_URI, - REST_CATALOG_PROP_WAREHOUSE, + IcebergRestCatalogOptions, IcebergRestCatalogProvider, REST_CATALOG_PROP_PREFIX, + REST_CATALOG_PROP_URI, REST_CATALOG_PROP_WAREHOUSE, }; #[cfg(test)] @@ -39,48 +39,126 @@ mod table_update_tests { "updates": [{"action": "set-properties", "updates": {"k": "v"}}] })) .expect("set-properties commit must deserialize"); + assert_eq!(request.requirements.len(), 1); + assert!(matches!( + &request.requirements[0], + models::TableRequirement::AssertTableUuid { uuid } + if uuid == "11111111-1111-1111-1111-111111111111" + )); assert_eq!(request.updates.len(), 1); assert!(matches!( - request.updates[0], - models::TableUpdate::SetPropertiesUpdate {} + &request.updates[0], + models::TableUpdate::SetPropertiesUpdate { updates } + if updates.get("k").map(String::as_str) == Some("v") )); } + #[test] + fn commit_table_request_preserves_ref_snapshot_requirement() { + let request = models::CommitTableRequest::new( + vec![models::TableRequirement::AssertRefSnapshotId { + r#ref: "main".to_string(), + snapshot_id: None, + }], + vec![models::TableUpdate::SetSnapshotRefUpdate { + ref_name: "main".to_string(), + r#type: models::set_snapshot_ref_update::Type::Branch, + snapshot_id: 123, + max_ref_age_ms: None, + max_snapshot_age_ms: None, + min_snapshots_to_keep: None, + }], + ); + let value = serde_json::to_value(request).expect("commit request must serialize"); + assert_eq!( + value["requirements"][0], + serde_json::json!({ + "type": "assert-ref-snapshot-id", + "ref": "main", + "snapshot-id": null + }) + ); + assert_eq!( + value["updates"][0], + serde_json::json!({ + "action": "set-snapshot-ref", + "ref-name": "main", + "type": "branch", + "snapshot-id": 123 + }) + ); + } + #[test] fn table_update_discriminates_on_action() { let cases = [ - ("assign-uuid", "AssignUuidUpdate"), - ("add-snapshot", "AddSnapshotUpdate"), - ("set-location", "SetLocationUpdate"), - ("remove-properties", "RemovePropertiesUpdate"), - ("upgrade-format-version", "UpgradeFormatVersionUpdate"), + serde_json::json!({"action": "assign-uuid", "uuid": "table-uuid"}), + serde_json::json!({"action": "set-location", "location": "s3://bucket/table"}), + serde_json::json!({"action": "remove-properties", "removals": ["k"]}), + serde_json::json!({"action": "set-properties", "updates": {"k": "v"}}), + serde_json::json!({"action": "upgrade-format-version", "format-version": 2}), ]; - for (action, _name) in cases { + for input in cases { let update: models::TableUpdate = - serde_json::from_value(serde_json::json!({"action": action})) - .unwrap_or_else(|err| panic!("action {action} must deserialize: {err}")); - // Round-trips back to the same tagged action. + serde_json::from_value(input.clone()).unwrap_or_else(|err| { + panic!("update {input} must deserialize and preserve fields: {err}") + }); let value = serde_json::to_value(&update).unwrap(); - assert_eq!(value["action"], serde_json::json!(action)); + assert_eq!(value, input); } } #[test] - fn view_update_accepts_set_properties() { + fn commit_view_request_preserves_requirements_and_updates() { let request: models::CommitViewRequest = serde_json::from_value(serde_json::json!({ + "requirements": [{"type": "assert-view-uuid", "uuid": "22222222-2222-2222-2222-222222222222"}], "updates": [{"action": "set-properties", "updates": {"k": "v"}}] })) .expect("view set-properties commit must deserialize"); + let requirement = request + .requirements + .as_ref() + .expect("view requirements must deserialize"); + assert!(matches!( + &requirement[0], + models::ViewRequirement::AssertViewUuid { uuid } + if uuid == "22222222-2222-2222-2222-222222222222" + )); assert!(matches!( - request.updates[0], - models::ViewUpdate::SetPropertiesUpdate {} + &request.updates[0], + models::ViewUpdate::SetPropertiesUpdate { updates } + if updates.get("k").map(String::as_str) == Some("v") )); } + #[test] + fn view_update_discriminates_on_action() { + let cases = [ + serde_json::json!({"action": "assign-uuid", "uuid": "view-uuid"}), + serde_json::json!({"action": "set-current-view-version", "view-version-id": 7}), + serde_json::json!({"action": "set-location", "location": "s3://bucket/view"}), + serde_json::json!({"action": "remove-properties", "removals": ["k"]}), + serde_json::json!({"action": "set-properties", "updates": {"k": "v"}}), + serde_json::json!({"action": "upgrade-format-version", "format-version": 1}), + ]; + for input in cases { + let update: models::ViewUpdate = + serde_json::from_value(input.clone()).unwrap_or_else(|err| { + panic!("view update {input} must deserialize and preserve fields: {err}") + }); + let value = serde_json::to_value(&update).unwrap(); + assert_eq!(value, input); + } + } + #[test] fn unknown_action_is_rejected() { let result: Result = serde_json::from_value(serde_json::json!({"action": "not-a-real-update"})); assert!(result.is_err(), "unknown action must not deserialize"); + + let result: Result = + serde_json::from_value(serde_json::json!({"action": "not-a-real-update"})); + assert!(result.is_err(), "unknown view action must not deserialize"); } } diff --git a/crates/sail-catalog-iceberg/src/models/assert_ref_snapshot_id.rs b/crates/sail-catalog-iceberg/src/models/assert_ref_snapshot_id.rs index 2c9e2c562..790cbdefe 100644 --- a/crates/sail-catalog-iceberg/src/models/assert_ref_snapshot_id.rs +++ b/crates/sail-catalog-iceberg/src/models/assert_ref_snapshot_id.rs @@ -32,12 +32,12 @@ pub struct AssertRefSnapshotId { #[serde(rename = "ref")] pub r#ref: String, #[serde(rename = "snapshot-id")] - pub snapshot_id: i64, + pub snapshot_id: Option, } impl AssertRefSnapshotId { /// The table branch or tag identified by the requirement's `ref` must reference the requirement's `snapshot-id`. The `snapshot-id` field is required in this object, but in the case of a `null` the ref must not already exist. - pub fn new(r#ref: String, snapshot_id: i64) -> AssertRefSnapshotId { + pub fn new(r#ref: String, snapshot_id: Option) -> AssertRefSnapshotId { AssertRefSnapshotId { r#type: None, r#ref, diff --git a/crates/sail-catalog-iceberg/src/models/commit_view_request.rs b/crates/sail-catalog-iceberg/src/models/commit_view_request.rs index 8796645ea..eeb402f56 100644 --- a/crates/sail-catalog-iceberg/src/models/commit_view_request.rs +++ b/crates/sail-catalog-iceberg/src/models/commit_view_request.rs @@ -30,7 +30,7 @@ pub struct CommitViewRequest { #[serde(rename = "identifier", skip_serializing_if = "Option::is_none")] pub identifier: Option>, #[serde(rename = "requirements", skip_serializing_if = "Option::is_none")] - pub requirements: Option>, + pub requirements: Option>, #[serde(rename = "updates")] pub updates: Vec, } diff --git a/crates/sail-catalog-iceberg/src/models/mod.rs b/crates/sail-catalog-iceberg/src/models/mod.rs index 9e6872d4f..84fb166c3 100644 --- a/crates/sail-catalog-iceberg/src/models/mod.rs +++ b/crates/sail-catalog-iceberg/src/models/mod.rs @@ -252,6 +252,8 @@ pub mod view_history_entry; pub use self::view_history_entry::ViewHistoryEntry; pub mod view_metadata; pub use self::view_metadata::ViewMetadata; +pub mod view_requirement; +pub use self::view_requirement::ViewRequirement; pub mod view_update; pub use self::view_update::ViewUpdate; pub mod view_version; diff --git a/crates/sail-catalog-iceberg/src/models/table_requirement.rs b/crates/sail-catalog-iceberg/src/models/table_requirement.rs index 2b4714fe4..e5fc6bd97 100644 --- a/crates/sail-catalog-iceberg/src/models/table_requirement.rs +++ b/crates/sail-catalog-iceberg/src/models/table_requirement.rs @@ -30,19 +30,39 @@ pub enum TableRequirement { #[serde(rename = "assert-create")] AssertCreate {}, #[serde(rename = "assert-current-schema-id")] - AssertCurrentSchemaId {}, + AssertCurrentSchemaId { + #[serde(rename = "current-schema-id")] + current_schema_id: i32, + }, #[serde(rename = "assert-default-sort-order-id")] - AssertDefaultSortOrderId {}, + AssertDefaultSortOrderId { + #[serde(rename = "default-sort-order-id")] + default_sort_order_id: i64, + }, #[serde(rename = "assert-default-spec-id")] - AssertDefaultSpecId {}, + AssertDefaultSpecId { + #[serde(rename = "default-spec-id")] + default_spec_id: i32, + }, #[serde(rename = "assert-last-assigned-field-id")] - AssertLastAssignedFieldId {}, + AssertLastAssignedFieldId { + #[serde(rename = "last-assigned-field-id")] + last_assigned_field_id: i32, + }, #[serde(rename = "assert-last-assigned-partition-id")] - AssertLastAssignedPartitionId {}, + AssertLastAssignedPartitionId { + #[serde(rename = "last-assigned-partition-id")] + last_assigned_partition_id: i32, + }, #[serde(rename = "assert-ref-snapshot-id")] - AssertRefSnapshotId {}, + AssertRefSnapshotId { + #[serde(rename = "ref")] + r#ref: String, + #[serde(rename = "snapshot-id")] + snapshot_id: Option, + }, #[serde(rename = "assert-table-uuid")] - AssertTableUuid {}, + AssertTableUuid { uuid: String }, } impl Default for TableRequirement { diff --git a/crates/sail-catalog-iceberg/src/models/table_update.rs b/crates/sail-catalog-iceberg/src/models/table_update.rs index 52dd12a88..497973efd 100644 --- a/crates/sail-catalog-iceberg/src/models/table_update.rs +++ b/crates/sail-catalog-iceberg/src/models/table_update.rs @@ -28,51 +28,135 @@ use crate::models; #[serde(tag = "action")] pub enum TableUpdate { #[serde(rename = "add-encryption-key")] - AddEncryptionKeyUpdate {}, + AddEncryptionKeyUpdate { + #[serde(rename = "encryption-key")] + encryption_key: Box, + }, #[serde(rename = "add-schema")] - AddSchemaUpdate {}, + AddSchemaUpdate { + #[serde(rename = "schema")] + schema: Box, + #[serde(rename = "last-column-id", skip_serializing_if = "Option::is_none")] + last_column_id: Option, + }, #[serde(rename = "add-snapshot")] - AddSnapshotUpdate {}, + AddSnapshotUpdate { + #[serde(rename = "snapshot")] + snapshot: Box, + }, #[serde(rename = "add-sort-order")] - AddSortOrderUpdate {}, + AddSortOrderUpdate { + #[serde(rename = "sort-order")] + sort_order: Box, + }, #[serde(rename = "add-spec")] - AddPartitionSpecUpdate {}, + AddPartitionSpecUpdate { + #[serde(rename = "spec")] + spec: Box, + }, #[serde(rename = "assign-uuid")] - AssignUuidUpdate {}, + AssignUuidUpdate { uuid: String }, #[serde(rename = "remove-encryption-key")] - RemoveEncryptionKeyUpdate {}, + RemoveEncryptionKeyUpdate { + #[serde(rename = "key-id")] + key_id: String, + }, #[serde(rename = "remove-partition-specs")] - RemovePartitionSpecsUpdate {}, + RemovePartitionSpecsUpdate { + #[serde(rename = "spec-ids")] + spec_ids: Vec, + }, + #[serde(rename = "remove-partition-statistics")] + RemovePartitionStatisticsUpdate { + #[serde(rename = "snapshot-id")] + snapshot_id: i64, + }, #[serde(rename = "remove-properties")] - RemovePropertiesUpdate {}, + RemovePropertiesUpdate { removals: Vec }, #[serde(rename = "remove-schemas")] - RemoveSchemasUpdate {}, + RemoveSchemasUpdate { + #[serde(rename = "schema-ids")] + schema_ids: Vec, + }, #[serde(rename = "remove-snapshot-ref")] - RemoveSnapshotRefUpdate {}, + RemoveSnapshotRefUpdate { + #[serde(rename = "ref-name")] + ref_name: String, + }, #[serde(rename = "remove-snapshots")] - RemoveSnapshotsUpdate {}, + RemoveSnapshotsUpdate { + #[serde(rename = "snapshot-ids")] + snapshot_ids: Vec, + }, #[serde(rename = "remove-statistics")] - RemoveStatisticsUpdate {}, + RemoveStatisticsUpdate { + #[serde(rename = "snapshot-id")] + snapshot_id: i64, + }, #[serde(rename = "set-current-schema")] - SetCurrentSchemaUpdate {}, + SetCurrentSchemaUpdate { + #[serde(rename = "schema-id")] + schema_id: i32, + }, #[serde(rename = "set-default-sort-order")] - SetDefaultSortOrderUpdate {}, + SetDefaultSortOrderUpdate { + #[serde(rename = "sort-order-id")] + sort_order_id: i64, + }, #[serde(rename = "set-default-spec")] - SetDefaultSpecUpdate {}, + SetDefaultSpecUpdate { + #[serde(rename = "spec-id")] + spec_id: i32, + }, #[serde(rename = "set-location")] - SetLocationUpdate {}, + SetLocationUpdate { location: String }, + #[serde(rename = "set-partition-statistics")] + SetPartitionStatisticsUpdate { + #[serde(rename = "partition-statistics")] + partition_statistics: Box, + }, #[serde(rename = "set-properties")] - SetPropertiesUpdate {}, + SetPropertiesUpdate { + updates: std::collections::HashMap, + }, #[serde(rename = "set-snapshot-ref")] - SetSnapshotRefUpdate {}, + SetSnapshotRefUpdate { + #[serde(rename = "ref-name")] + ref_name: String, + #[serde(rename = "type")] + r#type: models::set_snapshot_ref_update::Type, + #[serde(rename = "snapshot-id")] + snapshot_id: i64, + #[serde(rename = "max-ref-age-ms", skip_serializing_if = "Option::is_none")] + max_ref_age_ms: Option, + #[serde( + rename = "max-snapshot-age-ms", + skip_serializing_if = "Option::is_none" + )] + max_snapshot_age_ms: Option, + #[serde( + rename = "min-snapshots-to-keep", + skip_serializing_if = "Option::is_none" + )] + min_snapshots_to_keep: Option, + }, #[serde(rename = "set-statistics")] - SetStatisticsUpdate {}, + SetStatisticsUpdate { + #[serde(rename = "snapshot-id", skip_serializing_if = "Option::is_none")] + snapshot_id: Option, + statistics: Box, + }, #[serde(rename = "upgrade-format-version")] - UpgradeFormatVersionUpdate {}, + UpgradeFormatVersionUpdate { + #[serde(rename = "format-version")] + format_version: i32, + }, } impl Default for TableUpdate { fn default() -> Self { - Self::AddEncryptionKeyUpdate {} + Self::SetPropertiesUpdate { + updates: Default::default(), + } } } diff --git a/crates/sail-catalog-iceberg/src/models/view_requirement.rs b/crates/sail-catalog-iceberg/src/models/view_requirement.rs new file mode 100644 index 000000000..de1ec6a3a --- /dev/null +++ b/crates/sail-catalog-iceberg/src/models/view_requirement.rs @@ -0,0 +1,38 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/* + * Apache Iceberg REST Catalog API + * + * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. + * + * The version of the OpenAPI document: 0.0.1 + * + * Generated by: https://openapi-generator.tech + */ + +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(tag = "type")] +pub enum ViewRequirement { + #[serde(rename = "assert-view-uuid")] + AssertViewUuid { uuid: String }, +} + +impl Default for ViewRequirement { + fn default() -> Self { + Self::AssertViewUuid { + uuid: Default::default(), + } + } +} diff --git a/crates/sail-catalog-iceberg/src/models/view_update.rs b/crates/sail-catalog-iceberg/src/models/view_update.rs index b5eb570c6..d795066b7 100644 --- a/crates/sail-catalog-iceberg/src/models/view_update.rs +++ b/crates/sail-catalog-iceberg/src/models/view_update.rs @@ -28,25 +28,43 @@ use crate::models; #[serde(tag = "action")] pub enum ViewUpdate { #[serde(rename = "add-schema")] - AddSchemaUpdate {}, + AddSchemaUpdate { + #[serde(rename = "schema")] + schema: Box, + #[serde(rename = "last-column-id", skip_serializing_if = "Option::is_none")] + last_column_id: Option, + }, #[serde(rename = "add-view-version")] - AddViewVersionUpdate {}, + AddViewVersionUpdate { + #[serde(rename = "view-version")] + view_version: Box, + }, #[serde(rename = "assign-uuid")] - AssignUuidUpdate {}, + AssignUuidUpdate { uuid: String }, #[serde(rename = "remove-properties")] - RemovePropertiesUpdate {}, + RemovePropertiesUpdate { removals: Vec }, #[serde(rename = "set-current-view-version")] - SetCurrentViewVersionUpdate {}, + SetCurrentViewVersionUpdate { + #[serde(rename = "view-version-id")] + view_version_id: i32, + }, #[serde(rename = "set-location")] - SetLocationUpdate {}, + SetLocationUpdate { location: String }, #[serde(rename = "set-properties")] - SetPropertiesUpdate {}, + SetPropertiesUpdate { + updates: std::collections::HashMap, + }, #[serde(rename = "upgrade-format-version")] - UpgradeFormatVersionUpdate {}, + UpgradeFormatVersionUpdate { + #[serde(rename = "format-version")] + format_version: i32, + }, } impl Default for ViewUpdate { fn default() -> Self { - Self::AddSchemaUpdate {} + Self::SetPropertiesUpdate { + updates: Default::default(), + } } } diff --git a/crates/sail-catalog-iceberg/src/provider.rs b/crates/sail-catalog-iceberg/src/provider.rs index 92ff41380..9229d1dbb 100644 --- a/crates/sail-catalog-iceberg/src/provider.rs +++ b/crates/sail-catalog-iceberg/src/provider.rs @@ -10,11 +10,13 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::borrow::Cow; use std::collections::HashMap; use std::hash::{DefaultHasher, Hash, Hasher}; use std::sync::Arc; use percent_encoding::percent_decode_str; +use sail_catalog::credentials::CatalogCredentials; use sail_catalog::error::{CatalogError, CatalogObject, CatalogResult}; use sail_catalog::lakehouse::{ BeginTableAccessRequest, LakehouseCapability, LakehouseCommitOutcome, LakehouseCommitRequest, @@ -27,6 +29,7 @@ use sail_catalog::provider::{ PartitionTransform, }; use sail_catalog::utils::{get_property, quote_name_if_needed, quote_namespace_if_needed}; +use sail_common::http::SAIL_USER_AGENT; use sail_common_datafusion::catalog::managed::METADATA_LOCATION_KEY; use sail_common_datafusion::catalog::{ CapabilityFingerprint, CatalogTableBucketBy, CatalogTableConstraint, CatalogTableSort, @@ -59,217 +62,191 @@ const REST_REMOTE_SIGNING_ENABLED_KEY: &str = "s3.remote-signing-enabled"; // - https://iceberg.apache.org/docs/nightly/configuration/#catalog-properties // - https://iceberg.apache.org/docs/nightly/spark-configuration/ #[derive(Clone, Debug)] -pub struct RestCatalogConfig { - uri: String, - warehouse: Option, - props: HashMap, +pub struct CatalogConfig<'a> { + properties: Cow<'a, HashMap>, } -/// Provider for Apache Iceberg REST Catalog. -pub struct IcebergRestCatalogProvider { - name: String, - catalog_config: RestCatalogConfig, - merged_catalog_config: OnceCell, - client: OnceCell, -} +impl CatalogConfig<'_> { + fn uri(&self) -> Option { + self.properties + .get(REST_CATALOG_PROP_URI) + .cloned() + .map(|value| value.trim_end_matches('/').to_string()) + .filter(|value| !value.is_empty()) + } -impl IcebergRestCatalogProvider { - fn normalize_catalog_props(props: HashMap) -> HashMap { - props - .into_iter() - .map(|(key, value)| { - let key = if Self::is_namespace_separator_key(&key) { - REST_CATALOG_PROP_NAMESPACE_SEPARATOR.to_string() - } else { - key - }; - (key, value) - }) - .collect() + fn warehouse(&self) -> Option { + self.properties + .get(REST_CATALOG_PROP_WAREHOUSE) + .map(|value| value.trim_end_matches('/').to_string()) } - fn catalog_prefix(catalog_config: &RestCatalogConfig) -> Option<&str> { - catalog_config - .props + fn prefix(&self) -> Option<&str> { + self.properties .get(REST_CATALOG_PROP_PREFIX) .map(|value| value.trim()) .filter(|value| !value.is_empty()) } - fn is_namespace_separator_key(key: &str) -> bool { - matches!( - key.trim().to_ascii_lowercase().as_str(), - "namespace-separator" | "namespaceseparator" | "namespace_separator" - ) + fn namespace_separator(&self) -> CatalogResult { + let separator = self + .properties + .get(REST_CATALOG_PROP_NAMESPACE_SEPARATOR) + .map(|value| value.trim()); + + match separator { + Some("") | None => Ok(REST_CATALOG_DEFAULT_NAMESPACE_SEPARATOR.to_string()), + Some(separator) => percent_decode_str(separator) + .decode_utf8() + .map(|s| s.into_owned()) + .map_err(|e| { + CatalogError::InvalidArgument(format!( + "{REST_CATALOG_PROP_NAMESPACE_SEPARATOR} must be valid URL-encoded UTF-8: {e}" + )) + }), + } } - pub fn new(name: String, props: HashMap) -> Self { - let catalog_config = RestCatalogConfig { - uri: props - .get(REST_CATALOG_PROP_URI) - .cloned() - .unwrap_or(Configuration::new().base_path), - warehouse: props.get(REST_CATALOG_PROP_WAREHOUSE).cloned(), - props: Self::normalize_catalog_props( - props - .into_iter() - .filter(|(k, _)| k != REST_CATALOG_PROP_URI && k != REST_CATALOG_PROP_WAREHOUSE) - .collect(), - ), - }; + /// Converts a `Namespace` into a string representation for the REST API URL. + fn namespace_string(&self, database: &Namespace) -> CatalogResult { + let separator = self.namespace_separator()?; + let mut result = database.head.to_string(); + for s in &database.tail { + result.push_str(&separator); + result.push_str(s.as_ref()); + } + Ok(result) + } +} + +#[derive(Debug, Clone)] +pub struct IcebergRestCatalogOptions { + pub credentials: Arc, + pub properties: HashMap, +} + +/// Provider for Apache Iceberg REST Catalog. +pub struct IcebergRestCatalogProvider { + name: String, + options: IcebergRestCatalogOptions, + http_client: reqwest::Client, + resolved_catalog_config: OnceCell>, +} +impl IcebergRestCatalogProvider { + pub fn new(name: String, options: IcebergRestCatalogOptions) -> Self { Self { name, - catalog_config, - merged_catalog_config: OnceCell::new(), - client: OnceCell::new(), + options, + http_client: reqwest::Client::new(), + resolved_catalog_config: OnceCell::new(), } } - fn init_client_config( - &self, - catalog_config: &RestCatalogConfig, + async fn configuration( + catalog_config: &CatalogConfig<'_>, + credentials: &Arc, + http_client: reqwest::Client, ) -> CatalogResult { - let mut client_config = Configuration::new(); - client_config.user_agent = Some("Sail".to_string()); - client_config.base_path = catalog_config.uri.to_string(); - for (key, value) in &catalog_config.props { - // TODO: `basic_auth` and `api_key` are not used anything in the API yet. - // We may need to support them in the future. - match key.as_str() { - "oauth-access-token" => { - client_config.oauth_access_token = Some(value.to_string()); - } - "bearer-access-token" => { - client_config.bearer_access_token = Some(value.to_string()); - } - _ => {} - } + let base_path = catalog_config.uri().ok_or_else(|| { + CatalogError::InvalidArgument(format!( + "Iceberg REST catalog property '{REST_CATALOG_PROP_URI}' is required" + )) + })?; + let mut client_config = Configuration { + base_path, + user_agent: Some(SAIL_USER_AGENT.to_string()), + client: http_client, + basic_auth: None, + oauth_access_token: None, + bearer_access_token: None, + api_key: None, + }; + if let Some(credential) = credentials.retrieve().await? { + client_config.bearer_access_token = Some(credential); } Ok(client_config) } - fn init_client(&self, catalog_config: &RestCatalogConfig) -> CatalogResult { + async fn bootstrap_client(&self) -> CatalogResult { + let catalog_config = CatalogConfig { + properties: Cow::Borrowed(&self.options.properties), + }; Ok(ApiClient::new(Arc::new( - self.init_client_config(catalog_config)?, + Self::configuration( + &catalog_config, + &self.options.credentials, + self.http_client.clone(), + ) + .await?, ))) } - async fn load_catalog_config( - &self, - client: &ApiClient, - warehouse: Option<&str>, - ) -> CatalogResult { - let config = client - .configuration_api_api() - .get_config(warehouse) - .await - .map_err(|e| CatalogError::External(format!("Failed to load config: {e}")))?; - Ok(config) + async fn client(&self) -> CatalogResult { + let catalog_config = self.resolved_catalog_config().await?; + Ok(ApiClient::new(Arc::new( + Self::configuration( + catalog_config, + &self.options.credentials, + self.http_client.clone(), + ) + .await?, + ))) } - // Merge the `RestCatalogConfig` with the a [`CatalogConfig`] (fetched from the REST server). - // Then initialize the `ApiClient` with the merged config. + // Merge the local catalog config with the [`crate::models::CatalogConfig`] fetched from the REST server. // This only happens once, then the result is cached. - async fn load_client_and_merged_config( - &self, - ) -> CatalogResult<(&ApiClient, &RestCatalogConfig)> { - let merged_catalog_config = self - .merged_catalog_config + async fn resolved_catalog_config(&self) -> CatalogResult<&CatalogConfig<'static>> { + self.resolved_catalog_config .get_or_try_init(|| async { - let temp_client = self.init_client(&self.catalog_config)?; - let config = self - .load_catalog_config(&temp_client, self.catalog_config.warehouse.as_deref()) - .await?; - - let mut overrides = Self::normalize_catalog_props(config.overrides); - let mut client_props = self.catalog_config.props.clone(); - if let Some(warehouse) = &self.catalog_config.warehouse { - client_props.insert(REST_CATALOG_PROP_WAREHOUSE.to_string(), warehouse.clone()); - } - - let uri = if let Some(uri) = overrides.remove(REST_CATALOG_PROP_URI) { - uri.trim_end_matches('/').to_string() - } else { - self.catalog_config.uri.trim_end_matches('/').to_string() + let client = self.bootstrap_client().await?; + let catalog_config = CatalogConfig { + properties: Cow::Borrowed(&self.options.properties), }; + let warehouse = catalog_config.warehouse(); + let config = client + .configuration_api_api() + .get_config(warehouse.as_deref()) + .await + .map_err(|e| CatalogError::External(format!("Failed to load config: {e}")))?; + + let mut properties = config + .defaults + .into_iter() + .filter(|(key, _)| key != REST_CATALOG_PROP_URI) + .collect::>(); + properties.extend(self.options.properties.clone()); + properties.extend(config.overrides); - let mut props = Self::normalize_catalog_props(config.defaults); - props.extend(client_props); - props.extend(overrides); - let warehouse = props.get(REST_CATALOG_PROP_WAREHOUSE).cloned(); - - Ok::<_, CatalogError>(RestCatalogConfig { - uri, - warehouse, - props, + Ok::<_, CatalogError>(CatalogConfig { + properties: Cow::Owned(properties), }) }) - .await?; - - let client = self - .client - .get_or_try_init(|| async { self.init_client(merged_catalog_config) }) - .await?; - - Ok((client, merged_catalog_config)) - } - - fn namespace_separator(catalog_config: &RestCatalogConfig) -> CatalogResult { - let separator = catalog_config - .props - .get(REST_CATALOG_PROP_NAMESPACE_SEPARATOR) - .map(|value| value.trim()); - - match separator { - Some("") | None => Ok(REST_CATALOG_DEFAULT_NAMESPACE_SEPARATOR.to_string()), - Some(separator) => percent_decode_str(separator) - .decode_utf8() - .map(|s| s.into_owned()) - .map_err(|e| { - CatalogError::InvalidArgument(format!( - "{REST_CATALOG_PROP_NAMESPACE_SEPARATOR} must be valid URL-encoded UTF-8: {e}" - )) - }), - } - } - - /// Converts a `Namespace` into a string representation for the REST API URL. - fn namespace_string( - catalog_config: &RestCatalogConfig, - database: &Namespace, - ) -> CatalogResult { - let separator = Self::namespace_separator(catalog_config)?; - let mut result = database.head.to_string(); - for s in &database.tail { - result.push_str(&separator); - result.push_str(s.as_ref()); - } - Ok(result) + .await } - async fn load_table_result_with( + async fn load_table_result( &self, - client: &ApiClient, - catalog_config: &RestCatalogConfig, database: &Namespace, table: &str, access_delegation: Option<&str>, ) -> CatalogResult { + let client = self.client().await?; + let catalog_config = self.resolved_catalog_config().await?; client .catalog_api_api() .load_table( - &Self::namespace_string(catalog_config, database)?, + &catalog_config.namespace_string(database)?, table, access_delegation, None, None, - Self::catalog_prefix(catalog_config), + catalog_config.prefix(), ) .await .map_err(|e| match e { apis::Error::ResponseError(apis::ResponseContent { status, .. }) - if status == 404 => + if status == reqwest::StatusCode::NOT_FOUND => { CatalogError::NotFound( CatalogObject::Table, @@ -283,126 +260,11 @@ impl IcebergRestCatalogProvider { _ => CatalogError::External(format!( "Failed to load table {}.{}: {e}", quote_namespace_if_needed(database), - quote_name_if_needed(table), + quote_name_if_needed(table) )), }) } - async fn load_table_result( - &self, - database: &Namespace, - table: &str, - ) -> CatalogResult { - let (client, catalog_config) = self.load_client_and_merged_config().await?; - self.load_table_result_with(client, catalog_config, database, table, None) - .await - } - - async fn commit_table_request( - &self, - database: &Namespace, - table: &str, - format: &str, - requirements: Vec, - updates: Vec, - ) -> CatalogResult { - if !format.eq_ignore_ascii_case("iceberg") { - return Err(CatalogError::NotSupported(format!( - "Iceberg REST catalog cannot commit '{format}' tables", - ))); - } - - let (_client, catalog_config) = self.load_client_and_merged_config().await?; - let client_config = self.init_client_config(catalog_config)?; - let namespace = Self::namespace_string(catalog_config, database)?; - let prefix = catalog_config - .props - .get(REST_CATALOG_PROP_PREFIX) - .map(|prefix| format!("/{}", crate::apis::urlencode(prefix))) - .unwrap_or_default(); - let uri = format!( - "{}/v1{prefix}/namespaces/{namespace}/tables/{table}", - client_config.base_path, - namespace = crate::apis::urlencode(namespace), - table = crate::apis::urlencode(table) - ); - let body = serde_json::json!({ - "identifier": { - "namespace": Vec::::from(database.clone()), - "name": table, - }, - "requirements": requirements, - "updates": updates, - }); - - let mut request = client_config.client.post(uri); - if let Some(user_agent) = client_config.user_agent { - request = request.header(reqwest::header::USER_AGENT, user_agent); - } - if let Some(token) = client_config.oauth_access_token { - request = request.bearer_auth(token); - } - if let Some(token) = client_config.bearer_access_token { - request = request.bearer_auth(token); - } - - let response = request - .json(&body) - .send() - .await - .map_err(|e| CatalogError::External(format!("Failed to commit table: {e}")))?; - let status = response.status(); - let content = response - .text() - .await - .map_err(|e| CatalogError::External(format!("Failed to read commit response: {e}")))?; - - if status.is_success() { - serde_json::from_str(&content).map_err(|e| { - CatalogError::External(format!("Failed to parse commit response: {e}")) - }) - } else if status == reqwest::StatusCode::NOT_FOUND { - Err(CatalogError::NotFound( - CatalogObject::Table, - format!( - "{}.{}", - quote_namespace_if_needed(database), - quote_name_if_needed(table) - ), - )) - } else if status == reqwest::StatusCode::CONFLICT { - Err(CatalogError::Conflict(format!( - "Iceberg REST catalog commit conflict for {}.{}: {content}", - quote_namespace_if_needed(database), - quote_name_if_needed(table) - ))) - } else if status == reqwest::StatusCode::UNAUTHORIZED { - Err(CatalogError::Unauthorized(format!( - "Iceberg REST catalog commit unauthorized for {}.{}: {content}", - quote_namespace_if_needed(database), - quote_name_if_needed(table) - ))) - } else if status == reqwest::StatusCode::FORBIDDEN { - Err(CatalogError::Forbidden(format!( - "Iceberg REST catalog commit forbidden for {}.{}: {content}", - quote_namespace_if_needed(database), - quote_name_if_needed(table) - ))) - } else if status == reqwest::StatusCode::TOO_MANY_REQUESTS { - Err(CatalogError::RateLimited(format!( - "Iceberg REST catalog commit rate limited for {}.{}: {content}", - quote_namespace_if_needed(database), - quote_name_if_needed(table) - ))) - } else { - Err(CatalogError::External(format!( - "Failed to commit Iceberg table {}.{}: status {status}: {content}", - quote_namespace_if_needed(database), - quote_name_if_needed(table) - ))) - } - } - fn normalize_scan_planning_mode(value: &str) -> CatalogResult { match value.trim().to_ascii_lowercase().as_str() { "client" => Ok("client".to_string()), @@ -415,22 +277,30 @@ impl IcebergRestCatalogProvider { fn effective_scan_planning_mode( table_config: Option<&HashMap>, - catalog_config: &RestCatalogConfig, + catalog_config: &CatalogConfig<'_>, ) -> CatalogResult> { table_config .and_then(|config| config.get(REST_TABLE_SCAN_PLANNING_MODE_KEY)) - .or_else(|| catalog_config.props.get(REST_TABLE_SCAN_PLANNING_MODE_KEY)) + .or_else(|| { + catalog_config + .properties + .get(REST_TABLE_SCAN_PLANNING_MODE_KEY) + }) .map(|value| Self::normalize_scan_planning_mode(value)) .transpose() } fn remote_signing_enabled( table_config: Option<&HashMap>, - catalog_config: &RestCatalogConfig, + catalog_config: &CatalogConfig<'_>, ) -> bool { table_config .and_then(|config| config.get(REST_REMOTE_SIGNING_ENABLED_KEY)) - .or_else(|| catalog_config.props.get(REST_REMOTE_SIGNING_ENABLED_KEY)) + .or_else(|| { + catalog_config + .properties + .get(REST_REMOTE_SIGNING_ENABLED_KEY) + }) .map(|value| value.eq_ignore_ascii_case("true")) .unwrap_or(false) } @@ -445,7 +315,7 @@ impl IcebergRestCatalogProvider { } fn rest_table_session_fingerprint( - &self, + catalog: &str, database: &Namespace, table: &str, scan_planning_mode: Option<&str>, @@ -453,7 +323,7 @@ impl IcebergRestCatalogProvider { result: &crate::models::LoadTableResult, ) -> String { let mut hasher = DefaultHasher::new(); - self.name.hash(&mut hasher); + catalog.hash(&mut hasher); Vec::::from(database.clone()).hash(&mut hasher); table.hash(&mut hasher); result.metadata_location.hash(&mut hasher); @@ -480,17 +350,18 @@ impl IcebergRestCatalogProvider { } fn rest_table_session_ref( - &self, + catalog: &str, database: &Namespace, table: &str, - catalog_config: &RestCatalogConfig, + catalog_config: &CatalogConfig<'_>, result: &crate::models::LoadTableResult, ) -> CatalogResult { let config = result.config.as_ref(); let credentials = result.storage_credentials.as_deref(); let scan_planning_mode = Self::effective_scan_planning_mode(config, catalog_config)?; let remote_signing_enabled = Self::remote_signing_enabled(config, catalog_config); - let fingerprint = self.rest_table_session_fingerprint( + let fingerprint = Self::rest_table_session_fingerprint( + catalog, database, table, scan_planning_mode.as_deref(), @@ -508,13 +379,14 @@ impl IcebergRestCatalogProvider { } fn validate_create_table_access_session_requirements( - &self, + catalog: &str, database: &Namespace, table: &str, - catalog_config: &RestCatalogConfig, + catalog_config: &CatalogConfig<'_>, result: &crate::models::LoadTableResult, ) -> CatalogResult<()> { - let rest_session = self.rest_table_session_ref(database, table, catalog_config, result)?; + let rest_session = + Self::rest_table_session_ref(catalog, database, table, catalog_config, result)?; let mut requirements = Vec::new(); if rest_session.scan_planning_mode.as_deref() == Some("server") { requirements.push("server-side scan planning"); @@ -538,11 +410,19 @@ impl IcebergRestCatalogProvider { /// Converts an Iceberg REST API table load result into a catalog `TableStatus`. fn load_table_result_to_status( - &self, - table_name: &str, + catalog: &str, database: &Namespace, + table: &str, result: crate::models::LoadTableResult, ) -> CatalogResult { + log::trace!( + "Iceberg REST table load result: catalog={}, database={:?}, table={}, metadata.location={:?}, metadata-location={:?}", + catalog, + database, + table, + result.metadata.location, + result.metadata_location, + ); // Table-specific config and storage credentials are access-session state, not // display table properties. // TODO: Preserve unused fields in `TableMetadata` when Sail exposes them. @@ -762,9 +642,9 @@ impl IcebergRestCatalogProvider { let properties: Vec<_> = properties.into_iter().collect(); Ok(TableStatus { - catalog: Some(self.name.clone()), + catalog: Some(catalog.to_string()), database: database.clone().into(), - name: table_name.to_string(), + name: table.to_string(), kind: TableKind::Table { columns, comment, @@ -781,9 +661,9 @@ impl IcebergRestCatalogProvider { } fn load_view_result_to_status( - &self, - view_name: &str, + catalog: &str, database: &Namespace, + view: &str, result: crate::models::LoadViewResult, ) -> CatalogResult { // TODO: Do we want to do anything with: @@ -866,9 +746,9 @@ impl IcebergRestCatalogProvider { let properties: Vec<_> = properties.into_iter().collect(); Ok(TableStatus { - catalog: Some(self.name.clone()), + catalog: Some(catalog.to_string()), database: database.clone().into(), - name: view_name.to_string(), + name: view.to_string(), kind: TableKind::View { definition, columns, @@ -897,7 +777,8 @@ impl CatalogProvider for IcebergRestCatalogProvider { database: &Namespace, options: CreateDatabaseOptions, ) -> CatalogResult { - let (client, catalog_config) = self.load_client_and_merged_config().await?; + let catalog_config = self.resolved_catalog_config().await?; + let client = self.client().await?; let CreateDatabaseOptions { if_not_exists, @@ -921,7 +802,7 @@ impl CatalogProvider for IcebergRestCatalogProvider { let result = client .catalog_api_api() - .create_namespace(request, Self::catalog_prefix(catalog_config)) + .create_namespace(request, catalog_config.prefix()) .await; match result { @@ -946,7 +827,7 @@ impl CatalogProvider for IcebergRestCatalogProvider { }) } Err(apis::Error::ResponseError(apis::ResponseContent { status, .. })) - if status == 409 && if_not_exists => + if status == reqwest::StatusCode::CONFLICT && if_not_exists => { self.get_database(database).await } @@ -957,16 +838,17 @@ impl CatalogProvider for IcebergRestCatalogProvider { } async fn get_database(&self, database: &Namespace) -> CatalogResult { - let (client, catalog_config) = self.load_client_and_merged_config().await?; - let namespace = Self::namespace_string(catalog_config, database)?; + let catalog_config = self.resolved_catalog_config().await?; + let client = self.client().await?; + let namespace = catalog_config.namespace_string(database)?; let result = client .catalog_api_api() - .load_namespace_metadata(&namespace, Self::catalog_prefix(catalog_config)) + .load_namespace_metadata(&namespace, catalog_config.prefix()) .await .map_err(|e| match e { apis::Error::ResponseError(apis::ResponseContent { status, .. }) => { - if status == 404 { + if status == reqwest::StatusCode::NOT_FOUND { CatalogError::NotFound( CatalogObject::Namespace, quote_namespace_if_needed(database), @@ -1007,19 +889,15 @@ impl CatalogProvider for IcebergRestCatalogProvider { &self, prefix: Option<&Namespace>, ) -> CatalogResult> { - let (client, catalog_config) = self.load_client_and_merged_config().await?; + let catalog_config = self.resolved_catalog_config().await?; + let client = self.client().await?; let parent = prefix - .map(|namespace| Self::namespace_string(catalog_config, namespace)) + .map(|namespace| catalog_config.namespace_string(namespace)) .transpose()?; let result = client .catalog_api_api() - .list_namespaces( - None, - None, - parent.as_deref(), - Self::catalog_prefix(catalog_config), - ) + .list_namespaces(None, None, parent.as_deref(), catalog_config.prefix()) .await .map_err(|e| CatalogError::External(format!("Failed to list namespaces: {e}")))?; @@ -1042,14 +920,15 @@ impl CatalogProvider for IcebergRestCatalogProvider { database: &Namespace, options: DropDatabaseOptions, ) -> CatalogResult<()> { - let (client, catalog_config) = self.load_client_and_merged_config().await?; + let catalog_config = self.resolved_catalog_config().await?; + let client = self.client().await?; let DropDatabaseOptions { if_exists, cascade } = options; if cascade { // For CASCADE, first drop all tables and views in the namespace before dropping the namespace. - let prefix = Self::catalog_prefix(catalog_config); - let ns_string = Self::namespace_string(catalog_config, database)?; + let prefix = catalog_config.prefix(); + let ns_string = catalog_config.namespace_string(database)?; let tables_result = client .catalog_api_api() .list_tables(&ns_string, None, None, prefix) @@ -1079,14 +958,14 @@ impl CatalogProvider for IcebergRestCatalogProvider { match client .catalog_api_api() .drop_namespace( - &Self::namespace_string(catalog_config, database)?, - Self::catalog_prefix(catalog_config), + &catalog_config.namespace_string(database)?, + catalog_config.prefix(), ) .await { Ok(_) => Ok(()), Err(apis::Error::ResponseError(apis::ResponseContent { status, .. })) - if status == 404 && if_exists => + if status == reqwest::StatusCode::NOT_FOUND && if_exists => { Ok(()) } @@ -1123,7 +1002,8 @@ impl CatalogProvider for IcebergRestCatalogProvider { ))); } - let (client, catalog_config) = self.load_client_and_merged_config().await?; + let catalog_config = self.resolved_catalog_config().await?; + let client = self.client().await?; if mode.ignore_if_exists() { if let Ok(existing) = self.get_table(database, table).await { @@ -1195,40 +1075,42 @@ impl CatalogProvider for IcebergRestCatalogProvider { let result = client .catalog_api_api() .create_table( - &Self::namespace_string(catalog_config, database)?, + &catalog_config.namespace_string(database)?, request, None, - Self::catalog_prefix(catalog_config), + catalog_config.prefix(), ) .await .map_err(|e| CatalogError::External(format!("Failed to create table: {e}")))?; if is_write_precondition { - self.validate_create_table_access_session_requirements( + Self::validate_create_table_access_session_requirements( + &self.name, database, table, catalog_config, &result, )?; } - self.load_table_result_to_status(table, database, result) + Self::load_table_result_to_status(&self.name, database, table, result) } async fn get_table(&self, database: &Namespace, table: &str) -> CatalogResult { - let result = self.load_table_result(database, table).await?; - self.load_table_result_to_status(table, database, result) + let result = self.load_table_result(database, table, None).await?; + Self::load_table_result_to_status(&self.name, database, table, result) } async fn list_tables(&self, database: &Namespace) -> CatalogResult> { - let (client, catalog_config) = self.load_client_and_merged_config().await?; + let catalog_config = self.resolved_catalog_config().await?; + let client = self.client().await?; let result = client .catalog_api_api() .list_tables( - &Self::namespace_string(catalog_config, database)?, + &catalog_config.namespace_string(database)?, None, None, - Self::catalog_prefix(catalog_config), + catalog_config.prefix(), ) .await .map_err(|e| CatalogError::External(format!("Failed to list tables: {e}")))?; @@ -1263,21 +1145,22 @@ impl CatalogProvider for IcebergRestCatalogProvider { table: &str, options: DropTableOptions, ) -> CatalogResult<()> { - let (client, catalog_config) = self.load_client_and_merged_config().await?; + let catalog_config = self.resolved_catalog_config().await?; + let client = self.client().await?; let DropTableOptions { if_exists, purge } = options; match client .catalog_api_api() .drop_table( - &Self::namespace_string(catalog_config, database)?, + &catalog_config.namespace_string(database)?, table, Some(purge), - Self::catalog_prefix(catalog_config), + catalog_config.prefix(), ) .await { Ok(_) => Ok(()), Err(apis::Error::ResponseError(apis::ResponseContent { status, .. })) - if status == 404 && if_exists => + if status == reqwest::StatusCode::NOT_FOUND && if_exists => { Ok(()) } @@ -1309,9 +1192,95 @@ impl CatalogProvider for IcebergRestCatalogProvider { updates, payload, } = request; - let response = self - .commit_table_request(database, table, &format, requirements, updates) - .await?; + if !format.eq_ignore_ascii_case("iceberg") { + return Err(CatalogError::NotSupported(format!( + "Iceberg REST catalog cannot commit '{format}' tables", + ))); + } + + let catalog_config = self.resolved_catalog_config().await?; + let client = self.client().await?; + let namespace = catalog_config.namespace_string(database)?; + let requirements = requirements + .into_iter() + .map(serde_json::from_value) + .collect::, _>>() + .map_err(|e| { + CatalogError::External(format!( + "Failed to parse Iceberg REST commit requirements: {e}" + )) + })?; + let updates = updates + .into_iter() + .map(serde_json::from_value) + .collect::, _>>() + .map_err(|e| { + CatalogError::External(format!("Failed to parse Iceberg REST commit updates: {e}")) + })?; + let request = crate::models::CommitTableRequest { + identifier: Some(Box::new(crate::models::TableIdentifier::new( + Vec::::from(database.clone()), + table.to_string(), + ))), + requirements, + updates, + }; + let response = client + .catalog_api_api() + .update_table(&namespace, table, request, catalog_config.prefix()) + .await + .map_err(|e| match e { + apis::Error::ResponseError(apis::ResponseContent { + status, content: _, .. + }) if status == reqwest::StatusCode::NOT_FOUND => CatalogError::NotFound( + CatalogObject::Table, + format!( + "{}.{}", + quote_namespace_if_needed(database), + quote_name_if_needed(table) + ), + ), + apis::Error::ResponseError(apis::ResponseContent { + status, content, .. + }) if status == reqwest::StatusCode::CONFLICT => CatalogError::Conflict(format!( + "Iceberg REST catalog commit conflict for {}.{}: {content}", + quote_namespace_if_needed(database), + quote_name_if_needed(table) + )), + apis::Error::ResponseError(apis::ResponseContent { + status, content, .. + }) if status == reqwest::StatusCode::UNAUTHORIZED => { + CatalogError::Unauthorized(format!( + "Iceberg REST catalog commit unauthorized for {}.{}: {content}", + quote_namespace_if_needed(database), + quote_name_if_needed(table) + )) + } + apis::Error::ResponseError(apis::ResponseContent { + status, content, .. + }) if status == reqwest::StatusCode::FORBIDDEN => CatalogError::Forbidden(format!( + "Iceberg REST catalog commit forbidden for {}.{}: {content}", + quote_namespace_if_needed(database), + quote_name_if_needed(table) + )), + apis::Error::ResponseError(apis::ResponseContent { + status, content, .. + }) if status == reqwest::StatusCode::TOO_MANY_REQUESTS => { + CatalogError::RateLimited(format!( + "Iceberg REST catalog commit rate limited for {}.{}: {content}", + quote_namespace_if_needed(database), + quote_name_if_needed(table) + )) + } + apis::Error::ResponseError(apis::ResponseContent { + status, content, .. + }) => CatalogError::External(format!( + "Failed to commit Iceberg table {}.{}: status {status}: {content}", + quote_namespace_if_needed(database), + quote_name_if_needed(table) + )), + e => CatalogError::External(format!("Failed to commit table: {e}")), + })?; let payload = match payload { Some(payload) => Some(payload), None => Some(serde_json::to_value(response).map_err(|e| { @@ -1331,11 +1300,9 @@ impl CatalogProvider for IcebergRestCatalogProvider { mut context, purpose: _, } = request; - let (client, catalog_config) = self.load_client_and_merged_config().await?; + let catalog_config = self.resolved_catalog_config().await?; let result = self - .load_table_result_with( - client, - catalog_config, + .load_table_result( database, table, Some(REST_ACCESS_DELEGATION_VENDED_CREDENTIALS), @@ -1343,7 +1310,8 @@ impl CatalogProvider for IcebergRestCatalogProvider { .await?; // TODO: Convert preserved REST table-session credentials into operation-scoped // FileIO/object-store access instead of only fingerprinting the session. - let rest_session = self.rest_table_session_ref(database, table, catalog_config, &result)?; + let rest_session = + Self::rest_table_session_ref(&self.name, database, table, catalog_config, &result)?; if rest_session.scan_planning_mode.as_deref() == Some("server") { context.scan = ScanAuthority::IcebergRestServerSide; } else { @@ -1380,7 +1348,8 @@ impl CatalogProvider for IcebergRestCatalogProvider { view: &str, options: CreateViewOptions, ) -> CatalogResult { - let (client, catalog_config) = self.load_client_and_merged_config().await?; + let catalog_config = self.resolved_catalog_config().await?; + let client = self.client().await?; let CreateViewOptions { columns, @@ -1496,29 +1465,39 @@ impl CatalogProvider for IcebergRestCatalogProvider { let result = client .catalog_api_api() .create_view( - &Self::namespace_string(catalog_config, database)?, + &catalog_config.namespace_string(database)?, request, - Self::catalog_prefix(catalog_config), + catalog_config.prefix(), ) .await .map_err(|e| CatalogError::External(format!("Failed to create view: {e}")))?; - self.load_view_result_to_status(view, database, result) + Self::load_view_result_to_status(&self.name, database, view, result) } async fn get_view(&self, database: &Namespace, view: &str) -> CatalogResult { - let (client, catalog_config) = self.load_client_and_merged_config().await?; + let catalog_config = self.resolved_catalog_config().await?; + let client = self.client().await?; let result = client .catalog_api_api() .load_view( - &Self::namespace_string(catalog_config, database)?, + &catalog_config.namespace_string(database)?, view, - Self::catalog_prefix(catalog_config), + catalog_config.prefix(), ) .await .map_err(|e| match e { apis::Error::ResponseError(apis::ResponseContent { status, .. }) - if status == 404 => + if matches!( + status, + reqwest::StatusCode::METHOD_NOT_ALLOWED + | reqwest::StatusCode::NOT_IMPLEMENTED + ) => + { + CatalogError::NotSupported("get view".to_string()) + } + apis::Error::ResponseError(apis::ResponseContent { status, .. }) + if status == reqwest::StatusCode::NOT_FOUND => { CatalogError::NotFound( CatalogObject::View, @@ -1535,22 +1514,42 @@ impl CatalogProvider for IcebergRestCatalogProvider { quote_name_if_needed(view) )), })?; - self.load_view_result_to_status(view, database, result) + Self::load_view_result_to_status(&self.name, database, view, result) } async fn list_views(&self, database: &Namespace) -> CatalogResult> { - let (client, catalog_config) = self.load_client_and_merged_config().await?; + let catalog_config = self.resolved_catalog_config().await?; + let client = self.client().await?; let result = client .catalog_api_api() .list_views( - &Self::namespace_string(catalog_config, database)?, + &catalog_config.namespace_string(database)?, None, None, - Self::catalog_prefix(catalog_config), + catalog_config.prefix(), ) .await - .map_err(|e| CatalogError::External(format!("Failed to list views: {e}")))?; + .map_err(|e| match e { + apis::Error::ResponseError(apis::ResponseContent { status, .. }) + if matches!(status, reqwest::StatusCode::NOT_FOUND) => + { + CatalogError::NotFound( + CatalogObject::Namespace, + quote_namespace_if_needed(database), + ) + } + apis::Error::ResponseError(apis::ResponseContent { status, .. }) + if matches!( + status, + reqwest::StatusCode::METHOD_NOT_ALLOWED + | reqwest::StatusCode::NOT_IMPLEMENTED + ) => + { + CatalogError::NotSupported("list views".to_string()) + } + _ => CatalogError::External(format!("Failed to list views: {e}")), + })?; let catalog = &self.name; Ok(result .identifiers @@ -1576,20 +1575,21 @@ impl CatalogProvider for IcebergRestCatalogProvider { view: &str, options: DropViewOptions, ) -> CatalogResult<()> { - let (client, catalog_config) = self.load_client_and_merged_config().await?; + let catalog_config = self.resolved_catalog_config().await?; + let client = self.client().await?; let DropViewOptions { if_exists } = options; match client .catalog_api_api() .drop_view( - &Self::namespace_string(catalog_config, database)?, + &catalog_config.namespace_string(database)?, view, - Self::catalog_prefix(catalog_config), + catalog_config.prefix(), ) .await { Ok(_) => Ok(()), Err(apis::Error::ResponseError(apis::ResponseContent { status, .. })) - if status == 404 && if_exists => + if status == reqwest::StatusCode::NOT_FOUND && if_exists => { Ok(()) } @@ -1599,11 +1599,7 @@ impl CatalogProvider for IcebergRestCatalogProvider { } /// Finds an item by ID, falling back to the last item if not found or no ID is provided. -fn find_by_id_or_last<'a, T, F>( - items: Option<&'a Vec>, - id: Option, - get_id: F, -) -> Option<&'a T> +fn find_by_id_or_last(items: Option<&Vec>, id: Option, get_id: F) -> Option<&T> where F: Fn(&T) -> Option, { @@ -1957,6 +1953,7 @@ fn parse_unary_sort_transform( #[cfg(test)] mod tests { use arrow::datatypes::DataType; + use sail_catalog::credentials::EmptyCatalogCredentials; use sail_catalog::lakehouse::TableAccessPurpose; use sail_common_datafusion::catalog::{ CatalogProviderId, CatalogTableIdentity, CommitAuthority, LakehouseAuthority, @@ -1991,7 +1988,8 @@ mod tests { let name_str = name.unwrap_or(""); let props = HashMap::from([(REST_CATALOG_PROP_URI.to_string(), server.uri())]); - let catalog = IcebergRestCatalogProvider::new(name_str.to_string(), props); + let catalog = + IcebergRestCatalogProvider::new(name_str.to_string(), test_options(props)); Self { name: name_str.to_string(), @@ -2043,6 +2041,13 @@ mod tests { } } + fn test_options(properties: HashMap) -> IcebergRestCatalogOptions { + IcebergRestCatalogOptions { + credentials: Arc::new(EmptyCatalogCredentials), + properties, + } + } + fn simple_create_table_options() -> CreateTableOptions { CreateTableOptions { columns: vec![CreateTableColumnOptions { @@ -2112,7 +2117,7 @@ mod tests { mut client_props: HashMap, overrides: HashMap, expected_config_warehouse: Option<&str>, - ) -> RestCatalogConfig { + ) -> CatalogConfig<'static> { let server = MockServer::start().await; let response = ResponseTemplate::new(200).set_body_json(serde_json::json!({ "defaults": defaults, @@ -2141,34 +2146,13 @@ mod tests { .entry(REST_CATALOG_PROP_URI.to_string()) .or_insert_with(|| server.uri()); - let catalog = IcebergRestCatalogProvider::new("test".to_string(), client_props); + let catalog = + IcebergRestCatalogProvider::new("test".to_string(), test_options(client_props)); - let (_, config) = catalog.load_client_and_merged_config().await.unwrap(); + let config = catalog.resolved_catalog_config().await.unwrap(); config.clone() } - #[test] - fn test_namespace_separator_aliases_are_canonicalized() { - for alias in [ - "namespace_separator", - "namespace-separator", - "namespaceSeparator", - "namespaceseparator", - ] { - let props = IcebergRestCatalogProvider::normalize_catalog_props(HashMap::from([( - alias.to_string(), - "::".to_string(), - )])); - - assert_eq!( - props - .get(REST_CATALOG_PROP_NAMESPACE_SEPARATOR) - .map(String::as_str), - Some("::") - ); - } - } - #[tokio::test] async fn test_catalog_config_merge_precedence_for_properties() { let key = "rest-page-size".to_string(); @@ -2180,7 +2164,10 @@ mod tests { None, ) .await; - assert_eq!(config.props.get(&key).map(String::as_str), Some("defaults")); + assert_eq!( + config.properties.get(&key).map(String::as_str), + Some("defaults") + ); let config = load_merged_test_config( HashMap::from([(key.clone(), "defaults".to_string())]), @@ -2189,7 +2176,10 @@ mod tests { None, ) .await; - assert_eq!(config.props.get(&key).map(String::as_str), Some("client")); + assert_eq!( + config.properties.get(&key).map(String::as_str), + Some("client") + ); let config = load_merged_test_config( HashMap::from([(key.clone(), "defaults".to_string())]), @@ -2199,7 +2189,7 @@ mod tests { ) .await; assert_eq!( - config.props.get(&key).map(String::as_str), + config.properties.get(&key).map(String::as_str), Some("overrides") ); } @@ -2216,7 +2206,7 @@ mod tests { None, ) .await; - assert_ne!(config.uri, "http://default.example"); + assert_ne!(config.uri().as_deref(), Some("http://default.example")); let config = load_merged_test_config( HashMap::new(), @@ -2228,14 +2218,14 @@ mod tests { None, ) .await; - assert_eq!(config.uri, "http://server.example"); + assert_eq!(config.uri().as_deref(), Some("http://server.example")); } #[tokio::test] async fn test_warehouse_merge_matrix() { let config = load_merged_test_config(HashMap::new(), HashMap::new(), HashMap::new(), None).await; - assert_eq!(config.warehouse, None); + assert_eq!(config.warehouse(), None); let config = load_merged_test_config( HashMap::from([( @@ -2247,7 +2237,10 @@ mod tests { None, ) .await; - assert_eq!(config.warehouse.as_deref(), Some("s3://default/warehouse")); + assert_eq!( + config.warehouse().as_deref(), + Some("s3://default/warehouse") + ); let config = load_merged_test_config( HashMap::from([( @@ -2262,7 +2255,7 @@ mod tests { Some("s3://client/warehouse"), ) .await; - assert_eq!(config.warehouse.as_deref(), Some("s3://client/warehouse")); + assert_eq!(config.warehouse().as_deref(), Some("s3://client/warehouse")); let config = load_merged_test_config( HashMap::from([( @@ -2280,14 +2273,14 @@ mod tests { Some("s3://client/warehouse"), ) .await; - assert_eq!(config.warehouse.as_deref(), Some("s3://server/warehouse")); + assert_eq!(config.warehouse().as_deref(), Some("s3://server/warehouse")); } #[tokio::test] async fn test_prefix_merge_matrix() { let config = load_merged_test_config(HashMap::new(), HashMap::new(), HashMap::new(), None).await; - assert_eq!(IcebergRestCatalogProvider::catalog_prefix(&config), None); + assert_eq!(config.prefix(), None); let config = load_merged_test_config( HashMap::from([( @@ -2299,10 +2292,7 @@ mod tests { None, ) .await; - assert_eq!( - IcebergRestCatalogProvider::catalog_prefix(&config), - Some("default_prefix") - ); + assert_eq!(config.prefix(), Some("default_prefix")); let config = load_merged_test_config( HashMap::from([( @@ -2317,10 +2307,7 @@ mod tests { None, ) .await; - assert_eq!( - IcebergRestCatalogProvider::catalog_prefix(&config), - Some("client_prefix") - ); + assert_eq!(config.prefix(), Some("client_prefix")); let config = load_merged_test_config( HashMap::from([( @@ -2338,10 +2325,7 @@ mod tests { None, ) .await; - assert_eq!( - IcebergRestCatalogProvider::catalog_prefix(&config), - Some("server_prefix") - ); + assert_eq!(config.prefix(), Some("server_prefix")); let config = load_merged_test_config( HashMap::new(), @@ -2353,7 +2337,7 @@ mod tests { None, ) .await; - assert_eq!(IcebergRestCatalogProvider::catalog_prefix(&config), None); + assert_eq!(config.prefix(), None); } #[tokio::test] @@ -2364,7 +2348,7 @@ mod tests { let config = load_merged_test_config(HashMap::new(), HashMap::new(), HashMap::new(), None).await; assert_eq!( - IcebergRestCatalogProvider::namespace_string(&config, &namespace).unwrap(), + config.namespace_string(&namespace).unwrap(), "accounting\x1Ftax" ); @@ -2379,7 +2363,7 @@ mod tests { ) .await; assert_eq!( - IcebergRestCatalogProvider::namespace_string(&config, &namespace).unwrap(), + config.namespace_string(&namespace).unwrap(), "accounting/tax" ); @@ -2388,14 +2372,14 @@ mod tests { REST_CATALOG_PROP_NAMESPACE_SEPARATOR.to_string(), "/".to_string(), )]), - HashMap::from([("namespace_separator".to_string(), "%3A%3A".to_string())]), + HashMap::from([]), HashMap::new(), None, ) .await; assert_eq!( - IcebergRestCatalogProvider::namespace_string(&config, &namespace).unwrap(), - "accounting::tax" + config.namespace_string(&namespace).unwrap(), + "accounting/tax" ); let config = load_merged_test_config( @@ -2403,7 +2387,7 @@ mod tests { REST_CATALOG_PROP_NAMESPACE_SEPARATOR.to_string(), "/".to_string(), )]), - HashMap::from([("namespace_separator".to_string(), "::".to_string())]), + HashMap::from([]), HashMap::from([( REST_CATALOG_PROP_NAMESPACE_SEPARATOR.to_string(), "%7C".to_string(), @@ -2412,7 +2396,7 @@ mod tests { ) .await; assert_eq!( - IcebergRestCatalogProvider::namespace_string(&config, &namespace).unwrap(), + config.namespace_string(&namespace).unwrap(), "accounting|tax" ); @@ -2421,7 +2405,7 @@ mod tests { REST_CATALOG_PROP_NAMESPACE_SEPARATOR.to_string(), "/".to_string(), )]), - HashMap::from([("namespace_separator".to_string(), "::".to_string())]), + HashMap::from([]), HashMap::from([( REST_CATALOG_PROP_NAMESPACE_SEPARATOR.to_string(), " ".to_string(), @@ -2430,7 +2414,7 @@ mod tests { ) .await; assert_eq!( - IcebergRestCatalogProvider::namespace_string(&config, &namespace).unwrap(), + config.namespace_string(&namespace).unwrap(), "accounting\x1Ftax" ); } @@ -2623,6 +2607,22 @@ mod tests { test_list_views_impl(Some("test")).await; } + #[tokio::test] + async fn test_list_views_unsupported_endpoint() { + let ctx = TestContext::new(None).await; + + Mock::given(method("GET")) + .and(path(ctx.path("/namespaces/ns1/views"))) + .respond_with(ResponseTemplate::new(405)) + .mount(&ctx.server) + .await; + + let namespace = Namespace::try_from(vec!["ns1".to_string()]).unwrap(); + let error = ctx.catalog.list_views(&namespace).await.unwrap_err(); + + assert!(matches!(error, CatalogError::NotSupported(_))); + } + async fn test_drop_database_impl(name: Option<&str>) { let ctx = TestContext::new(name).await; @@ -3241,6 +3241,22 @@ mod tests { test_get_view_impl(Some("test")).await; } + #[tokio::test] + async fn test_get_view_unsupported_endpoint() { + let ctx = TestContext::new(None).await; + + Mock::given(method("GET")) + .and(path(ctx.path("/namespaces/db1/views/view1"))) + .respond_with(ResponseTemplate::new(501)) + .mount(&ctx.server) + .await; + + let namespace = Namespace::try_from(vec!["db1".to_string()]).unwrap(); + let error = ctx.catalog.get_view(&namespace, "view1").await.unwrap_err(); + + assert!(matches!(error, CatalogError::NotSupported(_))); + } + async fn test_create_database_impl(name: Option<&str>) { let ctx = TestContext::new(name).await; diff --git a/crates/sail-catalog-iceberg/tests/rest_integration_test.rs b/crates/sail-catalog-iceberg/tests/rest_integration_test.rs index 1219f8dbf..f1b26e8bf 100644 --- a/crates/sail-catalog-iceberg/tests/rest_integration_test.rs +++ b/crates/sail-catalog-iceberg/tests/rest_integration_test.rs @@ -13,15 +13,19 @@ #![expect(clippy::unwrap_used, clippy::expect_used, clippy::panic)] use std::collections::HashMap; +use std::sync::Arc; use arrow::datatypes::DataType; +use sail_catalog::credentials::EmptyCatalogCredentials; use sail_catalog::provider::{ CatalogPartitionField, CatalogProvider, CreateDatabaseOptions, CreateTableColumnOptions, CreateTableMode, CreateTableOptions, CreateViewColumnOptions, CreateViewOptions, DropDatabaseOptions, DropTableOptions, DropViewOptions, Namespace, PartitionTransform, RuntimeAwareCatalogProvider, }; -use sail_catalog_iceberg::{IcebergRestCatalogProvider, REST_CATALOG_PROP_URI}; +use sail_catalog_iceberg::{ + IcebergRestCatalogOptions, IcebergRestCatalogProvider, REST_CATALOG_PROP_URI, +}; use sail_common::runtime::RuntimeHandle; use sail_common_datafusion::catalog::{ CatalogTableConstraint, CatalogTableSort, TableColumnStatus, TableKind, @@ -108,7 +112,13 @@ async fn setup_catalog( let catalog = RuntimeAwareCatalogProvider::try_new( || { let props = HashMap::from([(REST_CATALOG_PROP_URI.to_string(), rest_url)]); - let provider = IcebergRestCatalogProvider::new("test".to_string(), props); + let provider = IcebergRestCatalogProvider::new( + "test".to_string(), + IcebergRestCatalogOptions { + credentials: Arc::new(EmptyCatalogCredentials), + properties: props, + }, + ); Ok(provider) }, runtime.io().clone(), diff --git a/crates/sail-catalog-onelake/Cargo.toml b/crates/sail-catalog-onelake/Cargo.toml index fab8543d7..41c1d24f0 100644 --- a/crates/sail-catalog-onelake/Cargo.toml +++ b/crates/sail-catalog-onelake/Cargo.toml @@ -9,9 +9,10 @@ workspace = true [dependencies] sail-common-datafusion = { path = "../sail-common-datafusion" } sail-catalog = { path = "../sail-catalog" } +sail-catalog-iceberg = { path = "../sail-catalog-iceberg" } +sail-catalog-unity = { path = "../sail-catalog-unity" } -arrow = { workspace = true } async-trait = { workspace = true } -reqwest = { workspace = true } -serde = { workspace = true } +object_store = { workspace = true } tokio = { workspace = true } +url = { workspace = true } diff --git a/crates/sail-catalog-onelake/src/credentials.rs b/crates/sail-catalog-onelake/src/credentials.rs new file mode 100644 index 000000000..65c4e6286 --- /dev/null +++ b/crates/sail-catalog-onelake/src/credentials.rs @@ -0,0 +1,63 @@ +use object_store::azure::{AzureCredential, AzureCredentialProvider, MicrosoftAzureBuilder}; +use sail_catalog::credentials::CatalogCredentials; +use sail_catalog::error::{CatalogError, CatalogResult}; +use tokio::sync::OnceCell; + +#[derive(Debug)] +pub enum OneLakeCredentials { + Static { + bearer_token: String, + }, + Dynamic { + workspace: String, + provider: OnceCell, + }, +} + +#[async_trait::async_trait] +impl CatalogCredentials for OneLakeCredentials { + async fn retrieve(&self) -> CatalogResult> { + match self { + Self::Static { bearer_token } => Ok(Some(bearer_token.clone())), + Self::Dynamic { + workspace, + provider, + } => { + let provider = provider + .get_or_try_init(|| async { + let mut builder = MicrosoftAzureBuilder::from_env() + .with_account("onelake") + .with_container_name(workspace) + .with_use_fabric_endpoint(true); + + if std::env::var("AZURE_STORAGE_TOKEN").is_err() { + if let Ok(token) = std::env::var("AZURE_ACCESS_TOKEN") { + builder = builder.with_bearer_token_authorization(token); + } + } + + let store = builder.build().map_err(|e| { + CatalogError::External(format!( + "Failed to build OneLake Azure credential provider: {e}" + )) + })?; + Ok::(store.credentials().clone()) + }) + .await?; + + let credential = provider.get_credential().await.map_err(|e| { + CatalogError::External(format!("Failed to get OneLake token: {e}")) + })?; + + match credential.as_ref() { + AzureCredential::BearerToken(token) => Ok(Some(token.clone())), + AzureCredential::AccessKey(_) | AzureCredential::SASToken(_) => { + Err(CatalogError::External( + "OneLake catalog requires Microsoft Entra bearer token authentication; Azure access keys and SAS tokens are not supported".to_string(), + )) + } + } + } + } + } +} diff --git a/crates/sail-catalog-onelake/src/lib.rs b/crates/sail-catalog-onelake/src/lib.rs index 9e0a63a22..ecf64b683 100644 --- a/crates/sail-catalog-onelake/src/lib.rs +++ b/crates/sail-catalog-onelake/src/lib.rs @@ -1,3 +1,5 @@ +mod credentials; mod provider; -pub use provider::OneLakeCatalogProvider; +pub use credentials::OneLakeCredentials; +pub use provider::{OneLakeApiKind, OneLakeCatalogProvider}; diff --git a/crates/sail-catalog-onelake/src/provider.rs b/crates/sail-catalog-onelake/src/provider.rs index ede79df10..f13ca2659 100644 --- a/crates/sail-catalog-onelake/src/provider.rs +++ b/crates/sail-catalog-onelake/src/provider.rs @@ -1,133 +1,57 @@ +use std::collections::HashMap; use std::sync::Arc; -use reqwest::Client; -use sail_catalog::error::{CatalogError, CatalogObject, CatalogResult}; +use sail_catalog::credentials::CatalogCredentials; +use sail_catalog::error::CatalogResult; +use sail_catalog::lakehouse::{ + BeginTableAccessRequest, DeltaRatifiedCommitRequest, DeltaRatifiedCommitResponse, + LakehouseCapability, LakehouseCommitOutcome, LakehouseCommitRequest, LakehouseCreatePlan, + LakehouseCreateRequest, LakehouseResolvedTable, LakehouseScanPlanningRequest, + LakehouseScanPlanningResponse, ResolveLakehouseTableRequest, TableAccessSession, +}; use sail_catalog::provider::{ - AlterTableOptions, CatalogProvider, CreateDatabaseOptions, CreateTableOptions, - CreateViewOptions, DropDatabaseOptions, DropTableOptions, DropViewOptions, Namespace, + AlterTableOptions, CatalogProvider, CreateDatabaseOptions, CreateTableMetadataRequirement, + CreateTableOptions, CreateViewOptions, DropDatabaseOptions, DropTableOptions, DropViewOptions, + Namespace, +}; +use sail_catalog_iceberg::{ + IcebergRestCatalogOptions, IcebergRestCatalogProvider, REST_CATALOG_PROP_PREFIX, + REST_CATALOG_PROP_URI, REST_CATALOG_PROP_WAREHOUSE, }; -use sail_catalog::utils::quote_namespace_if_needed; -use sail_common_datafusion::catalog::{DatabaseStatus, TableColumnStatus, TableKind, TableStatus}; -use serde::Deserialize; +use sail_catalog_unity::{UnityCatalogOptions, UnityCatalogProvider}; +use sail_common_datafusion::catalog::{DatabaseStatus, TableKind, TableStatus}; use tokio::sync::OnceCell; +use url::Url; -const ONELAKE_TABLE_API_BASE: &str = "https://onelake.table.fabric.microsoft.com/delta"; - -/// Fetches Azure access token using Azure CLI -async fn get_azure_cli_token() -> CatalogResult { - // On Windows, az is a batch file (.cmd) and must be run through cmd.exe - // See: https://doc.rust-lang.org/nightly/std/process/struct.Command.html - let (program, extra_args): (&str, &[&str]) = if cfg!(target_os = "windows") { - ("cmd", &["/C", "az"]) - } else { - ("az", &[]) - }; - - let output = tokio::process::Command::new(program) - .args(extra_args) - .args([ - "account", - "get-access-token", - "--resource", - "https://storage.azure.com/", - "--query", - "accessToken", - "-o", - "tsv", - ]) - .output() - .await - .map_err(|e| CatalogError::External(format!("Failed to run az cli: {e}")))?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - return Err(CatalogError::External(format!( - "Azure CLI failed: {stderr}" - ))); - } - - let token = String::from_utf8_lossy(&output.stdout).trim().to_string(); - if token.is_empty() { - return Err(CatalogError::External( - "Azure CLI returned empty token".to_string(), - )); - } - - Ok(token) -} - -/// Converts OneLake https:// URL to abfss:// URL for Delta Lake access -/// From: https://onelake.dfs.fabric.microsoft.com/workspace/item.ItemType/Tables/schema/table -/// To: abfss://workspace@onelake.dfs.fabric.microsoft.com/item.ItemType/Tables/schema/table -fn convert_to_abfss_url(https_url: &str) -> Option { - let url = https_url.strip_prefix("https://onelake.dfs.fabric.microsoft.com/")?; - let (workspace, rest) = url.split_once('/')?; - Some(format!( - "abfss://{}@onelake.dfs.fabric.microsoft.com/{}", - workspace, rest - )) -} - -/// Response from list schemas API -#[derive(Debug, Deserialize)] -struct ListSchemasResponse { - schemas: Vec, -} - -#[derive(Debug, Deserialize)] -struct SchemaInfo { - name: Option, - #[expect(dead_code)] - catalog_name: Option, - comment: Option, -} +use crate::credentials::OneLakeCredentials; -/// Response from list tables API -#[derive(Debug, Deserialize)] -struct ListTablesResponse { - tables: Vec, +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum OneLakeApiKind { + Delta, + Iceberg, } -#[derive(Debug, Deserialize)] -struct TableInfo { - name: Option, - #[expect(dead_code)] - catalog_name: Option, - schema_name: Option, - #[expect(dead_code)] - table_type: Option, - data_source_format: Option, - storage_location: Option, - columns: Option>, - comment: Option, -} +const ONE_LAKE_DELTA_ENDPOINT: &str = "https://onelake.table.fabric.microsoft.com/delta"; +const ONE_LAKE_ICEBERG_ENDPOINT: &str = "https://onelake.table.fabric.microsoft.com/iceberg"; -#[derive(Debug, Deserialize)] -struct ColumnInfo { - name: Option, - type_name: Option, - type_text: Option, - nullable: Option, - comment: Option, -} - -/// Configuration for OneLake catalog +/// Configuration for OneLake catalog. #[derive(Debug, Clone)] pub struct OneLakeCatalogConfig { + #[expect(unused)] pub workspace: String, pub item_name: String, - pub item_type: String, - pub bearer_token: Option, + pub item_type: Option, + pub api: OneLakeApiKind, } /// Provider for Microsoft Fabric OneLake catalog. /// -/// This catalog uses the OneLake Table APIs to list schemas and tables, -/// and returns Delta table locations that can be read/written via abfss://. +/// This catalog uses the OneLake Table APIs and delegates protocol-specific catalog +/// behavior to the Iceberg REST or Unity Catalog providers. pub struct OneLakeCatalogProvider { name: String, config: OneLakeCatalogConfig, - client: OnceCell>, + inner: Arc, } impl OneLakeCatalogProvider { @@ -135,191 +59,169 @@ impl OneLakeCatalogProvider { name: String, workspace: String, item_name: String, - item_type: String, + item_type: Option, + api: OneLakeApiKind, bearer_token: Option, - ) -> Self { - Self { + ) -> CatalogResult { + let credentials = match bearer_token { + Some(bearer_token) => OneLakeCredentials::Static { bearer_token }, + None => OneLakeCredentials::Dynamic { + workspace: workspace.clone(), + provider: OnceCell::new(), + }, + }; + let credentials = Arc::new(credentials) as Arc; + // `/.` for friendly names, or + // `/`(no type segment) for GUIDs + let item_path = onelake_item_path(&workspace, &item_name, item_type.as_deref()); + let inner: Arc = match api { + OneLakeApiKind::Delta => { + let uri = format!( + "{}/{}/api/2.1/unity-catalog", + ONE_LAKE_DELTA_ENDPOINT, item_path + ); + let catalog_name = onelake_catalog_name(&item_name, item_type.as_deref()); + Arc::new(UnityCatalogProvider::new( + name.clone(), + UnityCatalogOptions { + default_catalog: catalog_name, + uri, + credentials, + quote_object_name: false, + }, + )?) + } + OneLakeApiKind::Iceberg => { + let uri = ONE_LAKE_ICEBERG_ENDPOINT.to_string(); + let mut properties = HashMap::new(); + properties.insert(REST_CATALOG_PROP_URI.to_string(), uri); + properties.insert(REST_CATALOG_PROP_WAREHOUSE.to_string(), item_path.clone()); + properties.insert(REST_CATALOG_PROP_PREFIX.to_string(), item_path); + Arc::new(IcebergRestCatalogProvider::new( + name.clone(), + IcebergRestCatalogOptions { + credentials, + properties, + }, + )) + } + }; + + Ok(Self { name, config: OneLakeCatalogConfig { workspace, item_name, item_type, - bearer_token, + api, }, - client: OnceCell::new(), - } + inner, + }) } - fn base_url(&self) -> String { - format!( - "{}/{}/{}.{}", - ONELAKE_TABLE_API_BASE, - self.config.workspace, - self.config.item_name, - self.config.item_type - ) + fn catalog_name(&self) -> String { + onelake_catalog_name(&self.config.item_name, self.config.item_type.as_deref()) } - fn schema_name(database: &Namespace) -> CatalogResult { - if database.tail.is_empty() { - Ok(database.head.to_string()) - } else { - Err(CatalogError::InvalidArgument(format!( - "OneLake catalog does not support multi-level namespaces: {}", - quote_namespace_if_needed(database) - ))) + fn normalize_database_status(&self, mut status: DatabaseStatus) -> DatabaseStatus { + if matches!(self.config.api, OneLakeApiKind::Delta) { + status.database = self.normalize_delta_database(status.database); } + // OneLake's Iceberg REST API returns the namespace `location` as a workspace-relative + // path (e.g. `workspace/item.Type/Tables/schema`). `resolve_default_table_location` then + // seems to append the table name and hand the result to the Iceberg writer, which seems + // to reject any non-absolute location, so convert it to an absolute `abfss://` URL. + if let Some(location) = status.location.take() { + status.location = Some(normalize_onelake_location(&location).unwrap_or(location)); + } + // Keep the mirrored `location` table property consistent with the field above + // (DESCRIBE DATABASE EXTENDED surfaces both). + for (key, value) in status.properties.iter_mut() { + if key.eq_ignore_ascii_case("location") { + if let Some(abfss) = normalize_onelake_location(value) { + *value = abfss; + } + } + } + status } - fn catalog_name(&self) -> String { - format!("{}.{}", self.config.item_name, self.config.item_type) - } - - async fn get_client(&self) -> CatalogResult<&Arc> { - self.client - .get_or_try_init(|| async { - // Priority: 1) config token, 2) env var, 3) Azure CLI - let token = if let Some(token) = &self.config.bearer_token { - token.clone() - } else if let Ok(token) = std::env::var("AZURE_STORAGE_TOKEN") { - token - } else if let Ok(token) = std::env::var("AZURE_ACCESS_TOKEN") { - token - } else { - // Fall back to Azure CLI - get_azure_cli_token().await? - }; - - let mut headers = reqwest::header::HeaderMap::new(); - let header_value = reqwest::header::HeaderValue::from_str(&format!( - "Bearer {token}" - )) - .map_err(|e| CatalogError::External(format!("Invalid bearer token: {e}")))?; - headers.insert(reqwest::header::AUTHORIZATION, header_value); - - let client = Client::builder() - .default_headers(headers) - .build() - .map_err(|e| { - CatalogError::External(format!("Failed to build HTTP client: {e}")) - })?; - - Ok(Arc::new(client)) - }) - .await + fn normalize_table_status(&self, mut status: TableStatus) -> TableStatus { + if matches!(self.config.api, OneLakeApiKind::Delta) { + status.database = self.normalize_delta_database(status.database); + if let TableKind::Table { location, .. } = &mut status.kind { + if let Some(url) = location.take() { + *location = Some(normalize_onelake_location(&url).unwrap_or(url)); + } + } + } + status } - fn schema_info_to_database_status(&self, info: SchemaInfo) -> DatabaseStatus { - let schema_name = info.name.unwrap_or_else(|| "dbo".to_string()); - DatabaseStatus { - catalog: self.name.clone(), - database: vec![schema_name], - comment: info.comment, - location: None, - properties: vec![], + fn normalize_delta_database(&self, database: Vec) -> Vec { + if database.first().is_some_and(|x| x == &self.catalog_name()) { + database.into_iter().skip(1).collect() + } else { + database } } - fn table_info_to_table_status(&self, info: TableInfo) -> CatalogResult { - let name = info.name.unwrap_or_default(); - let schema_name = info.schema_name.unwrap_or_else(|| "dbo".to_string()); - let format = info - .data_source_format - .map(|f| f.to_lowercase()) - .unwrap_or_else(|| "delta".to_string()); - - // Convert https:// URL to abfss:// for Delta Lake access - let location = info - .storage_location - .and_then(|url| convert_to_abfss_url(&url).or(Some(url))); - - let columns = info - .columns - .unwrap_or_default() + fn normalize_database_statuses(&self, statuses: Vec) -> Vec { + statuses .into_iter() - .map(|col| { - // Prefer type_name over type_text (OneLake API often returns type_text as null) - let type_str = col - .type_name - .or(col.type_text) - .unwrap_or_else(|| "string".to_string()); - let data_type = parse_type_text(&type_str); - TableColumnStatus { - name: col.name.unwrap_or_default(), - data_type, - nullable: col.nullable.unwrap_or(true), - comment: col.comment, - default: None, - generated_always_as: None, - identity: None, - is_partition: false, - is_bucket: false, - is_cluster: false, - } - }) - .collect(); + .map(|status| self.normalize_database_status(status)) + .collect() + } - Ok(TableStatus { - catalog: Some(self.name.clone()), - database: vec![schema_name], - name, - kind: TableKind::Table { - columns, - comment: info.comment, - constraints: vec![], - location, - format, - partition_by: vec![], - sort_by: vec![], - bucket_by: None, - properties: vec![], - is_external: true, - }, - }) + fn normalize_table_statuses(&self, statuses: Vec) -> Vec { + statuses + .into_iter() + .map(|status| self.normalize_table_status(status)) + .collect() } } -/// Simple type text parser - converts Unity Catalog type strings to Arrow DataType -// TODO: Add support for complex types (array, map, struct) if OneLake returns them in JSON format. -fn parse_type_text(type_text: &str) -> arrow::datatypes::DataType { - use arrow::datatypes::DataType; +// OneLake reports locations in different forms depending on the API: +// - Delta (Unity) returns an absolute `https://` URL: +// From: https://onelake.dfs.fabric.microsoft.com/workspace/item.ItemType/Tables/schema/table +// To: abfss://workspace@onelake.dfs.fabric.microsoft.com/item.ItemType/Tables/schema/table +// - Iceberg (REST) returns the namespace `location` as a workspace-relative path: +// From: workspace/item.ItemType/Tables/schema +// To: abfss://workspace@onelake.dfs.fabric.microsoft.com/item.ItemType/Tables/schema +fn normalize_onelake_location(location: &str) -> Option { + let trimmed = location.trim(); + let relative = + if let Some(rest) = trimmed.strip_prefix("https://onelake.dfs.fabric.microsoft.com/") { + // OneLake `https://` form. + rest + } else if Url::parse(trimmed) + .ok() + .is_some_and(|url| url.scheme().len() > 1) + { + // Already absolute with another scheme (e.g., `abfss://`, `file://`, ...). + return None; + } else { + // Workspace-relative form (Iceberg namespace location). + trimmed + }; + let (workspace, rest) = relative.split_once('/')?; + (!workspace.is_empty() && !rest.is_empty()) + .then(|| format!("abfss://{workspace}@onelake.dfs.fabric.microsoft.com/{rest}")) +} - let lower = type_text.to_lowercase(); +// `/.` for friendly names, or `/` for GUIDs. +fn onelake_item_path(workspace: &str, item_name: &str, item_type: Option<&str>) -> String { + match item_type { + Some(item_type) => format!("{workspace}/{item_name}.{item_type}"), + None => format!("{workspace}/{item_name}"), + } +} - // Handle decimal(precision, scale) format - if lower.starts_with("decimal") { - if let Some(params) = lower - .strip_prefix("decimal(") - .and_then(|s| s.strip_suffix(')')) - { - let parts: Vec<&str> = params.split(',').collect(); - if parts.len() == 2 { - if let (Ok(precision), Ok(scale)) = - (parts[0].trim().parse::(), parts[1].trim().parse::()) - { - return DataType::Decimal128(precision, scale); - } - } - } - // Default decimal if parsing fails - return DataType::Decimal128(38, 18); - } - - match lower.as_str() { - "boolean" | "bool" => DataType::Boolean, - "byte" | "tinyint" => DataType::Int8, - "short" | "smallint" => DataType::Int16, - "int" | "integer" => DataType::Int32, - "long" | "bigint" => DataType::Int64, - "float" | "real" => DataType::Float32, - "double" => DataType::Float64, - "string" | "varchar" | "text" => DataType::Utf8, - "binary" => DataType::Binary, - "date" => DataType::Date32, - "timestamp" | "timestamp_ntz" => { - DataType::Timestamp(arrow::datatypes::TimeUnit::Microsecond, None) - } - _ => DataType::Utf8, // Default to string for unknown types +// `.` for friendly names, or `` for GUIDs. +fn onelake_catalog_name(item_name: &str, item_type: Option<&str>) -> String { + match item_type { + Some(item_type) => format!("{item_name}.{item_type}"), + None => item_name.to_string(), } } @@ -331,238 +233,209 @@ impl CatalogProvider for OneLakeCatalogProvider { async fn create_database( &self, - _database: &Namespace, - _options: CreateDatabaseOptions, + database: &Namespace, + options: CreateDatabaseOptions, ) -> CatalogResult { - Err(CatalogError::NotSupported( - "OneLake catalog does not support creating databases".to_string(), - )) + let status = self.inner.create_database(database, options).await?; + Ok(self.normalize_database_status(status)) } async fn get_database(&self, database: &Namespace) -> CatalogResult { - let schema_name = Self::schema_name(database)?; - let client = self.get_client().await?; - - // OneLake API requires full qualified schema name: catalog.schema - let full_schema_name = format!("{}.{}", self.catalog_name(), schema_name); - let url = format!( - "{}/api/2.1/unity-catalog/schemas/{}", - self.base_url(), - full_schema_name - ); - - let response = client - .head(&url) - .send() - .await - .map_err(|e| CatalogError::External(format!("Failed to check schema: {e}")))?; - - if response.status().is_success() { - Ok(DatabaseStatus { - catalog: self.name.clone(), - database: vec![schema_name], - comment: None, - location: None, - properties: vec![], - }) - } else if response.status().as_u16() == 404 { - Err(CatalogError::NotFound(CatalogObject::Schema, schema_name)) - } else { - Err(CatalogError::External(format!( - "Failed to get schema: HTTP {}", - response.status() - ))) - } + let status = self.inner.get_database(database).await?; + Ok(self.normalize_database_status(status)) } async fn list_databases( &self, - _prefix: Option<&Namespace>, + prefix: Option<&Namespace>, ) -> CatalogResult> { - let client = self.get_client().await?; - - // TODO: Use Url methods to construct URLs in case catalog/schema/table names contain special characters. - let url = format!( - "{}/api/2.1/unity-catalog/schemas?catalog_name={}", - self.base_url(), - self.catalog_name() - ); - - let response = client - .get(&url) - .send() - .await - .map_err(|e| CatalogError::External(format!("Failed to list schemas: {e}")))?; - - if !response.status().is_success() { - return Err(CatalogError::External(format!( - "Failed to list schemas: HTTP {}", - response.status() - ))); - } - - let list_response: ListSchemasResponse = response.json().await.map_err(|e| { - CatalogError::External(format!("Failed to parse schemas response: {e}")) - })?; - - Ok(list_response - .schemas - .into_iter() - .map(|s| self.schema_info_to_database_status(s)) - .collect()) + let statuses = self.inner.list_databases(prefix).await?; + Ok(self.normalize_database_statuses(statuses)) } async fn drop_database( &self, - _database: &Namespace, - _options: DropDatabaseOptions, + database: &Namespace, + options: DropDatabaseOptions, ) -> CatalogResult<()> { - Err(CatalogError::NotSupported( - "OneLake catalog does not support dropping databases".to_string(), - )) + self.inner.drop_database(database, options).await } async fn create_table( &self, - _database: &Namespace, - _table: &str, - _options: CreateTableOptions, + database: &Namespace, + table: &str, + options: CreateTableOptions, ) -> CatalogResult { - Err(CatalogError::NotSupported( - "OneLake catalog does not support creating tables via API".to_string(), - )) + let status = self.inner.create_table(database, table, options).await?; + Ok(self.normalize_table_status(status)) } - async fn get_table(&self, database: &Namespace, table: &str) -> CatalogResult { - let schema_name = Self::schema_name(database)?; - let client = self.get_client().await?; - - let full_table_name = format!("{}.{}.{}", self.catalog_name(), schema_name, table); - let url = format!( - "{}/api/2.1/unity-catalog/tables/{}", - self.base_url(), - full_table_name - ); - - let response = client - .get(&url) - .send() - .await - .map_err(|e| CatalogError::External(format!("Failed to get table: {e}")))?; - - if response.status().as_u16() == 404 { - return Err(CatalogError::NotFound( - CatalogObject::Table, - table.to_string(), - )); - } + fn create_table_metadata_requirement( + &self, + options: &CreateTableOptions, + ) -> CatalogResult { + self.inner.create_table_metadata_requirement(options) + } - if !response.status().is_success() { - return Err(CatalogError::External(format!( - "Failed to get table: HTTP {}", - response.status() - ))); - } + fn lakehouse_capabilities(&self) -> Vec { + self.inner.lakehouse_capabilities() + } - let table_info: TableInfo = response - .json() + async fn resolve_lakehouse_table( + &self, + database: &Namespace, + table: &str, + request: ResolveLakehouseTableRequest, + ) -> CatalogResult { + self.inner + .resolve_lakehouse_table(database, table, request) .await - .map_err(|e| CatalogError::External(format!("Failed to parse table response: {e}")))?; + } - self.table_info_to_table_status(table_info) + async fn plan_lakehouse_create( + &self, + database: &Namespace, + table: &str, + request: LakehouseCreateRequest, + ) -> CatalogResult { + self.inner + .plan_lakehouse_create(database, table, request) + .await } - async fn list_tables(&self, database: &Namespace) -> CatalogResult> { - let schema_name = Self::schema_name(database)?; - let client = self.get_client().await?; - - let url = format!( - "{}/api/2.1/unity-catalog/tables?catalog_name={}&schema_name={}", - self.base_url(), - self.catalog_name(), - schema_name - ); + async fn begin_table_access( + &self, + database: &Namespace, + table: &str, + request: BeginTableAccessRequest, + ) -> CatalogResult { + self.inner + .begin_table_access(database, table, request) + .await + } - let response = client - .get(&url) - .send() + async fn plan_lakehouse_scan( + &self, + database: &Namespace, + table: &str, + request: LakehouseScanPlanningRequest, + ) -> CatalogResult { + self.inner + .plan_lakehouse_scan(database, table, request) .await - .map_err(|e| CatalogError::External(format!("Failed to list tables: {e}")))?; + } - if !response.status().is_success() { - return Err(CatalogError::External(format!( - "Failed to list tables: HTTP {}", - response.status() - ))); - } + async fn commit_lakehouse_table( + &self, + database: &Namespace, + table: &str, + request: LakehouseCommitRequest, + ) -> CatalogResult { + self.inner + .commit_lakehouse_table(database, table, request) + .await + } - let list_response: ListTablesResponse = response - .json() + async fn get_delta_ratified_commits( + &self, + database: &Namespace, + table: &str, + request: DeltaRatifiedCommitRequest, + ) -> CatalogResult { + self.inner + .get_delta_ratified_commits(database, table, request) .await - .map_err(|e| CatalogError::External(format!("Failed to parse tables response: {e}")))?; + } - list_response - .tables - .into_iter() - .map(|t| self.table_info_to_table_status(t)) - .collect() + async fn get_table(&self, database: &Namespace, table: &str) -> CatalogResult { + let status = self.inner.get_table(database, table).await?; + Ok(self.normalize_table_status(status)) + } + + async fn list_tables(&self, database: &Namespace) -> CatalogResult> { + let statuses = self.inner.list_tables(database).await?; + Ok(self.normalize_table_statuses(statuses)) } async fn drop_table( &self, - _database: &Namespace, - _table: &str, - _options: DropTableOptions, + database: &Namespace, + table: &str, + options: DropTableOptions, ) -> CatalogResult<()> { - Err(CatalogError::NotSupported( - "OneLake catalog does not support dropping tables via API".to_string(), - )) + self.inner.drop_table(database, table, options).await } async fn alter_table( &self, - _database: &Namespace, - _table: &str, - _options: AlterTableOptions, + database: &Namespace, + table: &str, + options: AlterTableOptions, ) -> CatalogResult<()> { - // OneLake tables commonly use Delta storage, and property updates may already - // be committed at the storage layer before the catalog provider is called. - // Until OneLake REST propagation is implemented, treat this as a no-op so we - // do not report a failure after the underlying table has already been altered. - Ok(()) + self.inner.alter_table(database, table, options).await } async fn create_view( &self, - _database: &Namespace, - _view: &str, - _options: CreateViewOptions, + database: &Namespace, + view: &str, + options: CreateViewOptions, ) -> CatalogResult { - Err(CatalogError::NotSupported( - "OneLake catalog does not support views".to_string(), - )) + let status = self.inner.create_view(database, view, options).await?; + Ok(self.normalize_table_status(status)) } - async fn get_view(&self, _database: &Namespace, _view: &str) -> CatalogResult { - Err(CatalogError::NotSupported( - "OneLake catalog does not support views".to_string(), - )) + async fn get_view(&self, database: &Namespace, view: &str) -> CatalogResult { + let status = self.inner.get_view(database, view).await?; + Ok(self.normalize_table_status(status)) } - async fn list_views(&self, _database: &Namespace) -> CatalogResult> { - Err(CatalogError::NotSupported( - "OneLake catalog does not support views".to_string(), - )) + async fn list_views(&self, database: &Namespace) -> CatalogResult> { + let statuses = self.inner.list_views(database).await?; + Ok(self.normalize_table_statuses(statuses)) } async fn drop_view( &self, - _database: &Namespace, - _view: &str, - _options: DropViewOptions, + database: &Namespace, + view: &str, + options: DropViewOptions, ) -> CatalogResult<()> { - Err(CatalogError::NotSupported( - "OneLake catalog does not support views".to_string(), - )) + self.inner.drop_view(database, view, options).await + } +} + +#[cfg(test)] +mod tests { + use super::normalize_onelake_location; + + #[test] + fn normalize_onelake_location_handles_each_form() { + // Iceberg: workspace-relative namespace location -> abfss (the reported bug). + assert_eq!( + normalize_onelake_location("OneLake_LakeSail_Testing/LakeSail.Lakehouse/Tables/dbo") + .as_deref(), + Some("abfss://OneLake_LakeSail_Testing@onelake.dfs.fabric.microsoft.com/LakeSail.Lakehouse/Tables/dbo"), + ); + // Delta/Unity: absolute https OneLake URL -> abfss (unchanged from before). + assert_eq!( + normalize_onelake_location( + "https://onelake.dfs.fabric.microsoft.com/ws/item.Lakehouse/Tables/dbo/t", + ) + .as_deref(), + Some("abfss://ws@onelake.dfs.fabric.microsoft.com/item.Lakehouse/Tables/dbo/t"), + ); + // Already absolute -> None so the caller keeps the original (never double-convert). + assert_eq!( + normalize_onelake_location("abfss://ws@onelake.dfs.fabric.microsoft.com/x"), + None, + ); + assert_eq!(normalize_onelake_location("s3://bucket/x"), None); + // Single-slash scheme: `Url::parse` should detect it. + assert_eq!(normalize_onelake_location("file:/tmp/onelake"), None); + // Degenerate inputs -> no conversion. + assert_eq!(normalize_onelake_location("single-segment"), None); + assert_eq!(normalize_onelake_location("ws/"), None); } } diff --git a/crates/sail-catalog-unity/build.rs b/crates/sail-catalog-unity/build.rs index d1735fa92..3f3b36b3b 100644 --- a/crates/sail-catalog-unity/build.rs +++ b/crates/sail-catalog-unity/build.rs @@ -35,15 +35,100 @@ fn build_unity_catalog() -> Result<(), Box> { "Self::new_with_client(baseurl, client.build().unwrap())", "Ok(Self::new_with_client(baseurl, client.build()?))", ) + .replace( + "pub(crate) client: reqwest::Client,\n}", + "pub(crate) client: reqwest::Client,\n pub(crate) request_headers: reqwest::header::HeaderMap,\n}", + ) + .replace( + "Self {\n baseurl: baseurl.to_string(),\n client,\n }\n }\n}", + "Self::new_with_client_and_headers(baseurl, client, reqwest::header::HeaderMap::new())\n }\n\n /// Construct a new client with an existing `reqwest::Client` and per-request headers.\n pub fn new_with_client_and_headers(\n baseurl: &str,\n client: reqwest::Client,\n request_headers: reqwest::header::HeaderMap,\n ) -> Self {\n Self {\n baseurl: baseurl.to_string(),\n client,\n request_headers,\n }\n }\n}", + ) + .replace( + "impl ClientHooks<()> for &Client {}\n", + "impl ClientHooks<()> for &Client {}\nimpl ClientHooks<()> for Client {\n async fn pre(\n &self,\n request: &mut reqwest::Request,\n _info: &OperationInfo,\n ) -> ::std::result::Result<(), Error> {\n request.headers_mut().extend(self.request_headers.clone());\n Ok(())\n }\n}\n", + ) // We need to prevent the code examples from being treated as ignored tests. // These code snippets do not compile when running `cargo test -- --ignored`. .replace("```ignore", "```notrust"); + let content = add_column_type_name_aliases(content); + let content = add_nullable_array_deserializer(content); + let content = add_table_info_columns_deserializer(content); std::fs::write(out_file, content)?; Ok(()) } +fn add_column_type_name_aliases(content: String) -> String { + // OneLake returns data types in uppercase. + [ + ("BOOLEAN", "boolean", "Boolean"), + ("BYTE", "byte", "Byte"), + ("SHORT", "short", "Short"), + ("INT", "int", "Int"), + ("LONG", "long", "Long"), + ("FLOAT", "float", "Float"), + ("DOUBLE", "double", "Double"), + ("DATE", "date", "Date"), + ("TIMESTAMP", "timestamp", "Timestamp"), + ("TIMESTAMP_NTZ", "timestamp_ntz", "TimestampNtz"), + ("STRING", "string", "String"), + ("BINARY", "binary", "Binary"), + ("DECIMAL", "decimal", "Decimal"), + ("INTERVAL", "interval", "Interval"), + ("ARRAY", "array", "Array"), + ("STRUCT", "struct", "Struct"), + ("MAP", "map", "Map"), + ("CHAR", "char", "Char"), + ("NULL", "null", "Null"), + ("USER_DEFINED_TYPE", "user_defined_type", "UserDefinedType"), + ("TABLE_TYPE", "table_type", "TableType"), + ] + .into_iter() + .fold(content, |content, (name, alias, variant)| { + content.replace( + &format!("#[serde(rename = \"{name}\")]\n {variant},"), + &format!("#[serde(rename = \"{name}\", alias = \"{alias}\")]\n {variant},"), + ) + }) +} + +fn add_nullable_array_deserializer(content: String) -> String { + // OneLake may return `null` for columns when listing tables. + content.replace( + "pub mod types {\n", + r#"pub mod types { + fn deserialize_null_as_default<'de, D, T>( + deserializer: D, + ) -> ::std::result::Result + where + D: ::serde::Deserializer<'de>, + T: ::std::default::Default + ::serde::Deserialize<'de>, + { + Ok(<::std::option::Option as ::serde::Deserialize>::deserialize(deserializer)? + .unwrap_or_default()) + } +"#, + ) +} + +fn add_table_info_columns_deserializer(content: String) -> String { + let marker = " pub struct TableInfo {\n ///Name of parent catalog."; + let old = r#" #[serde(default, skip_serializing_if = "::std::vec::Vec::is_empty")] + pub columns: ::std::vec::Vec,"#; + let new = r#" #[serde( + default, + deserialize_with = "deserialize_null_as_default", + skip_serializing_if = "::std::vec::Vec::is_empty" + )] + pub columns: ::std::vec::Vec,"#; + if let Some((before, after)) = content.split_once(marker) { + format!("{before}{marker}{}", after.replacen(old, new, 1)) + } else { + content + } +} + fn main() -> Result<(), Box> { println!("cargo:rerun-if-changed=build.rs"); build_unity_catalog()?; diff --git a/crates/sail-catalog-unity/src/config.rs b/crates/sail-catalog-unity/src/config.rs index 57cd50f3a..48ac2ccbf 100644 --- a/crates/sail-catalog-unity/src/config.rs +++ b/crates/sail-catalog-unity/src/config.rs @@ -22,7 +22,6 @@ use std::collections::HashMap; use std::str::FromStr; use sail_catalog::error::{CatalogError, CatalogResult}; -use sail_catalog::utils::quote_name_if_needed; use secrecy::{ExposeSecret, SecretString}; use crate::credential::{ @@ -221,7 +220,6 @@ impl AsRef for UnityCatalogConfigKey { #[derive(Debug)] pub struct UnityCatalogConfig { - pub default_catalog: String, pub uri: String, pub bearer_token: Option, pub client_id: Option, @@ -234,7 +232,6 @@ pub struct UnityCatalogConfig { pub federated_token_file: Option, pub use_azure_cli: bool, pub allow_http_url: bool, - pub credential_provider: Option, } fn str_is_truthy(value: &str) -> bool { @@ -248,15 +245,11 @@ fn str_is_truthy(value: &str) -> bool { impl UnityCatalogConfig { pub fn new( - default_catalog: Option, uri: Option, token: &Option, options: Option>, ) -> CatalogResult { let mut config = Self { - default_catalog: quote_name_if_needed( - &default_catalog.unwrap_or_else(|| "unity".to_string()), - ), uri: uri.unwrap_or_else(|| DEFAULT_URI.to_string()), bearer_token: None, client_id: None, @@ -269,7 +262,6 @@ impl UnityCatalogConfig { federated_token_file: None, use_azure_cli: false, allow_http_url: false, - credential_provider: None, }; if let Some(token) = token { @@ -283,8 +275,6 @@ impl UnityCatalogConfig { config.apply_env()?; - config.credential_provider = config.get_credential_provider(); - Ok(config) } diff --git a/crates/sail-catalog-unity/src/credential.rs b/crates/sail-catalog-unity/src/credential.rs index a35ee9c2d..5d0bef97b 100644 --- a/crates/sail-catalog-unity/src/credential.rs +++ b/crates/sail-catalog-unity/src/credential.rs @@ -23,6 +23,7 @@ use std::time::{Duration, Instant}; use reqwest::header::{HeaderValue, ACCEPT}; use reqwest::{Client, Method, Response}; +use sail_catalog::credentials::CatalogCredentials; use sail_catalog::error::{CatalogError, CatalogResult}; use serde::de::DeserializeOwned; use serde::Deserialize; @@ -69,6 +70,24 @@ pub enum CredentialProvider { TokenCredential(TokenCache, Box), } +#[async_trait::async_trait] +impl CatalogCredentials for CredentialProvider { + async fn retrieve(&self) -> CatalogResult> { + match self { + CredentialProvider::BearerToken(token) => Ok(Some(token.clone())), + CredentialProvider::TokenCredential(cache, cred) => { + let oauth_client = Client::builder().build().map_err(|e| { + CatalogError::External(format!("Failed to build OAuth client: {e}")) + })?; + let token = cache + .get_or_insert_with(|| cred.fetch_token(&oauth_client)) + .await?; + Ok(Some(token)) + } + } + } +} + #[derive(Deserialize, Debug)] struct TokenResponse { access_token: String, diff --git a/crates/sail-catalog-unity/src/lib.rs b/crates/sail-catalog-unity/src/lib.rs index ccb5fae74..33d1ad099 100644 --- a/crates/sail-catalog-unity/src/lib.rs +++ b/crates/sail-catalog-unity/src/lib.rs @@ -20,4 +20,5 @@ pub mod unity { include!(concat!(env!("OUT_DIR"), "/unity_catalog.rs")); } -pub use provider::UnityCatalogProvider; +pub use config::UnityCatalogConfig; +pub use provider::{UnityCatalogOptions, UnityCatalogProvider}; diff --git a/crates/sail-catalog-unity/src/provider.rs b/crates/sail-catalog-unity/src/provider.rs index 0da87b956..a73241579 100644 --- a/crates/sail-catalog-unity/src/provider.rs +++ b/crates/sail-catalog-unity/src/provider.rs @@ -12,9 +12,10 @@ use std::collections::HashMap; use std::str::FromStr; +use std::sync::Arc; use arrow::datatypes::DataType; -use reqwest::header::HeaderValue; +use sail_catalog::credentials::CatalogCredentials; use sail_catalog::error::{CatalogError, CatalogObject, CatalogResult}; use sail_catalog::lakehouse::{ DeltaRatifiedCommit, DeltaRatifiedCommitRequest, DeltaRatifiedCommitResponse, @@ -26,18 +27,15 @@ use sail_catalog::provider::{ CreateViewOptions, DropDatabaseOptions, DropTableOptions, DropViewOptions, Namespace, TableFormatCreateMetadataMode, }; -use sail_catalog::utils::{get_property, quote_namespace_if_needed}; +use sail_catalog::utils::{get_property, quote_name_if_needed, quote_namespace_if_needed}; +use sail_common::http::SAIL_USER_AGENT; use sail_common_datafusion::catalog::delta::{ unity_table_id_value, DELTA_UNITY_TABLE_ID_KEY, DELTA_UNITY_TABLE_ID_LEGACY_KEY, }; use sail_common_datafusion::catalog::{ identity_partition_fields, DatabaseStatus, TableColumnStatus, TableKind, TableStatus, }; -use secrecy::SecretString; -use tokio::sync::OnceCell; -use crate::config::UnityCatalogConfig; -use crate::credential::CredentialProvider; use crate::data_type::{ data_type_to_unity_type, unity_struct_field_type_json, unity_type_to_data_type, }; @@ -48,82 +46,57 @@ pub(crate) const DEFAULT_URI: &str = "http://localhost:8080/api/2.1/unity-catalo /// Provider for Unity Catalog pub struct UnityCatalogProvider { name: String, - catalog_config: UnityCatalogConfig, - client: OnceCell, + options: UnityCatalogOptions, + http_client: reqwest::Client, } -impl UnityCatalogProvider { - pub fn new( - name: String, - default_catalog: &Option, - uri: &Option, - token: &Option, - ) -> CatalogResult { - Self::with_options(name, default_catalog.clone(), uri.clone(), token, None) - } +#[derive(Debug, Clone)] +pub struct UnityCatalogOptions { + pub default_catalog: String, + pub uri: String, + pub credentials: Arc, + pub quote_object_name: bool, +} - pub fn with_options( - name: String, - default_catalog: Option, - uri: Option, - token: &Option, - options: Option>, - ) -> CatalogResult { - let catalog_config = UnityCatalogConfig::new(default_catalog, uri, token, options)?; +impl UnityCatalogProvider { + pub fn new(name: String, options: UnityCatalogOptions) -> CatalogResult { + let http_client = reqwest::Client::new(); Ok(Self { name, - catalog_config, - client: OnceCell::new(), + options, + http_client, }) } - async fn get_client(&self) -> CatalogResult<&Client> { - let config = &self.catalog_config; - let client = self - .client - .get_or_try_init(|| async move { - let mut client_builder = reqwest::Client::builder(); - - if let Some(credential_provider) = &config.credential_provider { - let header = match credential_provider { - CredentialProvider::BearerToken(token) => { - HeaderValue::from_str(&format!("Bearer {token}")).map_err(|e| { - CatalogError::External(format!( - "Failed to create header value from token: {e}" - )) - }) - } - CredentialProvider::TokenCredential(cache, cred) => { - let oauth_client = reqwest::Client::builder().build().map_err(|e| { - CatalogError::External(format!("Failed to build OAuth client: {e}")) - })?; - let token = cache - .get_or_insert_with(|| cred.fetch_token(&oauth_client)) - .await?; - HeaderValue::from_str(&format!("Bearer {token}")).map_err(|e| { - CatalogError::External(format!( - "Failed to create header value from token: {e}" - )) - }) - } - }?; - let mut headers = reqwest::header::HeaderMap::new(); - headers.insert(reqwest::header::AUTHORIZATION, header); - client_builder = client_builder.default_headers(headers); - } + async fn get_client(&self) -> CatalogResult { + let mut headers = reqwest::header::HeaderMap::new(); - if config.allow_http_url { - client_builder = client_builder.https_only(false); - } - - let reqwest_client = client_builder.build().map_err(|e| { - CatalogError::External(format!("Failed to build HTTP client: {e}")) + if let Some(token) = self.options.credentials.retrieve().await? { + let header = reqwest::header::HeaderValue::from_str(&format!("Bearer {token}")) + .map_err(|e| { + CatalogError::External(format!("Failed to create header value from token: {e}")) })?; + headers.insert(reqwest::header::AUTHORIZATION, header); + } - Ok::<_, CatalogError>(Client::new_with_client(&config.uri, reqwest_client)) - }) - .await?; - Ok(client) + headers.insert( + reqwest::header::USER_AGENT, + reqwest::header::HeaderValue::from_static(SAIL_USER_AGENT), + ); + + Ok(Client::new_with_client_and_headers( + &self.options.uri, + self.http_client.clone(), + headers, + )) + } + + fn object_name(&self, name: &str) -> String { + if self.options.quote_object_name { + quote_name_if_needed(name) + } else { + name.to_string() + } } fn get_catalog_and_schema_name( @@ -132,7 +105,7 @@ impl UnityCatalogProvider { ) -> CatalogResult<(String, String)> { match namespace.tail.as_slice() { [] => Ok(( - self.catalog_config.default_catalog.to_string(), + self.options.default_catalog.to_string(), namespace.head.to_string(), )), [x] => Ok((namespace.head.to_string(), x.to_string())), @@ -143,15 +116,12 @@ impl UnityCatalogProvider { } } - // Unity Catalog does not quote names in full names even if they contain special characters, - // so we only concatenate the names with dot ('.'). - - fn get_full_schema_name(catalog_name: &str, schema_name: &str) -> String { - format!("{catalog_name}.{schema_name}") - } - - fn get_full_table_name(catalog_name: &str, schema_name: &str, table_name: &str) -> String { - format!("{catalog_name}.{schema_name}.{table_name}") + fn qualified_object_name(&self, names: &[&str]) -> String { + names + .iter() + .map(|name| self.object_name(name)) + .collect::>() + .join(".") } fn schema_info_to_database_status( @@ -241,6 +211,10 @@ impl UnityCatalogProvider { let database = vec![catalog.clone(), schema]; let mut partition_by: Vec = vec![]; + // If all partition indices are 0, treat the table as unpartitioned. + // This can happen in the Unity Catalog endpoint in OneLake. + let ignore_partition_index = + !columns.is_empty() && columns.iter().all(|col| col.partition_index == Some(0)); let columns = columns .into_iter() .map(|col| { @@ -257,6 +231,11 @@ impl UnityCatalogProvider { type_scale, type_text, } = col; + let partition_index = if ignore_partition_index { + None + } else { + partition_index + }; if partition_index.is_some() { if let Some(col_name) = &name { partition_by.push(col_name.to_string()); @@ -377,8 +356,8 @@ impl CatalogProvider for UnityCatalogProvider { let (catalog_name, schema_name) = self.get_catalog_and_schema_name(database)?; let request = types::CreateSchema::builder() - .catalog_name(&catalog_name) - .name(&schema_name) + .catalog_name(self.object_name(&catalog_name)) + .name(self.object_name(&schema_name)) .comment(comment) .properties(if props.is_empty() { None @@ -415,7 +394,7 @@ impl CatalogProvider for UnityCatalogProvider { .map_err(|e| CatalogError::External(format!("Failed to load config: {e}")))?; let (catalog_name, schema_name) = self.get_catalog_and_schema_name(database)?; - let full_name = Self::get_full_schema_name(&catalog_name, &schema_name); + let full_name = self.qualified_object_name(&[&catalog_name, &schema_name]); let result = client.get_schema().full_name(&full_name).send().await; @@ -447,7 +426,7 @@ impl CatalogProvider for UnityCatalogProvider { .map_err(|e| CatalogError::External(format!("Failed to load config: {e}")))?; let catalog_name = match prefix { - None => self.catalog_config.default_catalog.to_string(), + None => self.options.default_catalog.to_string(), Some(Namespace { head, tail }) if tail.is_empty() => head.to_string(), Some(x) => { return Err(CatalogError::InvalidArgument(format!( @@ -458,7 +437,7 @@ impl CatalogProvider for UnityCatalogProvider { }; let result = client .list_schemas() - .catalog_name(&catalog_name) + .catalog_name(self.object_name(&catalog_name)) .send() .await; @@ -492,7 +471,7 @@ impl CatalogProvider for UnityCatalogProvider { .map_err(|e| CatalogError::External(format!("Failed to load config: {e}")))?; let (catalog_name, schema_name) = self.get_catalog_and_schema_name(database)?; - let full_name = Self::get_full_schema_name(&catalog_name, &schema_name); + let full_name = self.qualified_object_name(&[&catalog_name, &schema_name]); let result = client .delete_schema() @@ -508,6 +487,7 @@ impl CatalogProvider for UnityCatalogProvider { { Ok(()) } + Err(e) if e.status().is_some_and(|status| status.is_success()) => Ok(()), Err(progenitor_client::Error::UnexpectedResponse(response)) if response.status().as_u16() == 404 && if_exists => { @@ -668,17 +648,17 @@ impl CatalogProvider for UnityCatalogProvider { // The SQL planner supplies generated default locations for managed tables. Unity // managed table creation must use a staging location allocated by the catalog. let request = types::CreateStagingTable::builder() - .name(table) - .catalog_name(&catalog_name) - .schema_name(&schema_name); - let response = client - .create_staging_table() - .body(request) - .send() - .await - .map_err(|e| { - CatalogError::External(format!("Failed to create staging table: {e}")) - })?; + .name(self.object_name(table)) + .catalog_name(self.object_name(&catalog_name)) + .schema_name(self.object_name(&schema_name)); + let response = match client.create_staging_table().body(request).send().await { + Ok(response) => response, + Err(e) => { + return Err(CatalogError::External(format!( + "Failed to create staging table: {e}" + ))); + } + }; response.into_inner().staging_location.ok_or_else(|| { CatalogError::External( "Unity Catalog staging table response is missing a staging location" @@ -688,9 +668,9 @@ impl CatalogProvider for UnityCatalogProvider { }; let request = types::CreateTable::builder() - .name(table) - .catalog_name(&catalog_name) - .schema_name(&schema_name) + .name(self.object_name(table)) + .catalog_name(self.object_name(&catalog_name)) + .schema_name(self.object_name(&schema_name)) .table_type(table_type) .data_source_format(data_source_format) .columns(unity_columns) @@ -737,7 +717,7 @@ impl CatalogProvider for UnityCatalogProvider { .map_err(|e| CatalogError::External(format!("Failed to load config: {e}")))?; let (catalog_name, schema_name) = self.get_catalog_and_schema_name(database)?; - let full_name = Self::get_full_table_name(&catalog_name, &schema_name, table); + let full_name = self.qualified_object_name(&[&catalog_name, &schema_name, table]); let result = client.get_table().full_name(&full_name).send().await; @@ -765,8 +745,8 @@ impl CatalogProvider for UnityCatalogProvider { let result = client .list_tables() - .catalog_name(&catalog_name) - .schema_name(&schema_name) + .catalog_name(self.object_name(&catalog_name)) + .schema_name(self.object_name(&schema_name)) .send() .await; @@ -807,7 +787,7 @@ impl CatalogProvider for UnityCatalogProvider { .map_err(|e| CatalogError::External(format!("Failed to load config: {e}")))?; let (catalog_name, schema_name) = self.get_catalog_and_schema_name(database)?; - let full_name = Self::get_full_table_name(&catalog_name, &schema_name, table); + let full_name = self.qualified_object_name(&[&catalog_name, &schema_name, table]); match client.delete_table().full_name(full_name).send().await { Ok(_) => Ok(()), @@ -818,6 +798,7 @@ impl CatalogProvider for UnityCatalogProvider { { Ok(()) } + Err(e) if e.status().is_some_and(|status| status.is_success()) => Ok(()), Err(e) if e.status() .is_some_and(|status| status.as_u16() == 404 && if_exists) => @@ -937,7 +918,7 @@ impl CatalogProvider for UnityCatalogProvider { CatalogError::RateLimited(format!("Unity Catalog Delta commit rate limited: {e}")), ), Err(e) if e.status().is_some_and(|status| status.as_u16() == 501) => Err( - CatalogError::NotSupported("Unity Catalog Delta commit endpoint".to_string()), + CatalogError::NotSupported(format!("Unity Catalog Delta commit endpoint: {e}")), ), Err(e) => Err(CatalogError::External(format!( "Failed to commit Delta table to Unity Catalog: {e}" @@ -1003,7 +984,7 @@ impl CatalogProvider for UnityCatalogProvider { .await .map_err(|e| CatalogError::External(format!("Failed to load client: {e}")))?; let (catalog_name, schema_name) = self.get_catalog_and_schema_name(database)?; - let full_name = Self::get_full_table_name(&catalog_name, &schema_name, table); + let full_name = self.qualified_object_name(&[&catalog_name, &schema_name, table]); match client.get_commits().body(request).send().await { Ok(response) => { @@ -1032,9 +1013,9 @@ impl CatalogProvider for UnityCatalogProvider { Err(CatalogError::NotFound(CatalogObject::Table, full_name)) } Err(e) if e.status().is_some_and(|status| status.as_u16() == 501) => { - Err(CatalogError::NotSupported( - "Unity Catalog Delta commit discovery endpoint".to_string(), - )) + Err(CatalogError::NotSupported(format!( + "Unity Catalog Delta commit discovery endpoint: {e}" + ))) } Err(e) => Err(CatalogError::External(format!( "Failed to get Delta commits from Unity Catalog: {e}" @@ -1080,9 +1061,39 @@ impl CatalogProvider for UnityCatalogProvider { #[expect(clippy::unwrap_used)] #[cfg(test)] mod tests { + use sail_catalog::credentials::{CatalogCredentials, EmptyCatalogCredentials}; use secrecy::ExposeSecret; use super::*; + use crate::config::UnityCatalogConfig; + + fn options_from_config( + default_catalog: impl Into, + config: UnityCatalogConfig, + ) -> UnityCatalogOptions { + let credentials = config + .get_credential_provider() + .map(|credentials| Arc::new(credentials) as Arc) + .unwrap_or_else(|| Arc::new(EmptyCatalogCredentials)); + UnityCatalogOptions { + default_catalog: default_catalog.into(), + uri: config.uri, + credentials, + quote_object_name: true, + } + } + + fn empty_options( + default_catalog: impl Into, + quote_object_name: bool, + ) -> UnityCatalogOptions { + UnityCatalogOptions { + default_catalog: default_catalog.into(), + uri: DEFAULT_URI.to_string(), + credentials: Arc::new(EmptyCatalogCredentials), + quote_object_name, + } + } #[test] fn test_config_with_token() { @@ -1093,7 +1104,7 @@ mod tests { "https://test.databricks.com".to_string(), ); - let config = UnityCatalogConfig::new(None, None, &None, Some(options)).unwrap(); + let config = UnityCatalogConfig::new(None, &None, Some(options)).unwrap(); assert!(config.bearer_token.is_some()); assert_eq!(config.bearer_token.unwrap().expose_secret(), "test_token"); assert_eq!(config.uri, "https://test.databricks.com"); @@ -1119,7 +1130,7 @@ mod tests { "test_tenant".to_string(), ); - let config = UnityCatalogConfig::new(None, None, &None, Some(options)).unwrap(); + let config = UnityCatalogConfig::new(None, &None, Some(options)).unwrap(); assert_eq!(config.uri, "https://test.databricks.com"); assert_eq!(config.client_id, Some("test_client_id".to_string())); assert!(config.client_secret.is_some()); @@ -1142,7 +1153,7 @@ mod tests { options.insert("unity_allow_http_url".to_string(), value.to_string()); options.insert("unity_use_azure_cli".to_string(), value.to_string()); - let config = UnityCatalogConfig::new(None, None, &None, Some(options)).unwrap(); + let config = UnityCatalogConfig::new(None, &None, Some(options)).unwrap(); assert_eq!(config.allow_http_url, expected, "Failed for value: {value}"); assert_eq!(config.use_azure_cli, expected, "Failed for value: {value}"); } @@ -1166,7 +1177,7 @@ mod tests { let test_value = format!("test_value_for_{key}"); options.insert(key.to_string(), test_value.clone()); - let result = UnityCatalogConfig::new(None, None, &None, Some(options)); + let result = UnityCatalogConfig::new(None, &None, Some(options)); assert!(result.is_ok(), "Failed to parse key: {key}"); let config = result.unwrap(); @@ -1188,18 +1199,197 @@ mod tests { "https://test.databricks.com".to_string(), ); - let provider = UnityCatalogProvider::with_options( + let config = UnityCatalogConfig::new(None, &None, Some(options)).unwrap(); + assert!(config.bearer_token.is_some()); + + let provider = UnityCatalogProvider::new( "test".to_string(), - Some("test_catalog".to_string()), - None, - &None, - Some(options), + options_from_config("test_catalog", config), ) .unwrap(); assert_eq!(provider.name, "test"); - assert_eq!(provider.catalog_config.default_catalog, "test_catalog"); - assert_eq!(provider.catalog_config.uri, "https://test.databricks.com"); - assert!(provider.catalog_config.bearer_token.is_some()); + assert_eq!(provider.options.default_catalog, "test_catalog"); + assert_eq!(provider.options.uri, "https://test.databricks.com"); + } + + #[test] + fn test_provider_quotes_object_names_by_default() { + let config = UnityCatalogConfig::new(None, &None, None).unwrap(); + let provider = UnityCatalogProvider::new( + "test".to_string(), + options_from_config("catalog.name", config), + ) + .unwrap(); + + assert_eq!(provider.options.default_catalog, "catalog.name"); + assert_eq!( + provider.object_name(&provider.options.default_catalog), + "`catalog.name`" + ); + assert_eq!(provider.object_name("schema.name"), "`schema.name`"); + assert_eq!( + provider.qualified_object_name(&["catalog.name", "schema.name", "table.name"]), + "`catalog.name`.`schema.name`.`table.name`" + ); + let namespace = Namespace::try_from(vec!["schema.name"]).unwrap(); + assert_eq!( + provider.get_catalog_and_schema_name(&namespace).unwrap(), + ("catalog.name".to_string(), "schema.name".to_string()) + ); + } + + #[test] + fn test_provider_can_preserve_unquoted_object_names() { + let provider = UnityCatalogProvider::new( + "test".to_string(), + empty_options("lakehouse.Lakehouse", false), + ) + .unwrap(); + + assert_eq!(provider.options.default_catalog, "lakehouse.Lakehouse"); + assert_eq!( + provider.object_name(&provider.options.default_catalog), + "lakehouse.Lakehouse" + ); + assert_eq!(provider.object_name("schema.name"), "schema.name"); + assert_eq!( + provider.qualified_object_name(&["catalog.name", "schema.name", "table.name"]), + "catalog.name.schema.name.table.name" + ); + } + + #[test] + #[expect(clippy::panic)] + fn test_table_info_accepts_null_columns_and_lowercase_column_type_name() { + let response: types::ListTablesResponse = serde_json::from_value(serde_json::json!({ + "tables": [{ + "catalog_name": "catalog", + "schema_name": "schema", + "name": "empty_table", + "columns": null + }, { + "catalog_name": "catalog", + "schema_name": "schema", + "name": "typed_table", + "columns": [{ + "name": "value", + "type_name": "string", + "type_text": "string", + "nullable": true + }] + }] + })) + .unwrap(); + + let provider = + UnityCatalogProvider::new("test".to_string(), empty_options("catalog", true)).unwrap(); + + let mut tables = response.tables.into_iter(); + let empty_table = provider + .table_info_to_table_status(tables.next().unwrap(), "catalog", "schema") + .unwrap(); + let typed_table = provider + .table_info_to_table_status(tables.next().unwrap(), "catalog", "schema") + .unwrap(); + + let TableKind::Table { + columns, comment, .. + } = empty_table.kind + else { + panic!("expected table status"); + }; + assert_eq!(comment, None); + assert!(columns.is_empty()); + + let TableKind::Table { columns, .. } = typed_table.kind else { + panic!("expected table status"); + }; + assert_eq!(columns[0].data_type, DataType::Utf8); + } + + #[test] + #[expect(clippy::panic)] + fn test_table_info_ignores_onelake_all_zero_partition_indexes() { + let response: types::TableInfo = serde_json::from_value(serde_json::json!({ + "catalog_name": "catalog", + "schema_name": "schema", + "name": "typed_table", + "columns": [{ + "name": "id", + "type_name": "int", + "type_text": "int", + "nullable": true, + "partition_index": 0 + }, { + "name": "value", + "type_name": "string", + "type_text": "string", + "nullable": true, + "partition_index": 0 + }] + })) + .unwrap(); + + let provider = + UnityCatalogProvider::new("test".to_string(), empty_options("catalog", true)).unwrap(); + + let status = provider + .table_info_to_table_status(response, "catalog", "schema") + .unwrap(); + + let TableKind::Table { + columns, + partition_by, + .. + } = status.kind + else { + panic!("expected table status"); + }; + assert!(partition_by.is_empty()); + assert!(columns.iter().all(|column| !column.is_partition)); + } + + #[test] + #[expect(clippy::panic)] + fn test_table_info_preserves_real_partition_index() { + let response: types::TableInfo = serde_json::from_value(serde_json::json!({ + "catalog_name": "catalog", + "schema_name": "schema", + "name": "typed_table", + "columns": [{ + "name": "id", + "type_name": "int", + "type_text": "int", + "nullable": true + }, { + "name": "value", + "type_name": "string", + "type_text": "string", + "nullable": true, + "partition_index": 0 + }] + })) + .unwrap(); + + let provider = + UnityCatalogProvider::new("test".to_string(), empty_options("catalog", true)).unwrap(); + + let status = provider + .table_info_to_table_status(response, "catalog", "schema") + .unwrap(); + + let TableKind::Table { + columns, + partition_by, + .. + } = status.kind + else { + panic!("expected table status"); + }; + assert_eq!(partition_by.len(), 1); + assert_eq!(partition_by[0].column, "value"); + assert!(!columns[0].is_partition); + assert!(columns[1].is_partition); } } diff --git a/crates/sail-catalog-unity/tests/rest_integration_test.rs b/crates/sail-catalog-unity/tests/rest_integration_test.rs index 7af8c226f..4bfc7c714 100644 --- a/crates/sail-catalog-unity/tests/rest_integration_test.rs +++ b/crates/sail-catalog-unity/tests/rest_integration_test.rs @@ -16,13 +16,14 @@ use std::collections::HashMap; use std::sync::Arc; use arrow::datatypes::{DataType, Field, Fields}; +use sail_catalog::credentials::EmptyCatalogCredentials; use sail_catalog::provider::{ CatalogPartitionField, CatalogProvider, CreateDatabaseOptions, CreateTableColumnOptions, CreateTableMode, CreateTableOptions, DropDatabaseOptions, DropTableOptions, Namespace, RuntimeAwareCatalogProvider, }; use sail_catalog_unity::unity::{types, Client}; -use sail_catalog_unity::UnityCatalogProvider; +use sail_catalog_unity::{UnityCatalogOptions, UnityCatalogProvider}; use sail_common::runtime::RuntimeHandle; use sail_common::spec::{ SAIL_LIST_FIELD_NAME, SAIL_MAP_FIELD_NAME, SAIL_MAP_KEY_FIELD_NAME, SAIL_MAP_VALUE_FIELD_NAME, @@ -145,9 +146,12 @@ async fn setup_catalog( || { UnityCatalogProvider::new( "sail".to_string(), - &Some(DEFAULT_CATALOG.to_string()), - &Some(rest_url.clone()), - &None, + UnityCatalogOptions { + default_catalog: DEFAULT_CATALOG.to_string(), + uri: rest_url.clone(), + credentials: Arc::new(EmptyCatalogCredentials), + quote_object_name: true, + }, ) }, runtime.io().clone(), diff --git a/crates/sail-catalog/src/credentials.rs b/crates/sail-catalog/src/credentials.rs new file mode 100644 index 000000000..4f70dcf15 --- /dev/null +++ b/crates/sail-catalog/src/credentials.rs @@ -0,0 +1,36 @@ +use std::fmt::Debug; + +use crate::error::CatalogResult; + +#[async_trait::async_trait] +pub trait CatalogCredentials: Debug + Send + Sync + 'static { + async fn retrieve(&self) -> CatalogResult>; +} + +#[derive(Debug, Default)] +pub struct EmptyCatalogCredentials; + +#[async_trait::async_trait] +impl CatalogCredentials for EmptyCatalogCredentials { + async fn retrieve(&self) -> CatalogResult> { + Ok(None) + } +} + +#[derive(Debug)] +pub struct StaticCatalogCredentials { + credential: String, +} + +impl StaticCatalogCredentials { + pub fn new(credential: String) -> Self { + Self { credential } + } +} + +#[async_trait::async_trait] +impl CatalogCredentials for StaticCatalogCredentials { + async fn retrieve(&self) -> CatalogResult> { + Ok(Some(self.credential.clone())) + } +} diff --git a/crates/sail-catalog/src/lib.rs b/crates/sail-catalog/src/lib.rs index d129e1572..bcb97e8f9 100644 --- a/crates/sail-catalog/src/lib.rs +++ b/crates/sail-catalog/src/lib.rs @@ -1,4 +1,5 @@ pub mod command; +pub mod credentials; pub mod error; pub mod hive_format; pub mod lakehouse; diff --git a/crates/sail-catalog/src/manager/table.rs b/crates/sail-catalog/src/manager/table.rs index aaf047dcc..d2ab736ef 100644 --- a/crates/sail-catalog/src/manager/table.rs +++ b/crates/sail-catalog/src/manager/table.rs @@ -139,7 +139,7 @@ impl CatalogManager { // list_views; treat that as "no views" so SHOW TABLES still works there. let views = match views_res { Ok(v) => v, - Err(CatalogError::NotSupported(_)) => vec![], + Err(CatalogError::NotSupported(_)) | Err(CatalogError::NotFound(_, _)) => vec![], Err(e) => return Err(e), }; tables.extend(views); diff --git a/crates/sail-common/src/config/application.rs b/crates/sail-common/src/config/application.rs index c8bcd4da3..bc60cf90e 100644 --- a/crates/sail-common/src/config/application.rs +++ b/crates/sail-common/src/config/application.rs @@ -509,6 +509,14 @@ pub struct OptimizerConfig { pub expand_views_at_output: bool, } +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum OneLakeApi { + Delta, + #[default] + Iceberg, +} + #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(deny_unknown_fields)] #[serde(tag = "type", rename_all = "snake_case")] @@ -562,6 +570,8 @@ pub enum CatalogType { OneLake { name: String, url: String, + #[serde(default)] + api: OneLakeApi, #[serde( skip_serializing_if = "Option::is_none", serialize_with = "serialize_optional_secret" diff --git a/crates/sail-common/src/http.rs b/crates/sail-common/src/http.rs new file mode 100644 index 000000000..d3ba67ab1 --- /dev/null +++ b/crates/sail-common/src/http.rs @@ -0,0 +1,7 @@ +/// User agent for Sail requests to remote services. +/// +/// References: +/// * +/// * (obsolete) +/// * +pub const SAIL_USER_AGENT: &str = concat!("Sail/", env!("CARGO_PKG_VERSION")); diff --git a/crates/sail-common/src/lib.rs b/crates/sail-common/src/lib.rs index c25236311..f31ea09ab 100644 --- a/crates/sail-common/src/lib.rs +++ b/crates/sail-common/src/lib.rs @@ -3,6 +3,7 @@ pub mod datetime; pub mod debug; pub mod error; pub mod geoarrow; +pub mod http; pub mod object; pub mod runtime; pub mod spec; diff --git a/crates/sail-iceberg/src/table/mod.rs b/crates/sail-iceberg/src/table/mod.rs index 0457d26c0..cc92fcfae 100644 --- a/crates/sail-iceberg/src/table/mod.rs +++ b/crates/sail-iceberg/src/table/mod.rs @@ -44,7 +44,11 @@ impl Table { table_url: Url, metadata_location: Option, ) -> Result { - log::trace!("Loading Iceberg table: {}", table_url); + log::trace!( + "Loading Iceberg table: table_url={}, metadata_location={:?}", + table_url, + metadata_location, + ); let object_store = ctx .runtime_env() .object_store_registry diff --git a/crates/sail-iceberg/src/table_format.rs b/crates/sail-iceberg/src/table_format.rs index c54d827a3..63509ca67 100644 --- a/crates/sail-iceberg/src/table_format.rs +++ b/crates/sail-iceberg/src/table_format.rs @@ -24,6 +24,7 @@ use datafusion::physical_plan::ExecutionPlan; use datafusion_expr::expr::Sort; use datafusion_expr::{Expr, Extension, UserDefinedLogicalNodeCore}; use educe::Educe; +use log::warn; use object_store::ObjectStoreExt; use sail_common_datafusion::catalog::iceberg::is_iceberg_table_marker; use sail_common_datafusion::catalog::managed::metadata_location_value; @@ -708,8 +709,8 @@ fn validate_iceberg_lakehouse_storage_access( .is_some_and(|session| session.remote_signing_enabled) { // TODO: Wire REST remote signing into Iceberg FileIO/object-store access. - return not_impl_err!( - "Iceberg REST catalog table {} requires remote signing, which is not implemented yet", + warn!( + "Iceberg REST catalog table {} advertises remote signing, which is not implemented yet", context.catalog_table().join(".") ); } @@ -719,8 +720,8 @@ fn validate_iceberg_lakehouse_storage_access( .is_some_and(|session| session.storage_credential_count > 0) { // TODO: Apply REST vended credentials to operation-scoped storage access. - return not_impl_err!( - "Iceberg REST catalog table {} requires vended storage credentials, which is not implemented yet", + warn!( + "Iceberg REST catalog table {} advertises vended storage credentials, which is not implemented yet", context.catalog_table().join(".") ); } @@ -1257,7 +1258,7 @@ mod tests { } #[test] - fn storage_access_rejects_required_rest_remote_signing() { + fn storage_access_allows_required_rest_remote_signing() { let mut context = LakehouseExecutionContext::catalog_table_context( CatalogProviderId("rest".to_string()), vec!["rest".to_string(), "db".to_string(), "tbl".to_string()], @@ -1282,14 +1283,11 @@ mod tests { }); let result = validate_iceberg_lakehouse_storage_access(Some(&context)); - assert!(matches!( - &result, - Err(err) if format!("{err}").contains("requires remote signing") - )); + assert!(result.is_ok()); } #[test] - fn storage_access_rejects_required_rest_vended_credentials() { + fn storage_access_allows_required_rest_vended_credentials() { let mut context = LakehouseExecutionContext::catalog_table_context( CatalogProviderId("rest".to_string()), vec!["rest".to_string(), "db".to_string(), "tbl".to_string()], @@ -1314,9 +1312,6 @@ mod tests { }); let result = validate_iceberg_lakehouse_storage_access(Some(&context)); - assert!(matches!( - &result, - Err(err) if format!("{err}").contains("vended storage credentials") - )); + assert!(result.is_ok()); } } diff --git a/crates/sail-session/src/catalog.rs b/crates/sail-session/src/catalog.rs index 58975ac6b..e3fdf9478 100644 --- a/crates/sail-session/src/catalog.rs +++ b/crates/sail-session/src/catalog.rs @@ -3,6 +3,9 @@ use std::sync::Arc; use datafusion::common::{plan_datafusion_err, Result}; use datafusion_common::plan_err; +use sail_catalog::credentials::{ + CatalogCredentials, EmptyCatalogCredentials, StaticCatalogCredentials, +}; use sail_catalog::error::CatalogResult; use sail_catalog::manager::{CatalogManager, CatalogManagerOptions}; use sail_catalog::provider::{ @@ -10,12 +13,12 @@ use sail_catalog::provider::{ }; use sail_catalog_glue::{GlueCatalogConfig, GlueCatalogProvider}; use sail_catalog_hms::{HmsCatalogConfig, HmsCatalogProvider}; -use sail_catalog_iceberg::IcebergRestCatalogProvider; +use sail_catalog_iceberg::{IcebergRestCatalogOptions, IcebergRestCatalogProvider}; use sail_catalog_memory::MemoryCatalogProvider; -use sail_catalog_onelake::OneLakeCatalogProvider; +use sail_catalog_onelake::{OneLakeApiKind, OneLakeCatalogProvider}; use sail_catalog_system::{SystemCatalogProvider, SYSTEM_CATALOG_NAME}; -use sail_catalog_unity::UnityCatalogProvider; -use sail_common::config::{AppConfig, CacheType, CatalogCacheConfig, CatalogType}; +use sail_catalog_unity::{UnityCatalogConfig, UnityCatalogOptions, UnityCatalogProvider}; +use sail_common::config::{AppConfig, CacheType, CatalogCacheConfig, CatalogType, OneLakeApi}; use sail_common::runtime::RuntimeHandle; use secrecy::ExposeSecret; @@ -66,23 +69,25 @@ pub fn create_catalog_manager( namespace_separator.to_string(), ); } - if let Some(oauth_access_token) = oauth_access_token { - properties.insert( - "oauth-access-token".to_string(), // Iceberg uses kebab-case - oauth_access_token.expose_secret().to_string(), // FIXME: Only expose when necessary - ); - } - if let Some(bearer_access_token) = bearer_access_token { - properties.insert( - "bearer-access-token".to_string(), // Iceberg uses kebab-case - bearer_access_token.expose_secret().to_string(), // FIXME: Only expose when necessary - ); - } + let credentials = bearer_access_token + .as_ref() + .or(oauth_access_token.as_ref()) + .map(|token| { + Arc::new(StaticCatalogCredentials::new( + token.expose_secret().to_string(), + )) as Arc + }) + .unwrap_or_else(|| Arc::new(EmptyCatalogCredentials)); let runtime_aware = RuntimeAwareCatalogProvider::try_new( || { - let provider = - IcebergRestCatalogProvider::new(name.to_string(), properties); + let provider = IcebergRestCatalogProvider::new( + name.to_string(), + IcebergRestCatalogOptions { + credentials, + properties, + }, + ); Ok(provider) }, runtime.io().clone(), @@ -103,7 +108,27 @@ pub fn create_catalog_manager( cache, } => { let runtime_aware = RuntimeAwareCatalogProvider::try_new( - || UnityCatalogProvider::new(name.to_string(), default_catalog, uri, token), + || { + let config = UnityCatalogConfig::new(uri.clone(), token, None)?; + let credentials = config + .get_credential_provider() + .map(|credentials| { + Arc::new(credentials) as Arc + }) + .unwrap_or_else(|| Arc::new(EmptyCatalogCredentials)); + let default_catalog = default_catalog + .clone() + .unwrap_or_else(|| "unity".to_string()); + UnityCatalogProvider::new( + name.to_string(), + UnityCatalogOptions { + default_catalog, + uri: config.uri, + credentials, + quote_object_name: true, + }, + ) + }, runtime.io().clone(), )?; let provider = wrap_catalog_provider( @@ -117,35 +142,44 @@ pub fn create_catalog_manager( CatalogType::OneLake { name, url, + api, bearer_token, cache, } => { - // Parse URL format: workspace/item.type (e.g., "duckrun/data.lakehouse", "duckrun/data.datawarehouse") + // Parse URL format: + // - `workspace/item.type`: friendly names (e.g. "duckrun/data.lakehouse") + // - `workspaceId/itemId`: GUIDs (e.g. "8f.../3a..."). + // GUID form is required when the workspace has friendly-name support disabled, + // because the data/metadata paths are then GUID-addressed at the storage layer. let (workspace, item) = url.split_once('/').ok_or_else(|| { plan_datafusion_err!( - "Invalid OneLake URL format: expected 'workspace/item.type', got '{}'", + "Invalid OneLake URL format: expected 'workspace/item.type' or 'workspaceId/itemId', got '{}'", url ) })?; - // Extract item name and type (e.g., "data.Lakehouse" -> name="data", type="Lakehouse") - let (item_name, item_type) = item.split_once('.').ok_or_else(|| { - plan_datafusion_err!( - "Invalid OneLake item format: expected 'name.type', got '{}'", - item - ) - })?; + // Friendly form: ("data.Lakehouse" -> name="data", type=Some("Lakehouse")) + // GUID form: ("data" -> name="data", type=None) + let (item_name, item_type) = match item.split_once('.') { + Some((name, item_type)) => (name, Some(item_type.to_string())), + None => (item, None), + }; let token = bearer_token.as_ref().map(|t| t.expose_secret().to_string()); + let api = match api { + OneLakeApi::Delta => OneLakeApiKind::Delta, + OneLakeApi::Iceberg => OneLakeApiKind::Iceberg, + }; let runtime_aware = RuntimeAwareCatalogProvider::try_new( || { - Ok(OneLakeCatalogProvider::new( + OneLakeCatalogProvider::new( name.clone(), workspace.to_string(), item_name.to_string(), - item_type.to_string(), + item_type, + api, token.clone(), - )) + ) }, runtime.io().clone(), )?; diff --git a/docs/development/recipes/iceberg-openapi-generation.md b/docs/development/recipes/iceberg-openapi-generation.md index 9bb65dee9..c469767e5 100644 --- a/docs/development/recipes/iceberg-openapi-generation.md +++ b/docs/development/recipes/iceberg-openapi-generation.md @@ -71,9 +71,28 @@ After running the generation script, you must manually fix the following: - Replace `models::StructField` with `NestedFieldRef` 4. In `src/models/table_update.rs` and `src/models/view_update.rs`: + - The upstream spec declares `TableUpdate`/`ViewUpdate` as a bare `anyOf` with no discriminator (unlike `BaseUpdate`, which has one), so the generator emits a degenerate flat struct with every field of every variant marked required. Replace each generated struct with a hand-written `#[serde(tag = "action")]` enum (mirroring `base_update.rs`), - with one variant per action. + with one variant per action and with all payload fields preserved. + +5. In `src/models/table_requirement.rs`, `src/models/view_requirement.rs`, + `src/models/commit_view_request.rs`, and + `src/models/assert_ref_snapshot_id.rs`: + + - Ensure `TableRequirement` is a hand-written `#[serde(tag = "type")]` enum + whose variants preserve their payload fields. In particular, + `AssertRefSnapshotId` must include `ref` and nullable `snapshot-id`; if it + is generated as an empty variant, Iceberg REST commits will serialize as + `{"type":"assert-ref-snapshot-id"}` and servers will reject the request + because `ref` is missing. + - Ensure `AssertRefSnapshotId.snapshot_id` is `Option` so create + transactions can send `"snapshot-id": null`, which means the ref must not + already exist. + - If the generator omits `ViewRequirement` because it has only one variant, + add it as a hand-written `#[serde(tag = "type")]` enum with + `AssertViewUuid { uuid }`, and keep `CommitViewRequest.requirements` typed + as `Option>`. diff --git a/docs/guide/catalog/onelake.md b/docs/guide/catalog/onelake.md index 8106cc8c2..e91675353 100644 --- a/docs/guide/catalog/onelake.md +++ b/docs/guide/catalog/onelake.md @@ -12,25 +12,85 @@ OneLake catalog can be configured using the following options: - `type` (required): The string `onelake`. - `name` (required): The name of the catalog. - `url` (required): The OneLake item location. +- `api` (optional): The OneLake catalog API. Use `delta` for the Unity Catalog endpoint or `iceberg` for the Iceberg REST catalog endpoint. The default is `iceberg`. - `bearer_token` (optional): The bearer token for authentication. See [Common Options](./index.md#common-options) for caching configuration. The `url` should be in the format . -If `bearer_token` is not provided, Sail will attempt to find credentials from the following sources in order: +If `bearer_token` is not specified, Sail creates credentials from environment variables, similar to how authentication is handled in [Azure storage](../storage/azure.md). +These credentials acquire Microsoft Entra tokens for the Azure Storage audience required by OneLake. -1. The `AZURE_STORAGE_TOKEN` environment variable. -2. The `AZURE_ACCESS_TOKEN` environment variable. -3. Azure CLI (`az account get-access-token`). +The following environment variables are supported: + +- `AZURE_STORAGE_TOKEN`, `AZURE_ACCESS_TOKEN` + + The bearer token for authorizing requests. + +- `AZURE_STORAGE_CLIENT_ID`, `AZURE_CLIENT_ID` + + The service principal client ID for authorizing requests. This can also identify a user-assigned managed identity. + +- `AZURE_STORAGE_CLIENT_SECRET`, `AZURE_CLIENT_SECRET` + + The service principal client secret for authorizing requests. + +- `AZURE_STORAGE_TENANT_ID`, `AZURE_STORAGE_AUTHORITY_ID`, `AZURE_TENANT_ID`, `AZURE_AUTHORITY_ID` + + The tenant ID used in OAuth flows. + +- `AZURE_STORAGE_AUTHORITY_HOST`, `AZURE_AUTHORITY_HOST` + + The authority host used in OAuth flows. + +- `AZURE_MSI_ENDPOINT`, `AZURE_IDENTITY_ENDPOINT` + + The endpoint for acquiring a managed identity token. + +- `AZURE_OBJECT_ID` + + The object ID for use with managed identity authentication. + +- `AZURE_MSI_RESOURCE_ID` + + The MSI resource ID for use with managed identity authentication. + +- `AZURE_FEDERATED_TOKEN_FILE` + + The file containing a token for Azure AD workload identity federation. + +- `AZURE_USE_AZURE_CLI` + + Whether to use Azure CLI for acquiring an access token. + +Tokens are refreshed automatically before expiry. + +::: warning +Azure account keys and SAS tokens are not supported for OneLake catalog authentication. OneLake requires Microsoft Entra bearer tokens. +::: ## Examples ```bash -export SAIL_CATALOG__LIST='[{type="onelake", name="fabric", url="workspace/lakehouse.Lakehouse"}]' +# Unity Catalog endpoint +export SAIL_CATALOG__LIST='[{type="onelake", name="fabric", url="workspace/lakehouse.Lakehouse", api="delta"}]' + +# Iceberg REST catalog endpoint +export SAIL_CATALOG__LIST='[{type="onelake", name="fabric", url="workspace/lakehouse.Lakehouse", api="iceberg"}]' # Bearer token authentication export SAIL_CATALOG__LIST='[{type="onelake", name="fabric", url="workspace/lakehouse.Lakehouse", bearer_token="..."}]' + +# OAuth authentication with a tenant ID +export AZURE_TENANT_ID="..." +export AZURE_CLIENT_ID="..." +export AZURE_CLIENT_SECRET="..." +export SAIL_CATALOG__LIST='[{type="onelake", name="fabric", url="workspace/lakehouse.Lakehouse"}]' + +# Azure CLI authentication via environment variables +export AZURE_USE_AZURE_CLI=true +export SAIL_CATALOG__LIST='[{type="onelake", name="fabric", url="workspace/lakehouse.Lakehouse"}]' ``` ')), '/ROW/msg') AS result - """ - Then query result - | result | - | | + Examples: + | case | value | result | + | Less-than character is escaped | 'a < b' | a < b | + | Greater-than character is escaped | 'a > b' | a > b | + | Ampersand character is escaped | 'a & b' | a & b | + | Multiple special characters together | 'a < b & c > d' | a < b & c > d | + | Script tag content is safely escaped | '' | | Rule: Timestamp and date formatting @@ -309,68 +281,24 @@ Feature: to_xml converts a struct value to an XML string Rule: Decimal and special values - Scenario: Decimal field is formatted as fixed-point string - When query - """ - SELECT xpath_string(to_xml(named_struct('price', CAST(2.99 AS DECIMAL(5,2)))), '/ROW/price') AS result - """ - Then query result - | result | - | 2.99 | - - Scenario: Negative decimal preserves sign - When query - """ - SELECT xpath_string(to_xml(named_struct('price', CAST(-0.83 AS DECIMAL(5,2)))), '/ROW/price') AS result - """ - Then query result - | result | - | -0.83 | - - Scenario: Very large decimal value - When query - """ - SELECT xpath_string(to_xml(named_struct('big', CAST(999999.99 AS DECIMAL(10,2)))), '/ROW/big') AS result - """ - Then query result - | result | - | 999999.99 | - - Scenario: Very small decimal value - When query - """ - SELECT xpath_string(to_xml(named_struct('tiny', CAST(0.0001 AS DECIMAL(5,4)))), '/ROW/tiny') AS result - """ - Then query result - | result | - | 0.0001 | - - Scenario: Floating-point NaN value + Scenario Outline: Decimal and special values: When query """ - SELECT xpath_string(to_xml(named_struct('nan', CAST('NaN' AS DOUBLE))), '/ROW/nan') AS result - """ - Then query result - | result | - | NaN | - - Scenario: Floating-point positive infinity - When query - """ - SELECT xpath_string(to_xml(named_struct('pos', CAST('Infinity' AS DOUBLE))), '/ROW/pos') AS result + SELECT xpath_string(to_xml(named_struct(, )), ) AS result """ Then query result | result | - | Infinity | - - Scenario: Floating-point negative infinity - When query - """ - SELECT xpath_string(to_xml(named_struct('neg', CAST('-Infinity' AS DOUBLE))), '/ROW/neg') AS result - """ - Then query result - | result | - | -Infinity | + | | + + Examples: + | case | field | value | path | result | + | Decimal field is formatted as fixed-point string | 'price' | CAST(2.99 AS DECIMAL(5,2)) | '/ROW/price' | 2.99 | + | Negative decimal preserves sign | 'price' | CAST(-0.83 AS DECIMAL(5,2)) | '/ROW/price' | -0.83 | + | Very large decimal value | 'big' | CAST(999999.99 AS DECIMAL(10,2)) | '/ROW/big' | 999999.99 | + | Very small decimal value | 'tiny' | CAST(0.0001 AS DECIMAL(5,4)) | '/ROW/tiny' | 0.0001 | + | Floating-point NaN value | 'nan' | CAST('NaN' AS DOUBLE) | '/ROW/nan' | NaN | + | Floating-point positive infinity | 'pos' | CAST('Infinity' AS DOUBLE) | '/ROW/pos' | Infinity | + | Floating-point negative infinity | 'neg' | CAST('-Infinity' AS DOUBLE) | '/ROW/neg' | -Infinity | Rule: Complex structures @@ -407,14 +335,22 @@ Feature: to_xml converts a struct value to an XML string Rule: Options - Scenario: Custom rowTag changes root element name + Scenario Outline: Option: When query """ - SELECT to_xml(named_struct('a', 1), map('rowTag', 'Person')) LIKE '%%' AS result + SELECT to_xml() LIKE AS result """ Then query result - | result | - | true | + | result | + | | + + Examples: + | case | args | pattern | result | + | Custom rowTag changes root element name | named_struct('a', 1), map('rowTag', 'Person') | '%%' | true | + | Option keys are case-insensitive | named_struct('a', 1), map('ROWTAG', 'Item') | '%%' | true | + | Suppressed declaration when empty string passed | named_struct('a', 1), map('declaration', '') | '%N/A%' | true | + | Attribute prefix fields become XML attributes | named_struct('_id', 99, 'name', 'Alice') | '%id="99"%' | true | Scenario: valueTag field is written as inline text content on the parent element When query @@ -436,42 +372,6 @@ Feature: to_xml converts a struct value to an XML string | has_content | has_element | | true | false | - Scenario: Option keys are case-insensitive - When query - """ - SELECT to_xml(named_struct('a', 1), map('ROWTAG', 'Item')) LIKE '%%' AS result - """ - Then query result - | result | - | true | - - Scenario: Suppressed declaration when empty string passed - When query - """ - SELECT to_xml(named_struct('a', 1), map('declaration', '')) LIKE '%N/A%' AS result - """ - Then query result - | result | - | true | - - Scenario: Attribute prefix fields become XML attributes - When query - """ - SELECT to_xml(named_struct('_id', 99, 'name', 'Alice')) LIKE '%id="99"%' AS result - """ - Then query result - | result | - | true | - Rule: Map fields Scenario: Map keys become child tag names @@ -564,7 +464,7 @@ Feature: to_xml converts a struct value to an XML string SELECT replace(to_xml(named_struct('a', 1, 'b', 2)), '\n', '~') AS r """ Then query result - | r | + | r | | ~ 1~ 2~ | Scenario: Struct whose only field is NULL is self-closing @@ -573,8 +473,8 @@ Feature: to_xml converts a struct value to an XML string SELECT replace(to_xml(named_struct('a', CAST(NULL AS INT))), '\n', '~') AS r """ Then query result - | r | - | | + | r | + | | Scenario: declaration option is ignored for string output When query @@ -582,55 +482,27 @@ Feature: to_xml converts a struct value to an XML string SELECT replace(to_xml(named_struct('a', 1), map('declaration', 'version="1.0"')), '\n', '~') AS r """ Then query result - | r | + | r | | ~ 1~ | Rule: Primitive arrays are repeated flat tags - Scenario: Primitive array repeats the field tag without an item wrapper + Scenario Outline: Primitive array: When query """ - SELECT replace(to_xml(named_struct('numbers', array(1, 2, 3))), '\n', '~') AS r + SELECT replace(to_xml(named_struct()), '\n', '~') AS r """ Then query result - | r | - | ~ 1~ 2~ 3~ | + | r | + | | - Scenario: Single-element primitive array - When query - """ - SELECT replace(to_xml(named_struct('items', array(1))), '\n', '~') AS r - """ - Then query result - | r | - | ~ 1~ | - - Scenario: NULL elements are dropped from a primitive array - When query - """ - SELECT replace(to_xml(named_struct('items', array(1, CAST(NULL AS INT), 3))), '\n', '~') AS r - """ - Then query result - | r | - | ~ 1~ 3~ | - - Scenario: Nested arrays wrap inner elements in item, outer keeps field tag - When query - """ - SELECT replace(to_xml(named_struct('m', array(array(1, 2), array(3)))), '\n', '~') AS r - """ - Then query result - | r | - | ~ ~ 1~ 2~ ~ ~ 3~ ~ | - - Scenario: Array of structs repeats the field tag around struct fields - When query - """ - SELECT replace(to_xml(named_struct('items', array(named_struct('x', 1), named_struct('x', 2)))), '\n', '~') AS r - """ - Then query result - | r | - | ~ ~ 1~ ~ ~ 2~ ~ | + Examples: + | case | args | result | + | Primitive array repeats the field tag without an item wrapper | 'numbers', array(1, 2, 3) | ~ 1~ 2~ 3~ | + | Single-element primitive array | 'items', array(1) | ~ 1~ | + | NULL elements are dropped from a primitive array | 'items', array(1, CAST(NULL AS INT), 3) | ~ 1~ 3~ | + | Nested arrays wrap inner elements in item, outer keeps field tag | 'm', array(array(1, 2), array(3)) | ~ ~ 1~ 2~ ~ ~ 3~ ~ | + | Array of structs repeats the field tag around struct fields | 'items', array(named_struct('x', 1), named_struct('x', 2)) | ~ ~ 1~ ~ ~ 2~ ~ | Rule: Binary fields @@ -640,7 +512,7 @@ Feature: to_xml converts a struct value to an XML string SELECT replace(to_xml(named_struct('b', X'48656C6C6F')), '\n', '~') AS r """ Then query result - | r | + | r | | ~ [48 65 6C 6C 6F]~ | Scenario: Empty binary renders as empty brackets @@ -649,7 +521,7 @@ Feature: to_xml converts a struct value to an XML string SELECT replace(to_xml(named_struct('b', X'')), '\n', '~') AS r """ Then query result - | r | + | r | | ~ []~ | Rule: XML escaping @@ -660,7 +532,7 @@ Feature: to_xml converts a struct value to an XML string SELECT replace(to_xml(named_struct('msg', 'a < b > c & d')), '\n', '~') AS r """ Then query result - | r | + | r | | ~ a < b > c & d~ | Scenario: Attribute escapes ampersand, less-than and double-quote but not greater-than @@ -669,7 +541,7 @@ Feature: to_xml converts a struct value to an XML string SELECT replace(to_xml(named_struct('_a', 'xz&"q')), '\n', '~') AS r """ Then query result - | r | + | r | | | Rule: valueTag is inline text content @@ -680,8 +552,8 @@ Feature: to_xml converts a struct value to an XML string SELECT replace(to_xml(named_struct('_VALUE', 'hello')), '\n', '~') AS r """ Then query result - | r | - | hello | + | r | + | hello | Scenario: valueTag with an attribute sibling When query @@ -689,8 +561,8 @@ Feature: to_xml converts a struct value to an XML string SELECT replace(to_xml(named_struct('_id', 1, '_VALUE', 'hello')), '\n', '~') AS r """ Then query result - | r | - | hello | + | r | + | hello | Scenario: valueTag text precedes element siblings inline When query @@ -698,8 +570,8 @@ Feature: to_xml converts a struct value to an XML string SELECT replace(to_xml(named_struct('_VALUE', 'hi', 'child', 5)), '\n', '~') AS r """ Then query result - | r | - | hi~ 5~ | + | r | + | hi~ 5~ | Rule: Maps @@ -709,7 +581,7 @@ Feature: to_xml converts a struct value to an XML string SELECT replace(to_xml(named_struct('m', map('k1', 1, 'k2', 2))), '\n', '~') AS r """ Then query result - | r | + | r | | ~ ~ 1~ 2~ ~ | Scenario: Map with array value repeats the key tag @@ -718,7 +590,7 @@ Feature: to_xml converts a struct value to an XML string SELECT replace(to_xml(named_struct('m', map('nums', array(1, 2, 3)))), '\n', '~') AS r """ Then query result - | r | + | r | | ~ ~ 1~ 2~ 3~ ~ | Rule: NULL handling and options @@ -738,7 +610,7 @@ Feature: to_xml converts a struct value to an XML string SELECT replace(to_xml(named_struct('a', 1, 'b', CAST(NULL AS INT))), '\n', '~') AS r """ Then query result - | r | + | r | | ~ 1~ | Scenario: Empty string field is an open-close pair, not self-closing @@ -747,8 +619,8 @@ Feature: to_xml converts a struct value to an XML string SELECT replace(to_xml(named_struct('text', '')), '\n', '~') AS r """ Then query result - | r | - | ~ ~ | + | r | + | ~ ~ | Scenario: nullValue option renders NULL fields with the configured string When query @@ -757,7 +629,7 @@ Feature: to_xml converts a struct value to an XML string """ Then query result | r | - | ~ NA~ | + | ~ NA~ | Rule: Temporal and decimal formatting @@ -767,7 +639,7 @@ Feature: to_xml converts a struct value to an XML string SELECT replace(to_xml(named_struct('d', DATE '2026-06-06')), '\n', '~') AS r """ Then query result - | r | + | r | | ~ 2026-06-06~ | Scenario: Decimal keeps trailing zeros to scale @@ -776,7 +648,7 @@ Feature: to_xml converts a struct value to an XML string SELECT replace(to_xml(named_struct('price', CAST(2.99 AS DECIMAL(5,2)))), '\n', '~') AS r """ Then query result - | r | + | r | | ~ 2.99~ | Scenario: TIMESTAMP_NTZ uses ISO 8601 with millisecond precision and no offset @@ -785,8 +657,8 @@ Feature: to_xml converts a struct value to an XML string SELECT replace(to_xml(named_struct('ts', CAST('2026-06-06 12:00:00' AS TIMESTAMP_NTZ))), '\n', '~') AS r """ Then query result - | r | - | ~ 2026-06-06T12:00:00.000~ | + | r | + | ~ 2026-06-06T12:00:00.000~ | Rule: Special float values @@ -796,7 +668,7 @@ Feature: to_xml converts a struct value to an XML string SELECT replace(to_xml(named_struct('d', CAST('NaN' AS DOUBLE))), '\n', '~') AS r """ Then query result - | r | + | r | | ~ NaN~ | Scenario: Positive infinity renders as Infinity @@ -805,82 +677,73 @@ Feature: to_xml converts a struct value to an XML string SELECT replace(to_xml(named_struct('d', CAST('Infinity' AS DOUBLE))), '\n', '~') AS r """ Then query result - | r | + | r | | ~ Infinity~ | Rule: Double and float use Java toString formatting - Scenario: Whole double keeps a decimal point + Scenario Outline: Java toString formatting: When query """ - SELECT replace(to_xml(named_struct('d', CAST(1.0 AS DOUBLE))), '\n', '~') AS r + SELECT replace(to_xml(named_struct()), '\n', '~') AS r """ Then query result - | r | - | ~ 1.0~ | + | r | + | | + + Examples: + | case | args | result | + | Whole double keeps a decimal point | 'd', CAST(1.0 AS DOUBLE) | ~ 1.0~ | + | Large double uses scientific notation | 'd', CAST(1e20 AS DOUBLE) | ~ 1.0E20~ | + | Small double uses scientific notation | 'd', CAST(1e-7 AS DOUBLE) | ~ 1.0E-7~ | + | Medium whole double keeps a decimal point | 'd', CAST(123456.0 AS DOUBLE) | ~ 123456.0~ | + | Negative zero double normalizes to positive zero | 'd', CAST(-0.0 AS DOUBLE) | ~ 0.0~ | + | Whole float keeps a decimal point | 'f', CAST(1.0 AS FLOAT) | ~ 1.0~ | + | Double array repeats the field tag with formatted values | 'v', array(CAST(1.0 AS DOUBLE), CAST(2.5 AS DOUBLE)) | ~ 1.0~ 2.5~ | - Scenario: Large double uses scientific notation + Rule: Array element naming option + + Scenario: Custom arrayElementName names the inner elements of nested arrays When query """ - SELECT replace(to_xml(named_struct('d', CAST(1e20 AS DOUBLE))), '\n', '~') AS r + SELECT replace(to_xml(named_struct('m', array(array(10, 20))), map('arrayElementName', 'val')), '\n', '~') AS r """ Then query result - | r | - | ~ 1.0E20~ | + | r | + | ~ ~ 10~ 20~ ~ | - Scenario: Small double uses scientific notation + @spark_null + Rule: Output schema + + Scenario: a non-null struct literal is nullable (to_xml is inherently nullable in Spark) When query """ - SELECT replace(to_xml(named_struct('d', CAST(1e-7 AS DOUBLE))), '\n', '~') AS r + SELECT to_xml(struct(1, 'a')) AS result """ - Then query result - | r | - | ~ 1.0E-7~ | - - Scenario: Medium whole double keeps a decimal point - When query + Then query schema """ - SELECT replace(to_xml(named_struct('d', CAST(123456.0 AS DOUBLE))), '\n', '~') AS r + root + |-- result: string (nullable = true) """ - Then query result - | r | - | ~ 123456.0~ | - Scenario: Negative zero double normalizes to positive zero + Scenario: a non-null struct column is nullable (to_xml is inherently nullable in Spark) When query """ - SELECT replace(to_xml(named_struct('d', CAST(-0.0 AS DOUBLE))), '\n', '~') AS r + SELECT to_xml(struct(id, id)) AS result FROM range(3) """ - Then query result - | r | - | ~ 0.0~ | - - Scenario: Whole float keeps a decimal point - When query + Then query schema """ - SELECT replace(to_xml(named_struct('f', CAST(1.0 AS FLOAT))), '\n', '~') AS r + root + |-- result: string (nullable = true) """ - Then query result - | r | - | ~ 1.0~ | - Scenario: Double array repeats the field tag with formatted values + Scenario: a nullable struct column stays nullable When query """ - SELECT replace(to_xml(named_struct('v', array(CAST(1.0 AS DOUBLE), CAST(2.5 AS DOUBLE)))), '\n', '~') AS r + SELECT to_xml(c) AS result FROM VALUES (named_struct('a', 1)), (CAST(NULL AS STRUCT)) AS t(c) """ - Then query result - | r | - | ~ 1.0~ 2.5~ | - - Rule: Array element naming option - - Scenario: Custom arrayElementName names the inner elements of nested arrays - When query + Then query schema """ - SELECT replace(to_xml(named_struct('m', array(array(10, 20))), map('arrayElementName', 'val')), '\n', '~') AS r + root + |-- result: string (nullable = true) """ - Then query result - | r | - | ~ ~ 10~ 20~ ~ | - diff --git a/python/pysail/tests/spark/function/features/xpath.feature b/python/pysail/tests/spark/function/features/xml/xpath.feature similarity index 74% rename from python/pysail/tests/spark/function/features/xpath.feature rename to python/pysail/tests/spark/function/features/xml/xpath.feature index 3c2889e98..b1035ecd2 100644 --- a/python/pysail/tests/spark/function/features/xpath.feature +++ b/python/pysail/tests/spark/function/features/xml/xpath.feature @@ -1,3 +1,4 @@ +@xpath Feature: xpath() extracts XML nodes with Spark-compatible semantics Rule: Node selection returns arrays of string-like values @@ -8,8 +9,8 @@ Feature: xpath() extracts XML nodes with Spark-compatible semantics SELECT xpath('b1b2b3', 'a/b/text()') AS result """ Then query result - | result | - | [b1, b2, b3] | + | result | + | [b1, b2, b3] | Scenario: xpath returns NULL entries for element nodes When query @@ -17,8 +18,8 @@ Feature: xpath() extracts XML nodes with Spark-compatible semantics SELECT xpath('b1b2', 'a/b') AS result """ Then query result - | result | - | [NULL, NULL] | + | result | + | [NULL, NULL] | Scenario: xpath returns an empty list when no nodes match When query @@ -107,3 +108,42 @@ Feature: xpath() extracts XML nodes with Spark-compatible semantics SELECT xpath('b1b2b3c1c2', c) AS result FROM VALUES (1, 'a/b'), (2, 'a/b') AS t(i, c) ORDER BY i """ Then query error NON_FOLDABLE_INPUT + + @spark_null + Rule: Output schema + + Scenario: a non-null xml literal yields an array + When query + """ + SELECT xpath('12', 'a/b/text()') AS result + """ + Then query schema + """ + root + |-- result: array (nullable = true) + | |-- element: string (containsNull = true) + """ + + Scenario: a non-null xml column yields an array + When query + """ + SELECT xpath(CONCAT('', CAST(id AS STRING), ''), 'a/b/text()') AS result FROM range(3) + """ + Then query schema + """ + root + |-- result: array (nullable = true) + | |-- element: string (containsNull = true) + """ + + Scenario: a nullable xml column stays nullable + When query + """ + SELECT xpath(c, 'a/b/text()') AS result FROM VALUES ('1'), (CAST(NULL AS STRING)) AS t(c) + """ + Then query schema + """ + root + |-- result: array (nullable = true) + | |-- element: string (containsNull = true) + """ diff --git a/python/pysail/tests/spark/function/features/xpath_boolean.feature b/python/pysail/tests/spark/function/features/xml/xpath_boolean.feature similarity index 59% rename from python/pysail/tests/spark/function/features/xpath_boolean.feature rename to python/pysail/tests/spark/function/features/xml/xpath_boolean.feature index 7e883527d..36a0a2eb5 100644 --- a/python/pysail/tests/spark/function/features/xpath_boolean.feature +++ b/python/pysail/tests/spark/function/features/xml/xpath_boolean.feature @@ -1,3 +1,4 @@ +@xpath_boolean Feature: xpath_boolean with an argument coming from a column # A behaviour-governing argument given as a literal is constant-folded, so the literal # scenarios never exercise the columnar kernel. These scenarios pass the same argument @@ -32,3 +33,39 @@ Feature: xpath_boolean with an argument coming from a column SELECT xpath_boolean('1', c) AS result FROM VALUES (1, 'a/b'), (2, 'a/b') AS t(i, c) ORDER BY i """ Then query error NON_FOLDABLE_INPUT + + @spark_null + Rule: Output schema + + Scenario: a non-null xml literal yields a boolean + When query + """ + SELECT xpath_boolean('1', 'a/b=1') AS result + """ + Then query schema + """ + root + |-- result: boolean (nullable = true) + """ + + Scenario: a non-null xml column yields a boolean + When query + """ + SELECT xpath_boolean(CONCAT('', CAST(id AS STRING), ''), 'a/b=1') AS result FROM range(3) + """ + Then query schema + """ + root + |-- result: boolean (nullable = true) + """ + + Scenario: a nullable xml column stays nullable + When query + """ + SELECT xpath_boolean(c, 'a/b=1') AS result FROM VALUES ('1'), (CAST(NULL AS STRING)) AS t(c) + """ + Then query schema + """ + root + |-- result: boolean (nullable = true) + """ diff --git a/python/pysail/tests/spark/function/features/xpath_double.feature b/python/pysail/tests/spark/function/features/xml/xpath_double.feature similarity index 99% rename from python/pysail/tests/spark/function/features/xpath_double.feature rename to python/pysail/tests/spark/function/features/xml/xpath_double.feature index acc6173e8..fe638aa69 100644 --- a/python/pysail/tests/spark/function/features/xpath_double.feature +++ b/python/pysail/tests/spark/function/features/xml/xpath_double.feature @@ -1,3 +1,4 @@ +@xpath_double Feature: xpath_double with an argument coming from a column # A behaviour-governing argument given as a literal is constant-folded, so the literal # scenarios never exercise the columnar kernel. These scenarios pass the same argument diff --git a/python/pysail/tests/spark/function/features/xpath_float.feature b/python/pysail/tests/spark/function/features/xml/xpath_float.feature similarity index 58% rename from python/pysail/tests/spark/function/features/xpath_float.feature rename to python/pysail/tests/spark/function/features/xml/xpath_float.feature index b13da4e68..bf6521e2a 100644 --- a/python/pysail/tests/spark/function/features/xpath_float.feature +++ b/python/pysail/tests/spark/function/features/xml/xpath_float.feature @@ -1,3 +1,4 @@ +@xpath_float Feature: xpath_float with an argument coming from a column # A behaviour-governing argument given as a literal is constant-folded, so the literal # scenarios never exercise the columnar kernel. These scenarios pass the same argument @@ -32,3 +33,39 @@ Feature: xpath_float with an argument coming from a column SELECT xpath_float('12', c) AS result FROM VALUES (1, 'sum(a/b)'), (2, 'sum(a/b)') AS t(i, c) ORDER BY i """ Then query error NON_FOLDABLE_INPUT + + @spark_null + Rule: Output schema + + Scenario: a non-null literal input to xpath_float yields the schema Spark declares + When query + """ + SELECT xpath_float('12', 'sum(a/b)') AS result + """ + Then query schema + """ + root + |-- result: float (nullable = true) + """ + + Scenario: a non-null column input to xpath_float yields the schema Spark declares + When query + """ + SELECT xpath_float(CAST(id AS STRING), 'sum(a/b)') AS result FROM range(3) + """ + Then query schema + """ + root + |-- result: float (nullable = true) + """ + + Scenario: a nullable column input to xpath_float stays nullable + When query + """ + SELECT xpath_float(c, 'sum(a/b)') AS result FROM VALUES ('12'), (CAST(NULL AS STRING)) AS t(c) + """ + Then query schema + """ + root + |-- result: float (nullable = true) + """ diff --git a/python/pysail/tests/spark/function/features/xpath_int.feature b/python/pysail/tests/spark/function/features/xml/xpath_int.feature similarity index 60% rename from python/pysail/tests/spark/function/features/xpath_int.feature rename to python/pysail/tests/spark/function/features/xml/xpath_int.feature index 65b9189a7..7a0cff754 100644 --- a/python/pysail/tests/spark/function/features/xpath_int.feature +++ b/python/pysail/tests/spark/function/features/xml/xpath_int.feature @@ -1,3 +1,4 @@ +@xpath_int Feature: xpath_int with an argument coming from a column # A behaviour-governing argument given as a literal is constant-folded, so the literal # scenarios never exercise the columnar kernel. These scenarios pass the same argument @@ -32,3 +33,39 @@ Feature: xpath_int with an argument coming from a column SELECT xpath_int('12', c) AS result FROM VALUES (1, 'sum(a/b)'), (2, 'sum(a/b)') AS t(i, c) ORDER BY i """ Then query error NON_FOLDABLE_INPUT + + @spark_null + Rule: Output schema + + Scenario: a non-null xml literal yields an integer + When query + """ + SELECT xpath_int('5', 'a/b') AS result + """ + Then query schema + """ + root + |-- result: integer (nullable = true) + """ + + Scenario: a non-null xml column yields an integer + When query + """ + SELECT xpath_int(CONCAT('', CAST(id AS STRING), ''), 'a/b') AS result FROM range(3) + """ + Then query schema + """ + root + |-- result: integer (nullable = true) + """ + + Scenario: a nullable xml column stays nullable + When query + """ + SELECT xpath_int(c, 'a/b') AS result FROM VALUES ('5'), (CAST(NULL AS STRING)) AS t(c) + """ + Then query schema + """ + root + |-- result: integer (nullable = true) + """ diff --git a/python/pysail/tests/spark/function/features/xpath_long.feature b/python/pysail/tests/spark/function/features/xml/xpath_long.feature similarity index 58% rename from python/pysail/tests/spark/function/features/xpath_long.feature rename to python/pysail/tests/spark/function/features/xml/xpath_long.feature index 0d87ddd10..7f6fda35a 100644 --- a/python/pysail/tests/spark/function/features/xpath_long.feature +++ b/python/pysail/tests/spark/function/features/xml/xpath_long.feature @@ -1,3 +1,4 @@ +@xpath_long Feature: xpath_long with an argument coming from a column # A behaviour-governing argument given as a literal is constant-folded, so the literal # scenarios never exercise the columnar kernel. These scenarios pass the same argument @@ -32,3 +33,39 @@ Feature: xpath_long with an argument coming from a column SELECT xpath_long('12', c) AS result FROM VALUES (1, 'sum(a/b)'), (2, 'sum(a/b)') AS t(i, c) ORDER BY i """ Then query error NON_FOLDABLE_INPUT + + @spark_null + Rule: Output schema + + Scenario: a non-null literal input to xpath_long yields the schema Spark declares + When query + """ + SELECT xpath_long('12', 'sum(a/b)') AS result + """ + Then query schema + """ + root + |-- result: long (nullable = true) + """ + + Scenario: a non-null column input to xpath_long yields the schema Spark declares + When query + """ + SELECT xpath_long(CAST(id AS STRING), 'sum(a/b)') AS result FROM range(3) + """ + Then query schema + """ + root + |-- result: long (nullable = true) + """ + + Scenario: a nullable column input to xpath_long stays nullable + When query + """ + SELECT xpath_long(c, 'sum(a/b)') AS result FROM VALUES ('12'), (CAST(NULL AS STRING)) AS t(c) + """ + Then query schema + """ + root + |-- result: long (nullable = true) + """ diff --git a/python/pysail/tests/spark/function/features/xpath_number.feature b/python/pysail/tests/spark/function/features/xml/xpath_number.feature similarity index 99% rename from python/pysail/tests/spark/function/features/xpath_number.feature rename to python/pysail/tests/spark/function/features/xml/xpath_number.feature index 36edb515d..5932ed99f 100644 --- a/python/pysail/tests/spark/function/features/xpath_number.feature +++ b/python/pysail/tests/spark/function/features/xml/xpath_number.feature @@ -1,3 +1,4 @@ +@xpath_number Feature: xpath_number with an argument coming from a column # A behaviour-governing argument given as a literal is constant-folded, so the literal # scenarios never exercise the columnar kernel. These scenarios pass the same argument diff --git a/python/pysail/tests/spark/function/features/xpath_short.feature b/python/pysail/tests/spark/function/features/xml/xpath_short.feature similarity index 58% rename from python/pysail/tests/spark/function/features/xpath_short.feature rename to python/pysail/tests/spark/function/features/xml/xpath_short.feature index 7c9a22429..5212d08d4 100644 --- a/python/pysail/tests/spark/function/features/xpath_short.feature +++ b/python/pysail/tests/spark/function/features/xml/xpath_short.feature @@ -1,3 +1,4 @@ +@xpath_short Feature: xpath_short with an argument coming from a column # A behaviour-governing argument given as a literal is constant-folded, so the literal # scenarios never exercise the columnar kernel. These scenarios pass the same argument @@ -32,3 +33,39 @@ Feature: xpath_short with an argument coming from a column SELECT xpath_short('12', c) AS result FROM VALUES (1, 'sum(a/b)'), (2, 'sum(a/b)') AS t(i, c) ORDER BY i """ Then query error NON_FOLDABLE_INPUT + + @spark_null + Rule: Output schema + + Scenario: a non-null literal input to xpath_short yields the schema Spark declares + When query + """ + SELECT xpath_short('12', 'sum(a/b)') AS result + """ + Then query schema + """ + root + |-- result: short (nullable = true) + """ + + Scenario: a non-null column input to xpath_short yields the schema Spark declares + When query + """ + SELECT xpath_short(CAST(id AS STRING), 'sum(a/b)') AS result FROM range(3) + """ + Then query schema + """ + root + |-- result: short (nullable = true) + """ + + Scenario: a nullable column input to xpath_short stays nullable + When query + """ + SELECT xpath_short(c, 'sum(a/b)') AS result FROM VALUES ('12'), (CAST(NULL AS STRING)) AS t(c) + """ + Then query schema + """ + root + |-- result: short (nullable = true) + """ diff --git a/python/pysail/tests/spark/function/features/xpath_string.feature b/python/pysail/tests/spark/function/features/xml/xpath_string.feature similarity index 60% rename from python/pysail/tests/spark/function/features/xpath_string.feature rename to python/pysail/tests/spark/function/features/xml/xpath_string.feature index 965144d10..43cbb8c5b 100644 --- a/python/pysail/tests/spark/function/features/xpath_string.feature +++ b/python/pysail/tests/spark/function/features/xml/xpath_string.feature @@ -1,3 +1,4 @@ +@xpath_string Feature: xpath_string with an argument coming from a column # A behaviour-governing argument given as a literal is constant-folded, so the literal # scenarios never exercise the columnar kernel. These scenarios pass the same argument @@ -32,3 +33,39 @@ Feature: xpath_string with an argument coming from a column SELECT xpath_string('bcc', c) AS result FROM VALUES (1, 'a/c'), (2, 'a/c') AS t(i, c) ORDER BY i """ Then query error NON_FOLDABLE_INPUT + + @spark_null + Rule: Output schema + + Scenario: a non-null xml literal yields a string + When query + """ + SELECT xpath_string('x', 'a/b') AS result + """ + Then query schema + """ + root + |-- result: string (nullable = true) + """ + + Scenario: a non-null xml column yields a string + When query + """ + SELECT xpath_string(CONCAT('', CAST(id AS STRING), ''), 'a/b') AS result FROM range(3) + """ + Then query schema + """ + root + |-- result: string (nullable = true) + """ + + Scenario: a nullable xml column stays nullable + When query + """ + SELECT xpath_string(c, 'a/b') AS result FROM VALUES ('x'), (CAST(NULL AS STRING)) AS t(c) + """ + Then query schema + """ + root + |-- result: string (nullable = true) + """ diff --git a/python/pysail/tests/spark/function/features/xpath_typed.feature b/python/pysail/tests/spark/function/features/xml/xpath_typed.feature similarity index 52% rename from python/pysail/tests/spark/function/features/xpath_typed.feature rename to python/pysail/tests/spark/function/features/xml/xpath_typed.feature index 3c0a44e74..dce28da2c 100644 --- a/python/pysail/tests/spark/function/features/xpath_typed.feature +++ b/python/pysail/tests/spark/function/features/xml/xpath_typed.feature @@ -1,133 +1,102 @@ +@xpath_typed Feature: xpath_boolean/double/float/int/long/number/short/string extract typed values from XML Rule: xpath_boolean evaluates XPath to a boolean - Scenario: xpath_boolean returns true when node exists + Scenario Outline: Boolean: When query """ - SELECT xpath_boolean('1', 'a/b') AS result + SELECT xpath_boolean('1', '') AS result """ Then query result - | result | - | true | + | result | + | | - Scenario: xpath_boolean returns false when node does not exist - When query - """ - SELECT xpath_boolean('1', 'a/c') AS result - """ - Then query result - | result | - | false | + Examples: + | case | path | result | + | xpath_boolean returns true when node exists | a/b | true | + | xpath_boolean returns false when node does not exist | a/c | false | Rule: xpath_double and xpath_number evaluate XPath to a double - Scenario: xpath_double returns a sum as double + Scenario Outline: Double: When query """ - SELECT xpath_double('12', 'sum(a/b)') AS result + SELECT ('', '') AS result """ Then query result - | result | - | 3.0 | + | result | + | | - Scenario: xpath_number returns a sum as double - When query - """ - SELECT xpath_number('12', 'sum(a/b)') AS result - """ - Then query result - | result | - | 3.0 | - - Scenario: xpath_double returns NaN for non-numeric value - When query - """ - SELECT xpath_double('text', 'a/b') AS result - """ - Then query result - | result | - | NaN | + Examples: + | case | fn | xml | path | result | + | xpath_double returns a sum as double | xpath_double | 12 | sum(a/b) | 3.0 | + | xpath_number returns a sum as double | xpath_number | 12 | sum(a/b) | 3.0 | + | xpath_double returns NaN for non-numeric value | xpath_double | text | a/b | NaN | Rule: xpath_float evaluates XPath to a float - Scenario: xpath_float returns a sum as float + Scenario Outline: Float: When query """ - SELECT xpath_float('12', 'sum(a/b)') AS result + SELECT xpath_float('', '') AS result """ Then query result - | result | - | 3.0 | + | result | + | | - Scenario: xpath_float returns NaN for non-numeric value - When query - """ - SELECT xpath_float('text', 'a/b') AS result - """ - Then query result - | result | - | NaN | + Examples: + | case | xml | path | result | + | xpath_float returns a sum as float | 12 | sum(a/b) | 3.0 | + | xpath_float returns NaN for non-numeric value | text | a/b | NaN | Rule: xpath_int evaluates XPath to an integer - Scenario: xpath_int returns a sum as integer + Scenario Outline: Int: When query """ - SELECT xpath_int('12', 'sum(a/b)') AS result + SELECT xpath_int('', '') AS result """ Then query result - | result | - | 3 | + | result | + | | - Scenario: xpath_int returns zero for non-numeric value - When query - """ - SELECT xpath_int('text', 'a/b') AS result - """ - Then query result - | result | - | 0 | + Examples: + | case | xml | path | result | + | xpath_int returns a sum as integer | 12 | sum(a/b) | 3 | + | xpath_int returns zero for non-numeric value | text | a/b | 0 | Rule: xpath_long evaluates XPath to a long integer - Scenario: xpath_long returns a sum as long + Scenario Outline: Long: When query """ - SELECT xpath_long('12', 'sum(a/b)') AS result + SELECT xpath_long('', '') AS result """ Then query result - | result | - | 3 | + | result | + | | - Scenario: xpath_long returns zero for non-numeric value - When query - """ - SELECT xpath_long('text', 'a/b') AS result - """ - Then query result - | result | - | 0 | + Examples: + | case | xml | path | result | + | xpath_long returns a sum as long | 12 | sum(a/b) | 3 | + | xpath_long returns zero for non-numeric value | text | a/b | 0 | Rule: xpath_short evaluates XPath to a short integer - Scenario: xpath_short returns a sum as short + Scenario Outline: Short: When query """ - SELECT xpath_short('12', 'sum(a/b)') AS result + SELECT xpath_short('', '') AS result """ Then query result - | result | - | 3 | + | result | + | | - Scenario: xpath_short returns zero for non-numeric value - When query - """ - SELECT xpath_short('text', 'a/b') AS result - """ - Then query result - | result | - | 0 | + Examples: + | case | xml | path | result | + | xpath_short returns a sum as short | 12 | sum(a/b) | 3 | + | xpath_short returns zero for non-numeric value | text | a/b | 0 | Rule: xpath_string evaluates XPath to a string diff --git a/python/pysail/tests/spark/function/features/xxhash64.feature b/python/pysail/tests/spark/function/features/xxhash64.feature deleted file mode 100644 index f3e8dc07a..000000000 --- a/python/pysail/tests/spark/function/features/xxhash64.feature +++ /dev/null @@ -1,141 +0,0 @@ -Feature: xxhash64() returns 64-bit xxHash - - Rule: Basic usage - - Scenario: xxhash64 integer - When query - """ - SELECT xxhash64(42) AS result - """ - Then query result - | result | - | -387659249110444264 | - - Scenario: xxhash64 string - When query - """ - SELECT xxhash64('hello') AS result - """ - Then query result - | result | - | -4367754540140381902 | - - Scenario: xxhash64 multiple args - When query - """ - SELECT xxhash64(1, 'a', 2) AS result - """ - Then query result - | result | - | 4450643625805672383 | - - Rule: Null handling - - Scenario: xxhash64 null input - When query - """ - SELECT xxhash64(CAST(NULL AS INT)) AS result - """ - Then query result - | result | - | 42 | - - Scenario: xxhash64 null string input also returns the seed - When query - """ - SELECT xxhash64(CAST(NULL AS STRING)) AS result - """ - Then query result - | result | - | 42 | - - Rule: Type coverage - - # The hash must agree with Spark for every input type, since the encoding fed - # to xxHash is type-specific. All values verified against the Spark JVM. - - Scenario: xxhash64 bigint - When query - """ - SELECT xxhash64(CAST(1 AS BIGINT)) AS result - """ - Then query result - | result | - | -7001672635703045582 | - - Scenario: xxhash64 int and boolean true hash identically - When query - """ - SELECT xxhash64(CAST(1 AS INT)) AS i, xxhash64(true) AS b - """ - Then query result - | i | b | - | -6698625589789238999 | -6698625589789238999 | - - Scenario: xxhash64 double - When query - """ - SELECT xxhash64(CAST(1.5 AS DOUBLE)) AS result - """ - Then query result - | result | - | 7738255526519901366 | - - Scenario: xxhash64 float - When query - """ - SELECT xxhash64(CAST(1.5 AS FLOAT)) AS result - """ - Then query result - | result | - | 6163473420726370430 | - - Scenario: xxhash64 decimal - When query - """ - SELECT xxhash64(CAST(1.50 AS DECIMAL(10,2))) AS result - """ - Then query result - | result | - | -6873856301616164681 | - - Scenario: xxhash64 date - When query - """ - SELECT xxhash64(DATE '2024-01-15') AS result - """ - Then query result - | result | - | 2166432641145730595 | - - # NOTE: TIMESTAMP (LTZ) is intentionally NOT asserted with a golden value — - # its hash depends on the session timezone (micros-since-epoch), so the value - # is not portable across environments. The migration still hashes it; only the - # golden value would be flaky. - - Scenario: xxhash64 binary - When query - """ - SELECT xxhash64(X'48656C6C6F') AS result - """ - Then query result - | result | - | 6777584228807376986 | - - Scenario: xxhash64 array - When query - """ - SELECT xxhash64(array(1, 2, 3)) AS result - """ - Then query result - | result | - | 8592097078962733837 | - - Scenario: xxhash64 struct - When query - """ - SELECT xxhash64(named_struct('a', 1, 'b', 'x')) AS result - """ - Then query result - | result | - | 8510603489595372987 | diff --git a/python/pysail/tests/spark/function/test_arrays_zip.txt b/python/pysail/tests/spark/function/test_arrays_zip.txt deleted file mode 100644 index 6239703b4..000000000 --- a/python/pysail/tests/spark/function/test_arrays_zip.txt +++ /dev/null @@ -1,55 +0,0 @@ ->>> from pyspark.sql.functions import arrays_zip ->>> df = spark.createDataFrame([([1, 2, 3], [2, 4, 6], [3, 6])], ['vals1', 'vals2', 'vals3']) ->>> df = df.select(arrays_zip(df.vals1, df.vals2, df.vals3).alias('zipped')) ->>> df.show(truncate=False) -+------------------------------------+ -|zipped | -+------------------------------------+ -|[{1, 2, 3}, {2, 4, 6}, {3, 6, NULL}]| -+------------------------------------+ - ->>> df.printSchema() -root - |-- zipped: array (nullable = true) - | |-- element: struct (containsNull = false) - | | |-- vals1: long (nullable = true) - | | |-- vals2: long (nullable = true) - | | |-- vals3: long (nullable = true) - - ->>> from pyspark.sql import functions as sf ->>> df = spark.createDataFrame([([1, 2, 3], ['a', 'b', 'c'])], ['nums', 'letters']) ->>> df.select(sf.arrays_zip(df.nums, df.letters)).show(truncate=False) -+-------------------------+ -|arrays_zip(nums, letters)| -+-------------------------+ -|[{1, a}, {2, b}, {3, c}] | -+-------------------------+ - ->>> from pyspark.sql import functions as sf ->>> df = spark.createDataFrame([([1, 2], ['a', 'b', 'c'])], ['nums', 'letters']) ->>> df.select(sf.arrays_zip(df.nums, df.letters)).show(truncate=False) -+---------------------------+ -|arrays_zip(nums, letters) | -+---------------------------+ -|[{1, a}, {2, b}, {NULL, c}]| -+---------------------------+ - ->>> from pyspark.sql import functions as sf ->>> df = spark.createDataFrame( -... [([1, 2], ['a', 'b'], [True, False])], ['nums', 'letters', 'bools']) ->>> df.select(sf.arrays_zip(df.nums, df.letters, df.bools)).show(truncate=False) -+--------------------------------+ -|arrays_zip(nums, letters, bools)| -+--------------------------------+ -|[{1, a, true}, {2, b, false}] | -+--------------------------------+ - ->>> from pyspark.sql import functions as sf ->>> df = spark.createDataFrame([([1, 2, None], ['a', None, 'c'])], ['nums', 'letters']) ->>> df.select(sf.arrays_zip(df.nums, df.letters)).show(truncate=False) -+------------------------------+ -|arrays_zip(nums, letters) | -+------------------------------+ -|[{1, a}, {2, NULL}, {NULL, c}]| -+------------------------------+ diff --git a/python/pysail/tests/spark/function/test_ceil.txt b/python/pysail/tests/spark/function/test_ceil.txt deleted file mode 100644 index ec85f2fb1..000000000 --- a/python/pysail/tests/spark/function/test_ceil.txt +++ /dev/null @@ -1,167 +0,0 @@ ->>> spark.sql("SELECT ceil(-0.1), typeof(ceil(-0.1)), typeof(-0.1)").show() -+----------+------------------+------------+ -|CEIL(-0.1)|typeof(CEIL(-0.1))|typeof(-0.1)| -+----------+------------------+------------+ -| 0| decimal(1,0)|decimal(1,1)| -+----------+------------------+------------+ - ->>> spark.sql("SELECT ceil(5), typeof(ceil(5)), typeof(5)").show() -+-------+---------------+---------+ -|CEIL(5)|typeof(CEIL(5))|typeof(5)| -+-------+---------------+---------+ -| 5| bigint| int| -+-------+---------------+---------+ - ->>> spark.sql("SELECT ceil(5.4), typeof(ceil(5.4)), typeof(5.4)").show() -+---------+-----------------+------------+ -|CEIL(5.4)|typeof(CEIL(5.4))| typeof(5.4)| -+---------+-----------------+------------+ -| 6| decimal(2,0)|decimal(2,1)| -+---------+-----------------+------------+ - ->>> spark.sql("SELECT ceil(3.1411, -3), typeof(ceil(3.1411, -3)), typeof(3.1411)").show() -+----------------+------------------------+--------------+ -|ceil(3.1411, -3)|typeof(ceil(3.1411, -3))|typeof(3.1411)| -+----------------+------------------------+--------------+ -| 1000| decimal(4,0)| decimal(5,4)| -+----------------+------------------------+--------------+ - ->>> spark.sql("SELECT ceil(3.1411, 3), typeof(ceil(3.1411, 3)), typeof(3.1411)").show() -+---------------+-----------------------+--------------+ -|ceil(3.1411, 3)|typeof(ceil(3.1411, 3))|typeof(3.1411)| -+---------------+-----------------------+--------------+ -| 3.142| decimal(5,3)| decimal(5,4)| -+---------------+-----------------------+--------------+ - ->>> spark.sql("SELECT ceil(3345.1, -2), typeof(ceil(3345.1, -2)), typeof(3345.1)").show() -+----------------+------------------------+--------------+ -|ceil(3345.1, -2)|typeof(ceil(3345.1, -2))|typeof(3345.1)| -+----------------+------------------------+--------------+ -| 3400| decimal(5,0)| decimal(5,1)| -+----------------+------------------------+--------------+ - ->>> spark.sql("SELECT ceil(-12.345, 1), typeof(ceil(-12.345, 1)), typeof(3345.1)").show() -+----------------+------------------------+--------------+ -|ceil(-12.345, 1)|typeof(ceil(-12.345, 1))|typeof(3345.1)| -+----------------+------------------------+--------------+ -| -12.3| decimal(4,1)| decimal(5,1)| -+----------------+------------------------+--------------+ - ->>> spark.sql("SELECT ceil(CAST(5 as TINYINT), 1), typeof(ceil(CAST(5 as TINYINT), 1))").show() -+---------------------------+-----------------------------------+ -|ceil(CAST(5 AS TINYINT), 1)|typeof(ceil(CAST(5 AS TINYINT), 1))| -+---------------------------+-----------------------------------+ -| 5| decimal(4,0)| -+---------------------------+-----------------------------------+ - ->>> spark.sql("SELECT ceil(CAST(5 as TINYINT), -4), typeof(ceil(CAST(5 as TINYINT), -4))").show() -+----------------------------+------------------------------------+ -|ceil(CAST(5 AS TINYINT), -4)|typeof(ceil(CAST(5 AS TINYINT), -4))| -+----------------------------+------------------------------------+ -| 10000| decimal(5,0)| -+----------------------------+------------------------------------+ - ->>> spark.sql("SELECT ceil(CAST(5 as SMALLINT), 1), typeof(ceil(CAST(5 as SMALLINT), 1))").show() -+----------------------------+------------------------------------+ -|ceil(CAST(5 AS SMALLINT), 1)|typeof(ceil(CAST(5 AS SMALLINT), 1))| -+----------------------------+------------------------------------+ -| 5| decimal(6,0)| -+----------------------------+------------------------------------+ - ->>> spark.sql("SELECT ceil(CAST(5 as SMALLINT), -6), typeof(ceil(CAST(5 as SMALLINT), -6))").show() -+-----------------------------+-------------------------------------+ -|ceil(CAST(5 AS SMALLINT), -6)|typeof(ceil(CAST(5 AS SMALLINT), -6))| -+-----------------------------+-------------------------------------+ -| 1000000| decimal(7,0)| -+-----------------------------+-------------------------------------+ - ->>> spark.sql("SELECT ceil(CAST(5 as INT), 1), typeof(ceil(CAST(5 as INT), 1))").show() -+-----------------------+-------------------------------+ -|ceil(CAST(5 AS INT), 1)|typeof(ceil(CAST(5 AS INT), 1))| -+-----------------------+-------------------------------+ -| 5| decimal(11,0)| -+-----------------------+-------------------------------+ - ->>> spark.sql("SELECT ceil(CAST(5 as INT), -11), typeof(ceil(CAST(5 as INT), -11))").show() -+-------------------------+---------------------------------+ -|ceil(CAST(5 AS INT), -11)|typeof(ceil(CAST(5 AS INT), -11))| -+-------------------------+---------------------------------+ -| 100000000000| decimal(12,0)| -+-------------------------+---------------------------------+ - ->>> spark.sql("SELECT ceil(CAST(5 as BIGINT), 1), typeof(ceil(CAST(5 as BIGINT), 1))").show() -+--------------------------+----------------------------------+ -|ceil(CAST(5 AS BIGINT), 1)|typeof(ceil(CAST(5 AS BIGINT), 1))| -+--------------------------+----------------------------------+ -| 5| decimal(21,0)| -+--------------------------+----------------------------------+ - ->>> spark.sql("SELECT ceil(CAST(5 as BIGINT), -21), typeof(ceil(CAST(5 as BIGINT), -21))").show() -+----------------------------+------------------------------------+ -|ceil(CAST(5 AS BIGINT), -21)|typeof(ceil(CAST(5 AS BIGINT), -21))| -+----------------------------+------------------------------------+ -| 10000000000000000...| decimal(22,0)| -+----------------------------+------------------------------------+ - ->>> spark.sql("SELECT ceil(CAST(5 as FLOAT), 1), typeof(ceil(CAST(5 as FLOAT), 1))").show() -+-------------------------+---------------------------------+ -|ceil(CAST(5 AS FLOAT), 1)|typeof(ceil(CAST(5 AS FLOAT), 1))| -+-------------------------+---------------------------------+ -| 5.0| decimal(9,1)| -+-------------------------+---------------------------------+ - ->>> spark.sql("SELECT ceil(CAST(5 as FLOAT), -15), typeof(ceil(CAST(5 as FLOAT), -15))").show() -+---------------------------+-----------------------------------+ -|ceil(CAST(5 AS FLOAT), -15)|typeof(ceil(CAST(5 AS FLOAT), -15))| -+---------------------------+-----------------------------------+ -| 1000000000000000| decimal(16,0)| -+---------------------------+-----------------------------------+ - ->>> spark.sql("SELECT ceil(CAST(5 as DOUBLE), 1), typeof(ceil(CAST(5 as DOUBLE), 1))").show() -+--------------------------+----------------------------------+ -|ceil(CAST(5 AS DOUBLE), 1)|typeof(ceil(CAST(5 AS DOUBLE), 1))| -+--------------------------+----------------------------------+ -| 5.0| decimal(17,1)| -+--------------------------+----------------------------------+ - ->>> spark.sql("SELECT ceil(CAST(5 as DOUBLE), -31), typeof(ceil(CAST(5 as DOUBLE), -31))").show() -+----------------------------+------------------------------------+ -|ceil(CAST(5 AS DOUBLE), -31)|typeof(ceil(CAST(5 AS DOUBLE), -31))| -+----------------------------+------------------------------------+ -| 10000000000000000...| decimal(32,0)| -+----------------------------+------------------------------------+ - ->>> spark.sql("SELECT ceil(CAST(5 as DECIMAL(5, 2)), 1), typeof(ceil(CAST(5 as DECIMAL(5, 2)), 1))").show() -+--------------------------------+----------------------------------------+ -|ceil(CAST(5 AS DECIMAL(5,2)), 1)|typeof(ceil(CAST(5 AS DECIMAL(5,2)), 1))| -+--------------------------------+----------------------------------------+ -| 5.0| decimal(5,1)| -+--------------------------------+----------------------------------------+ - ->>> spark.sql("SELECT ceil(CAST(5 as DECIMAL(5, 2))), typeof(ceil(CAST(5 as DECIMAL(5, 2))))").show() -+-----------------------------+-------------------------------------+ -|CEIL(CAST(5 AS DECIMAL(5,2)))|typeof(CEIL(CAST(5 AS DECIMAL(5,2))))| -+-----------------------------+-------------------------------------+ -| 5| decimal(4,0)| -+-----------------------------+-------------------------------------+ - ->>> spark.sql("SELECT ceil(5.4, -1), typeof(ceil(5.4, -1)), typeof(5.4)").show() -+-------------+---------------------+------------+ -|ceil(5.4, -1)|typeof(ceil(5.4, -1))| typeof(5.4)| -+-------------+---------------------+------------+ -| 10| decimal(2,0)|decimal(2,1)| -+-------------+---------------------+------------+ - ->>> spark.sql("SELECT ceil(5, -1), typeof(ceil(5, -1)), typeof(5)").show() -+-----------+-------------------+---------+ -|ceil(5, -1)|typeof(ceil(5, -1))|typeof(5)| -+-----------+-------------------+---------+ -| 10| decimal(11,0)| int| -+-----------+-------------------+---------+ - ->>> spark.sql("SELECT ceil(5, 0), typeof(ceil(5, 0)), typeof(5)").show() -+----------+------------------+---------+ -|ceil(5, 0)|typeof(ceil(5, 0))|typeof(5)| -+----------+------------------+---------+ -| 5| decimal(11,0)| int| -+----------+------------------+---------+ diff --git a/python/pysail/tests/spark/function/test_concat_ws.txt b/python/pysail/tests/spark/function/test_concat_ws.txt deleted file mode 100644 index d39c5af42..000000000 --- a/python/pysail/tests/spark/function/test_concat_ws.txt +++ /dev/null @@ -1,19 +0,0 @@ ->>> from datetime import date ->>> from pyspark.sql import functions as F, types as T - ->>> schema = T.StructType([ -... T.StructField("string_col", T.StringType(), True), -... T.StructField("int_col", T.IntegerType(), True), -... T.StructField("date_col", T.DateType(), True), -... ]) ->>> df = spark.createDataFrame([("a", 1, date(2024, 1, 15))], schema) ->>> df.select(F.concat_ws("-", F.col("string_col"), F.col("int_col"), F.col("date_col"))).collect() -[Row(concat_ws(-, string_col, int_col, date_col)='a-1-2024-01-15')] - ->>> schema = T.StructType([ -... T.StructField("string_col", T.StringType(), True), -... T.StructField("int_col", T.IntegerType(), True), -... ]) ->>> df = spark.createDataFrame([("a", None)], schema) ->>> df.select(F.concat_ws("-", F.col("string_col"), F.col("int_col"))).collect() -[Row(concat_ws(-, string_col, int_col)='a')] \ No newline at end of file diff --git a/python/pysail/tests/spark/function/test_conv.txt b/python/pysail/tests/spark/function/test_conv.txt deleted file mode 100644 index 6c0cd2572..000000000 --- a/python/pysail/tests/spark/function/test_conv.txt +++ /dev/null @@ -1,7 +0,0 @@ ->>> df = spark.sql("""SELECT conv('10', 10, 2) as to_binary,conv('10', 10, 8) as to_octal,conv('10', 10, 16) as to_hex""") ->>> df.show() -+---------+--------+------+ -|to_binary|to_octal|to_hex| -+---------+--------+------+ -| 1010| 12| A| -+---------+--------+------+ diff --git a/python/pysail/tests/spark/function/test_csc.txt b/python/pysail/tests/spark/function/test_csc.txt deleted file mode 100644 index c86569f28..000000000 --- a/python/pysail/tests/spark/function/test_csc.txt +++ /dev/null @@ -1,45 +0,0 @@ ->>> from pyspark.sql import functions as sf ->>> import math - ->>> spark.sql("SELECT csc(radians(90)) AS csc").show() -+---+ -|csc| -+---+ -|1.0| -+---+ - ->>> df = spark.range(1) ->>> df.select(sf.csc(sf.lit(math.radians(90))).alias("csc")).show() -+---+ -|csc| -+---+ -|1.0| -+---+ - ->>> df.select(sf.csc(sf.lit(90)).alias("csc")).show() -+------------------+ -| csc| -+------------------+ -|1.1185724071637084| -+------------------+ - ->>> df.select(sf.csc(sf.lit(90.0)).alias("csc")).show() -+------------------+ -| csc| -+------------------+ -|1.1185724071637084| -+------------------+ - ->>> df.select(sf.csc(sf.lit(-90.0)).alias("csc")).show() -+-------------------+ -| csc| -+-------------------+ -|-1.1185724071637084| -+-------------------+ - ->>> df.select(sf.csc(sf.lit(0)).alias("csc")).show() -+--------+ -| csc| -+--------+ -|Infinity| -+--------+ diff --git a/python/pysail/tests/spark/function/test_elt.txt b/python/pysail/tests/spark/function/test_elt.txt deleted file mode 100644 index 7017343a1..000000000 --- a/python/pysail/tests/spark/function/test_elt.txt +++ /dev/null @@ -1,33 +0,0 @@ ->>> spark.sql("SELECT elt(1, 10, 20)").printSchema() -root - |-- elt(1, 10, 20): string (nullable = true) - ->>> spark.sql("SELECT elt(1, 10, 20)").show() -+--------------+ -|elt(1, 10, 20)| -+--------------+ -| 10| -+--------------+ - ->>> spark.sql("SELECT elt(1, 'scala', 'java')").show() -+-------------------+ -|elt(1, scala, java)| -+-------------------+ -| scala| -+-------------------+ - ->>> spark.sql("SELECT elt(11, 10, 20)").show() -+---------------+ -|elt(11, 10, 20)| -+---------------+ -| NULL| -+---------------+ - ->>> spark.sql("SELECT elt(6, 'scala', 'java', 'c', 'c++', 'python', 'rust')").show() -+-----------------------------------------+ -|elt(6, scala, java, c, c++, python, rust)| -+-----------------------------------------+ -| rust| -+-----------------------------------------+ - -# If spark.sql.ansi.enabled is set to true, it throws ArrayIndexOutOfBoundsException for invalid indices. diff --git a/python/pysail/tests/spark/function/test_floor.txt b/python/pysail/tests/spark/function/test_floor.txt deleted file mode 100644 index eabf021a4..000000000 --- a/python/pysail/tests/spark/function/test_floor.txt +++ /dev/null @@ -1,167 +0,0 @@ ->>> spark.sql("SELECT floor(-0.1), typeof(floor(-0.1)), typeof(-0.1)").show() -+-----------+-------------------+------------+ -|FLOOR(-0.1)|typeof(FLOOR(-0.1))|typeof(-0.1)| -+-----------+-------------------+------------+ -| -1| decimal(1,0)|decimal(1,1)| -+-----------+-------------------+------------+ - ->>> spark.sql("SELECT floor(5), typeof(floor(5)), typeof(5)").show() -+--------+----------------+---------+ -|FLOOR(5)|typeof(FLOOR(5))|typeof(5)| -+--------+----------------+---------+ -| 5| bigint| int| -+--------+----------------+---------+ - ->>> spark.sql("SELECT floor(5.4), typeof(floor(5.4)), typeof(5.4)").show() -+----------+------------------+------------+ -|FLOOR(5.4)|typeof(FLOOR(5.4))| typeof(5.4)| -+----------+------------------+------------+ -| 5| decimal(2,0)|decimal(2,1)| -+----------+------------------+------------+ - ->>> spark.sql("SELECT floor(3.1411, -3), typeof(floor(3.1411, -3)), typeof(3.1411)").show() -+-----------------+-------------------------+--------------+ -|floor(3.1411, -3)|typeof(floor(3.1411, -3))|typeof(3.1411)| -+-----------------+-------------------------+--------------+ -| 0| decimal(4,0)| decimal(5,4)| -+-----------------+-------------------------+--------------+ - ->>> spark.sql("SELECT floor(3.1411, 3), typeof(floor(3.1411, 3)), typeof(3.1411)").show() -+----------------+------------------------+--------------+ -|floor(3.1411, 3)|typeof(floor(3.1411, 3))|typeof(3.1411)| -+----------------+------------------------+--------------+ -| 3.141| decimal(5,3)| decimal(5,4)| -+----------------+------------------------+--------------+ - ->>> spark.sql("SELECT floor(3345.1, -2), typeof(floor(3345.1, -2)), typeof(3345.1)").show() -+-----------------+-------------------------+--------------+ -|floor(3345.1, -2)|typeof(floor(3345.1, -2))|typeof(3345.1)| -+-----------------+-------------------------+--------------+ -| 3300| decimal(5,0)| decimal(5,1)| -+-----------------+-------------------------+--------------+ - ->>> spark.sql("SELECT floor(-12.345, 1), typeof(floor(-12.345, 1)), typeof(3345.1)").show() -+-----------------+-------------------------+--------------+ -|floor(-12.345, 1)|typeof(floor(-12.345, 1))|typeof(3345.1)| -+-----------------+-------------------------+--------------+ -| -12.4| decimal(4,1)| decimal(5,1)| -+-----------------+-------------------------+--------------+ - ->>> spark.sql("SELECT floor(CAST(5 as TINYINT), 1), typeof(floor(CAST(5 as TINYINT), 1))").show() -+----------------------------+------------------------------------+ -|floor(CAST(5 AS TINYINT), 1)|typeof(floor(CAST(5 AS TINYINT), 1))| -+----------------------------+------------------------------------+ -| 5| decimal(4,0)| -+----------------------------+------------------------------------+ - ->>> spark.sql("SELECT floor(CAST(5 as TINYINT), -4), typeof(floor(CAST(5 as TINYINT), -4))").show() -+-----------------------------+-------------------------------------+ -|floor(CAST(5 AS TINYINT), -4)|typeof(floor(CAST(5 AS TINYINT), -4))| -+-----------------------------+-------------------------------------+ -| 0| decimal(5,0)| -+-----------------------------+-------------------------------------+ - ->>> spark.sql("SELECT floor(CAST(5 as SMALLINT), 1), typeof(floor(CAST(5 as SMALLINT), 1))").show() -+-----------------------------+-------------------------------------+ -|floor(CAST(5 AS SMALLINT), 1)|typeof(floor(CAST(5 AS SMALLINT), 1))| -+-----------------------------+-------------------------------------+ -| 5| decimal(6,0)| -+-----------------------------+-------------------------------------+ - ->>> spark.sql("SELECT floor(CAST(5 as SMALLINT), -6), typeof(floor(CAST(5 as SMALLINT), -6))").show() -+------------------------------+--------------------------------------+ -|floor(CAST(5 AS SMALLINT), -6)|typeof(floor(CAST(5 AS SMALLINT), -6))| -+------------------------------+--------------------------------------+ -| 0| decimal(7,0)| -+------------------------------+--------------------------------------+ - ->>> spark.sql("SELECT floor(CAST(5 as INT), 1), typeof(floor(CAST(5 as INT), 1))").show() -+------------------------+--------------------------------+ -|floor(CAST(5 AS INT), 1)|typeof(floor(CAST(5 AS INT), 1))| -+------------------------+--------------------------------+ -| 5| decimal(11,0)| -+------------------------+--------------------------------+ - ->>> spark.sql("SELECT floor(CAST(5 as INT), -11), typeof(floor(CAST(5 as INT), -11))").show() -+--------------------------+----------------------------------+ -|floor(CAST(5 AS INT), -11)|typeof(floor(CAST(5 AS INT), -11))| -+--------------------------+----------------------------------+ -| 0| decimal(12,0)| -+--------------------------+----------------------------------+ - ->>> spark.sql("SELECT floor(CAST(5 as BIGINT), 1), typeof(floor(CAST(5 as BIGINT), 1))").show() -+---------------------------+-----------------------------------+ -|floor(CAST(5 AS BIGINT), 1)|typeof(floor(CAST(5 AS BIGINT), 1))| -+---------------------------+-----------------------------------+ -| 5| decimal(21,0)| -+---------------------------+-----------------------------------+ - ->>> spark.sql("SELECT floor(CAST(5 as BIGINT), -21), typeof(floor(CAST(5 as BIGINT), -21))").show() -+-----------------------------+-------------------------------------+ -|floor(CAST(5 AS BIGINT), -21)|typeof(floor(CAST(5 AS BIGINT), -21))| -+-----------------------------+-------------------------------------+ -| 0| decimal(22,0)| -+-----------------------------+-------------------------------------+ - ->>> spark.sql("SELECT floor(CAST(5 as FLOAT), 1), typeof(floor(CAST(5 as FLOAT), 1))").show() -+--------------------------+----------------------------------+ -|floor(CAST(5 AS FLOAT), 1)|typeof(floor(CAST(5 AS FLOAT), 1))| -+--------------------------+----------------------------------+ -| 5.0| decimal(9,1)| -+--------------------------+----------------------------------+ - ->>> spark.sql("SELECT floor(CAST(5 as FLOAT), -15), typeof(floor(CAST(5 as FLOAT), -15))").show() -+----------------------------+------------------------------------+ -|floor(CAST(5 AS FLOAT), -15)|typeof(floor(CAST(5 AS FLOAT), -15))| -+----------------------------+------------------------------------+ -| 0| decimal(16,0)| -+----------------------------+------------------------------------+ - ->>> spark.sql("SELECT floor(CAST(5 as DOUBLE), 1), typeof(floor(CAST(5 as DOUBLE), 1))").show() -+---------------------------+-----------------------------------+ -|floor(CAST(5 AS DOUBLE), 1)|typeof(floor(CAST(5 AS DOUBLE), 1))| -+---------------------------+-----------------------------------+ -| 5.0| decimal(17,1)| -+---------------------------+-----------------------------------+ - ->>> spark.sql("SELECT floor(CAST(5 as DOUBLE), -31), typeof(floor(CAST(5 as DOUBLE), -31))").show() -+-----------------------------+-------------------------------------+ -|floor(CAST(5 AS DOUBLE), -31)|typeof(floor(CAST(5 AS DOUBLE), -31))| -+-----------------------------+-------------------------------------+ -| 0| decimal(32,0)| -+-----------------------------+-------------------------------------+ - ->>> spark.sql("SELECT floor(CAST(5 as DECIMAL(5, 2)), 1), typeof(floor(CAST(5 as DECIMAL(5, 2)), 1))").show() -+---------------------------------+-----------------------------------------+ -|floor(CAST(5 AS DECIMAL(5,2)), 1)|typeof(floor(CAST(5 AS DECIMAL(5,2)), 1))| -+---------------------------------+-----------------------------------------+ -| 5.0| decimal(5,1)| -+---------------------------------+-----------------------------------------+ - ->>> spark.sql("SELECT floor(CAST(5 as DECIMAL(5, 2))), typeof(floor(CAST(5 as DECIMAL(5, 2))))").show() -+------------------------------+--------------------------------------+ -|FLOOR(CAST(5 AS DECIMAL(5,2)))|typeof(FLOOR(CAST(5 AS DECIMAL(5,2))))| -+------------------------------+--------------------------------------+ -| 5| decimal(4,0)| -+------------------------------+--------------------------------------+ - ->>> spark.sql("SELECT floor(5.4, -1), typeof(floor(5.4, -1)), typeof(5.4)").show() -+--------------+----------------------+------------+ -|floor(5.4, -1)|typeof(floor(5.4, -1))| typeof(5.4)| -+--------------+----------------------+------------+ -| 0| decimal(2,0)|decimal(2,1)| -+--------------+----------------------+------------+ - ->>> spark.sql("SELECT floor(5, -1), typeof(floor(5, -1)), typeof(5)").show() -+------------+--------------------+---------+ -|floor(5, -1)|typeof(floor(5, -1))|typeof(5)| -+------------+--------------------+---------+ -| 0| decimal(11,0)| int| -+------------+--------------------+---------+ - ->>> spark.sql("SELECT floor(5, 0), typeof(floor(5, 0)), typeof(5)").show() -+-----------+-------------------+---------+ -|floor(5, 0)|typeof(floor(5, 0))|typeof(5)| -+-----------+-------------------+---------+ -| 5| decimal(11,0)| int| -+-----------+-------------------+---------+ diff --git a/python/pysail/tests/spark/function/test_from_csv.txt b/python/pysail/tests/spark/function/test_from_csv.txt deleted file mode 100644 index 55a382256..000000000 --- a/python/pysail/tests/spark/function/test_from_csv.txt +++ /dev/null @@ -1,35 +0,0 @@ ->>> spark.sql("SELECT from_csv('1, 0.8', 'a INT, b DOUBLE') AS result, typeof(from_csv('1, 0.8', 'a INT, b DOUBLE')) AS type").show(truncate=False) -+--------+----------------------+ -|result |type | -+--------+----------------------+ -|{1, 0.8}|struct| -+--------+----------------------+ - ->>> spark.sql("SELECT from_csv('2015-08-26 00:00:00', 'time TIMESTAMP') AS result, typeof(from_csv('2015-08-26 00:00:00', 'time TIMESTAMP')) AS type").show(truncate=False) -+---------------------+----------------------+ -|result |type | -+---------------------+----------------------+ -|{2015-08-26 00:00:00}|struct| -+---------------------+----------------------+ - ->>> spark.sql("SELECT from_csv('42;3.14;2024-12-01 10:00:00', 'id INT, value DOUBLE, timestamp TIMESTAMP', map('sep', ';')) AS result, typeof(from_csv('42;3.14;2024-12-01 10:00:00', 'id INT, value DOUBLE, timestamp TIMESTAMP', map('sep', ';'))) AS type").show(truncate=False) -+-------------------------------+-----------------------------------------------+ -|result |type | -+-------------------------------+-----------------------------------------------+ -|{42, 3.14, 2024-12-01 10:00:00}|struct| -+-------------------------------+-----------------------------------------------+ - ->>> spark.sql("SELECT from_csv('42;3.14;2024/12/01 10:00:00', 'id INT, value DOUBLE, timestamp TIMESTAMP', map('sep', ';', 'timestampFormat', 'yyyy/MM/dd HH:mm:ss')) AS result, typeof(from_csv('42;3.14;2024/12/01 10:00:00', 'id INT, value DOUBLE, timestamp TIMESTAMP', map('sep', ';', 'timestampFormat', 'yyyy/MM/dd HH:mm:ss'))) AS type").show(truncate=False) -+-------------------------------+-----------------------------------------------+ -|result |type | -+-------------------------------+-----------------------------------------------+ -|{42, 3.14, 2024-12-01 10:00:00}|struct| -+-------------------------------+-----------------------------------------------+ - - ->>> spark.sql("SELECT from_csv('26/08/2015', 'time TIMESTAMP', map('timestampFormat', 'dd/MM/yyyy')) AS result, typeof(from_csv('26/08/2015', 'time TIMESTAMP', map('timestampFormat', 'dd/MM/yyyy'))) AS type").show(truncate=False) -+---------------------+----------------------+ -|result |type | -+---------------------+----------------------+ -|{2015-08-26 00:00:00}|struct| -+---------------------+----------------------+ diff --git a/python/pysail/tests/spark/function/test_from_json.txt b/python/pysail/tests/spark/function/test_from_json.txt deleted file mode 100644 index 930ed4ca6..000000000 --- a/python/pysail/tests/spark/function/test_from_json.txt +++ /dev/null @@ -1,72 +0,0 @@ ->>> from pyspark.sql.functions import from_json ->>> from pyspark.sql.types import * - ->>> data = [(1, '''{"a": 1}''')] ->>> schema = StructType([StructField("a", IntegerType())]) ->>> df = spark.createDataFrame(data, ("key", "value")) ->>> df.select(from_json(df.value, schema).alias("json")).show(truncate=False) -+----+ -|json| -+----+ -|{1} | -+----+ - - ->>> df.select(from_json(df.value, "a INT").alias("json")).show(truncate=False) -+----+ -|json| -+----+ -|{1} | -+----+ - - ->>> df.select(from_json(df.value, "MAP").alias("json")).show(truncate=False) -+--------+ -|json | -+--------+ -|{a -> 1}| -+--------+ - - ->>> schema = ArrayType(StructType([StructField("a", IntegerType())])) ->>> df.select(from_json(df.value, schema).alias("json")).show(truncate=False) -+-----+ -|json | -+-----+ -|[{1}]| -+-----+ - - ->>> data = [(1, '''{"a": 1}'''), (2, '''{"a": 2}'''), (3, None)] ->>> df = spark.createDataFrame(data, ("key", "value")) ->>> df.select(from_json(df.value, "a INT").alias("json")).orderBy("key").show(truncate=False) -+----+ -|json| -+----+ -|{1} | -|{2} | -|NULL| -+----+ - - ->>> data = [(1, '''{"a": 1, "b": "hello"}''')] ->>> df = spark.createDataFrame(data, ("key", "value")) ->>> schema = StructType([StructField("a", IntegerType()), StructField("b", StringType())]) ->>> df.select(from_json(df.value, schema).alias("json")).show(truncate=False) -+----------+ -|json | -+----------+ -|{1, hello}| -+----------+ - - ->>> data = [(1, '''{"price": 19.99}''')] ->>> df = spark.createDataFrame(data, ("key", "value")) ->>> schema = StructType([StructField("price", DecimalType(10, 2))]) ->>> df.select(from_json(df.value, schema).alias("json")).show(truncate=False) -+-------+ -|json | -+-------+ -|{19.99}| -+-------+ - diff --git a/python/pysail/tests/spark/function/test_get_json_object.txt b/python/pysail/tests/spark/function/test_get_json_object.txt deleted file mode 100644 index 2b5d69341..000000000 --- a/python/pysail/tests/spark/function/test_get_json_object.txt +++ /dev/null @@ -1,27 +0,0 @@ ->>> spark.sql("""SELECT -... get_json_object('{"a":"x","b":1,"c":{"d":true},"e":[10,20]}', '$.a') AS a, -... get_json_object('{"a":"x","b":1,"c":{"d":true},"e":[10,20]}', '$.b') AS b, -... get_json_object('{"a":"x","b":1,"c":{"d":true},"e":[10,20]}', '$.c') AS c, -... get_json_object('{"a":"x","b":1,"c":{"d":true},"e":[10,20]}', '$.e') AS e -... """).show(truncate=False) -+---+---+----------+-------+ -|a |b |c |e | -+---+---+----------+-------+ -|x |1 |{"d":true}|[10,20]| -+---+---+----------+-------+ - ->>> spark.sql("""SELECT -... get_json_object('{"a":null}', '$.a') AS null_value, -... get_json_object('{"a":1}', '$.missing') AS missing_key, -... get_json_object(CAST(NULL AS STRING), '$.a') AS null_json, -... get_json_object('{"a":1}', CAST(NULL AS STRING)) AS null_path, -... get_json_object('not a json', '$.a') AS invalid_json, -... get_json_object('{"a":1}', 'a') AS invalid_path -... """).show(truncate=False) -+----------+-----------+---------+---------+------------+------------+ -|null_value|missing_key|null_json|null_path|invalid_json|invalid_path| -+----------+-----------+---------+---------+------------+------------+ -|NULL |NULL |NULL |NULL |NULL |NULL | -+----------+-----------+---------+---------+------------+------------+ - - diff --git a/python/pysail/tests/spark/function/test_input_order.py b/python/pysail/tests/spark/function/test_input_order.py new file mode 100644 index 000000000..c7204f2a1 --- /dev/null +++ b/python/pysail/tests/spark/function/test_input_order.py @@ -0,0 +1,123 @@ +"""Input-order preservation cases that are NOT expressible as SQL feature scenarios. + +The SQL-expressible majority of the old ``test_input_order.txt`` doctests now lives in +``features/aggregate/input_order.feature``. This module keeps the cases that need the +DataFrame API, Python UDFs, file I/O, partitioning, or plan inspection — plus one known +Sail-vs-Spark divergence (floating-point summation order). + +Note: the ``pandas_udf`` tests require the Python UDF worker, which does not run against a +local Sail server (they pass against JVM Spark, matching ``udf/test_pandas_agg_window.py``). +""" + +import re +import tempfile + +import pandas as pd +import pytest +from pyspark.sql import Window +from pyspark.sql import functions as F # noqa: N812 +from pyspark.sql.functions import pandas_udf + +from pysail.testing.spark.utils.common import is_jvm_spark + + +@pytest.mark.skipif( + not is_jvm_spark(), + reason="requires the Python UDF worker (JVM Spark or a sail-server-python launch)", +) +def test_pandas_udf_first_arrival_preserves_order(spark): + df = spark.createDataFrame([(30, "c"), (10, "a"), (20, "b")], ("v", "k")) + + @pandas_udf("long") + def first_arrival(v: pd.Series) -> int: + return int(v.iloc[0]) + + result = df.orderBy("k").groupBy().agg(first_arrival("v").alias("fa")).collect() + assert [tuple(r) for r in result] == [(10,)] + + +@pytest.mark.skipif( + not is_jvm_spark(), + reason="requires the Python UDF worker (JVM Spark or a sail-server-python launch)", +) +def test_pandas_udf_first_arrival_stable_coalesce(spark): + stable = spark.createDataFrame([(30, "c"), (10, "a"), (20, "b")], ("v", "k")).coalesce(1) + + @pandas_udf("long") + def first_arrival(v: pd.Series) -> int: + return int(v.iloc[0]) + + result = stable.orderBy("k").groupBy().agg(first_arrival("v").alias("fa")).collect() + assert [tuple(r) for r in result] == [(10,)] + + +def test_repartition_then_order_preserved(spark): + dft = spark.createDataFrame([(1, "a"), (2, "a"), (3, "a")], ("v", "k")).repartition(3) + result = dft.orderBy("k").groupBy().agg(F.first("v")).collect() + assert [tuple(r) for r in result] == [(1,)] + + +def test_coalesce_window_sees_input_order(spark): + df = spark.createDataFrame([(30, "c", "x"), (10, "a", "x"), (20, "b", "x")], ("v", "k", "g")).coalesce(1) + w_desc = Window.partitionBy("g").orderBy(F.desc("k")) + w_none = Window.partitionBy("g") + result = df.orderBy("k").select("v", "k", F.sum("v").over(w_desc).alias("s"), F.last("v").over(w_none).alias("l")) + got = sorted((r.v, r.k, r.s, r.l) for r in result.collect()) + assert got == [(10, "a", 60, 10), (20, "b", 50, 10), (30, "c", 30, 10)] + + +def test_projected_column_order_survives_parquet_roundtrip(spark): + df = spark.createDataFrame([(30, "c", "x"), (10, "a", "x"), (20, "b", "x")], ("v", "k", "g")).coalesce(1) + w_desc = Window.partitionBy("g").orderBy(F.desc("k")) + w_none = Window.partitionBy("g") + result = df.orderBy("k").select("*", F.sum("v").over(w_desc).alias("s"), F.last("v").over(w_none).alias("l")) + assert result.columns == ["v", "k", "g", "s", "l"] + assert result.select("*").columns == ["v", "k", "g", "s", "l"] + with tempfile.TemporaryDirectory() as path: + result.write.mode("overwrite").parquet(path) + written_columns = spark.read.parquet(path).columns + assert written_columns == ["v", "k", "g", "s", "l"] + + +def test_monotonically_increasing_id_does_not_disturb_order(spark): + df = spark.createDataFrame([(30, "c"), (10, "a"), (20, "b")], ("v", "k")) + result = df.orderBy("k").withColumn("i", F.monotonically_increasing_id()).groupBy().agg(F.first("v")).collect() + assert [tuple(r) for r in result] == [(10,)] + + +def test_spark_partition_id_does_not_disturb_order(spark): + df = spark.createDataFrame([(30, "c"), (10, "a"), (20, "b")], ("v", "k")) + result = df.orderBy("k").withColumn("p", F.spark_partition_id()).groupBy().agg(F.first("v")).collect() + assert [tuple(r) for r in result] == [(10,)] + + +def test_pivot_first_preserves_order(spark): + pdf = spark.createDataFrame([(30, "c", "x"), (10, "a", "x"), (20, "b", "y")], ("v", "k", "p")) + result = pdf.orderBy("k").groupBy().pivot("p").agg(F.first("v")).collect() + assert [tuple(r) for r in result] == [(10, 20)] + + +def test_first_is_not_rewritten_with_an_order_by_in_the_plan(spark): + explain = spark.sql( + "EXPLAIN EXTENDED SELECT first(v) FROM (SELECT * FROM VALUES (30,'c'),(10,'a'),(20,'b') AS t(v, k) ORDER BY k)" + ).first()[0] + assert re.search(r"first(?:_value)?\([^\n]* ORDER BY \[", explain, re.IGNORECASE) is None + + +@pytest.mark.skipif( + not is_jvm_spark(), + reason="DataFusion reduces a batch with tree/SIMD summation, so the floating-point " + "skewness of these near-cancelling values is unstable around 0 on Sail (see the FIXME " + "in the original doctest). JVM Spark sums sequentially and yields a stable non-zero value.", +) +def test_skewness_is_order_of_summation_dependent(spark): + sk = ( + spark.sql( + "SELECT skewness(v) AS sk FROM (SELECT * FROM VALUES " + "(CAST(1e16 AS DOUBLE),'b'),(CAST(1.0 AS DOUBLE),'c'),(CAST(-1e16 AS DOUBLE),'a') " + "AS t(v, k) ORDER BY k)" + ) + .first() + .sk + ) + assert sk != 0.0 diff --git a/python/pysail/tests/spark/function/test_json_array_length.txt b/python/pysail/tests/spark/function/test_json_array_length.txt deleted file mode 100644 index 8cbbdc2cc..000000000 --- a/python/pysail/tests/spark/function/test_json_array_length.txt +++ /dev/null @@ -1,26 +0,0 @@ ->>> spark.sql("SELECT json_array_length('[1,2,3]') AS arr_len, typeof(json_array_length('[1,2,3]')) AS type").show(truncate=False) -+-------+----+ -|arr_len|type| -+-------+----+ -|3 |int | -+-------+----+ - ->>> spark.sql("SELECT json_array_length('[]') AS arr_len, typeof(json_array_length('[]')) AS type").show(truncate=False) -+-------+----+ -|arr_len|type| -+-------+----+ -|0 |int | -+-------+----+ - ->>> spark.sql("""SELECT -... json_array_length(CAST(NULL AS STRING)) AS null_json, -... json_array_length('not a json') AS invalid_json, -... json_array_length('1') AS scalar_json -... """).show(truncate=False) -+---------+------------+-----------+ -|null_json|invalid_json|scalar_json| -+---------+------------+-----------+ -|NULL |NULL |NULL | -+---------+------------+-----------+ - - diff --git a/python/pysail/tests/spark/function/test_json_object_keys.txt b/python/pysail/tests/spark/function/test_json_object_keys.txt deleted file mode 100644 index b7bb05317..000000000 --- a/python/pysail/tests/spark/function/test_json_object_keys.txt +++ /dev/null @@ -1,31 +0,0 @@ ->>> spark.sql("""SELECT -... json_object_keys('{"f1":"abc","f2":{"f3":"a","f4":"b"}}') AS keys, -... typeof(json_object_keys('{"f1":"abc","f2":{"f3":"a","f4":"b"}}')) AS type -... """).show(truncate=False) -+--------+-------------+ -|keys |type | -+--------+-------------+ -|[f1, f2]|array| -+--------+-------------+ - - ->>> spark.sql("SELECT json_object_keys('{}') AS keys, typeof(json_object_keys('{}')) AS type").show(truncate=False) -+----+-------------+ -|keys|type | -+----+-------------+ -|[] |array| -+----+-------------+ - - ->>> spark.sql("""SELECT -... json_object_keys('[1,2]') AS non_object, -... json_object_keys(CAST(NULL AS STRING)) AS null_json, -... json_object_keys('not a json') AS invalid_json -... """).show(truncate=False) -+----------+---------+------------+ -|non_object|null_json|invalid_json| -+----------+---------+------------+ -|NULL |NULL |NULL | -+----------+---------+------------+ - - diff --git a/python/pysail/tests/spark/function/test_make_dt_interval.txt b/python/pysail/tests/spark/function/test_make_dt_interval.txt deleted file mode 100644 index 493d99faf..000000000 --- a/python/pysail/tests/spark/function/test_make_dt_interval.txt +++ /dev/null @@ -1,124 +0,0 @@ -# null ->>> spark.sql("SELECT (make_dt_interval(null, 0, 0, 0))").show(truncate=False) -+-------------------------------+ -|make_dt_interval(NULL, 0, 0, 0)| -+-------------------------------+ -|NULL | -+-------------------------------+ ->>> spark.sql("SELECT (make_dt_interval(0, null, 0, 0))").show(truncate=False) -+-------------------------------+ -|make_dt_interval(0, NULL, 0, 0)| -+-------------------------------+ -|NULL | -+-------------------------------+ ->>> spark.sql("SELECT (make_dt_interval(0, 0, null, 0))").show(truncate=False) -+-------------------------------+ -|make_dt_interval(0, 0, NULL, 0)| -+-------------------------------+ -|NULL | -+-------------------------------+ ->>> spark.sql("SELECT (make_dt_interval(0, 0, 0, null))").show(truncate=False) -+-------------------------------+ -|make_dt_interval(0, 0, 0, NULL)| -+-------------------------------+ -|NULL | -+-------------------------------+ - -# missing params ->>> spark.sql("SELECT (make_dt_interval()) AS make_dt_interval").show(truncate=False) -+-----------------------------------+ -|make_dt_interval | -+-----------------------------------+ -|INTERVAL '0 00:00:00' DAY TO SECOND| -+-----------------------------------+ ->>> spark.sql("SELECT (make_dt_interval(1)) AS make_dt_interval").show(truncate=False) -+-----------------------------------+ -|make_dt_interval | -+-----------------------------------+ -|INTERVAL '1 00:00:00' DAY TO SECOND| -+-----------------------------------+ ->>> spark.sql("SELECT (make_dt_interval(1, 1)) AS make_dt_interval").show(truncate=False) -+-----------------------------------+ -|make_dt_interval | -+-----------------------------------+ -|INTERVAL '1 01:00:00' DAY TO SECOND| -+-----------------------------------+ ->>> spark.sql("SELECT (make_dt_interval(1, 1, 1)) AS make_dt_interval").show(truncate=False) -+-----------------------------------+ -|make_dt_interval | -+-----------------------------------+ -|INTERVAL '1 01:01:00' DAY TO SECOND| -+-----------------------------------+ ->>> spark.sql("SELECT (make_dt_interval(1, 1, 1, 1)) AS make_dt_interval").show(truncate=False) -+-----------------------------------+ -|make_dt_interval | -+-----------------------------------+ -|INTERVAL '1 01:01:01' DAY TO SECOND| -+-----------------------------------+ - -# all 0 values ->>> spark.sql("SELECT (make_dt_interval(0, 0, 0, 0))").show(truncate=False) -+-----------------------------------+ -|make_dt_interval(0, 0, 0, 0) | -+-----------------------------------+ -|INTERVAL '0 00:00:00' DAY TO SECOND| -+-----------------------------------+ - ->>> spark.sql("SELECT (make_dt_interval(-1, 24, 0, 0)) df").show(truncate=False) -+-----------------------------------+ -|df | -+-----------------------------------+ -|INTERVAL '0 00:00:00' DAY TO SECOND| -+-----------------------------------+ ->>> spark.sql("SELECT (make_dt_interval(1, -24, 0, 0)) dt").show(truncate=False) -+-----------------------------------+ -|dt | -+-----------------------------------+ -|INTERVAL '0 00:00:00' DAY TO SECOND| -+-----------------------------------+ - ->>> spark.sql("SELECT (make_dt_interval(0, 0, 0, 0.1))").show(truncate=False) -+-------------------------------------+ -|make_dt_interval(0, 0, 0, 0.1) | -+-------------------------------------+ -|INTERVAL '0 00:00:00.1' DAY TO SECOND| -+-------------------------------------+ - -# overflow -# >>> spark.sql("SELECT (make_dt_interval(214748364, 0, 0, 0))").show(truncate=False) -# pyspark.errors.exceptions.captured.ArithmeticException: long overflow - - -# doctest https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.make_dt_interval.html ->>> import pyspark.sql.functions as sf ->>> df = spark.createDataFrame([[1, 12, 30, 01.001001]], ['day', 'hour', 'min', 'sec']) ->>> df.select('*', sf.make_dt_interval('day')).show(truncate=False) -+---+----+---+--------+-----------------------------------+ -|day|hour|min|sec |make_dt_interval(day, 0, 0, 0) | -+---+----+---+--------+-----------------------------------+ -|1 |12 |30 |1.001001|INTERVAL '1 00:00:00' DAY TO SECOND| -+---+----+---+--------+-----------------------------------+ - ->>> df = spark.createDataFrame([[1, 12, 30, 01.001001]], ['day', 'hour', 'min', 'sec']) ->>> df.select('*', sf.make_dt_interval(df.day, df.hour)).show(truncate=False) -+---+----+---+--------+-----------------------------------+ -|day|hour|min|sec |make_dt_interval(day, hour, 0, 0) | -+---+----+---+--------+-----------------------------------+ -|1 |12 |30 |1.001001|INTERVAL '1 12:00:00' DAY TO SECOND| -+---+----+---+--------+-----------------------------------+ - ->>> df = spark.createDataFrame([[1, 12, 30, 01.001001]], ['day', 'hour', 'min', 'sec']) ->>> df.select('*', sf.make_dt_interval(df.day, 'hour', df.min)).show(truncate=False) -+---+----+---+--------+-----------------------------------+ -|day|hour|min|sec |make_dt_interval(day, hour, min, 0)| -+---+----+---+--------+-----------------------------------+ -|1 |12 |30 |1.001001|INTERVAL '1 12:30:00' DAY TO SECOND| -+---+----+---+--------+-----------------------------------+ - ->>> df = spark.createDataFrame([[1, 12, 30, 01.001001]], ['day', 'hour', 'min', 'sec']) ->>> df.select('*', sf.make_dt_interval(df.day, df.hour, df.min, df.sec)).show(truncate=False) -+---+----+---+--------+------------------------------------------+ -|day|hour|min|sec |make_dt_interval(day, hour, min, sec) | -+---+----+---+--------+------------------------------------------+ -|1 |12 |30 |1.001001|INTERVAL '1 12:30:01.001001' DAY TO SECOND| -+---+----+---+--------+------------------------------------------+ diff --git a/python/pysail/tests/spark/function/test_make_interval.txt b/python/pysail/tests/spark/function/test_make_interval.txt deleted file mode 100644 index dcf59520c..000000000 --- a/python/pysail/tests/spark/function/test_make_interval.txt +++ /dev/null @@ -1,37 +0,0 @@ ->>> from pyspark.sql import Row, functions as F ->>> data = [ -... Row(y=1, m=2, w=3, d=4, h=5, mi=6, s=7.5), -... Row(y=None, m=2, w=3, d=4, h=5, mi=6, s=7.5), -... Row(y=1, m=None, w=3, d=4, h=5, mi=6, s=7.5), -... Row(y=1, m=2, w=None, d=4, h=5, mi=6, s=7.5), -... Row(y=1, m=2, w=3, d=None, h=5, mi=6, s=7.5), -... Row(y=1, m=2, w=3, d=4, h=None, mi=6, s=7.5), -... Row(y=1, m=2, w=3, d=4, h=5, mi=None, s=7.5), -... Row(y=1, m=2, w=3, d=4, h=5, mi=6, s=None), -... Row(y=1, m=1, w=1, d=1, h=1, mi=1, s=1.0), -... # NaN and infinite -... Row(y=0, m=0, w=0, d=0, h=0, mi=0, s=float('nan')), -... Row(y=0, m=0, w=0, d=0, h=0, mi=0, s=float('inf')), -... Row(y=0, m=0, w=0, d=0, h=0, mi=0, s=float('-inf')), -... ] ->>> df = spark.createDataFrame(data) ->>> df.withColumn( -... "interval", -... F.make_interval("y", "m", "w", "d", "h", "mi", "s") -... ).show(truncate=False) -+----+----+----+----+----+----+---------+------------------------------------------------------+ -|y |m |w |d |h |mi |s |interval | -+----+----+----+----+----+----+---------+------------------------------------------------------+ -|1 |2 |3 |4 |5 |6 |7.5 |1 years 2 months 25 days 5 hours 6 minutes 7.5 seconds| -|NULL|2 |3 |4 |5 |6 |7.5 |NULL | -|1 |NULL|3 |4 |5 |6 |7.5 |NULL | -|1 |2 |NULL|4 |5 |6 |7.5 |NULL | -|1 |2 |3 |NULL|5 |6 |7.5 |NULL | -|1 |2 |3 |4 |NULL|6 |7.5 |NULL | -|1 |2 |3 |4 |5 |NULL|7.5 |NULL | -|1 |2 |3 |4 |5 |6 |NULL |NULL | -|1 |1 |1 |1 |1 |1 |1.0 |1 years 1 months 8 days 1 hours 1 minutes 1 seconds | -|0 |0 |0 |0 |0 |0 |NaN |NULL | -|0 |0 |0 |0 |0 |0 |Infinity |NULL | -|0 |0 |0 |0 |0 |0 |-Infinity|NULL | -+----+----+----+----+----+----+---------+------------------------------------------------------+ diff --git a/python/pysail/tests/spark/function/test_map_from_arrays.txt b/python/pysail/tests/spark/function/test_map_from_arrays.txt deleted file mode 100644 index 3d4794681..000000000 --- a/python/pysail/tests/spark/function/test_map_from_arrays.txt +++ /dev/null @@ -1,30 +0,0 @@ ->>> spark.sql(""" -... select map_from_arrays(a, b) as result, typeof(map_from_arrays(a, b)) as type -... from values -... (array(1), array('a')), -... (NULL, NULL), -... (array(1,2,3), NULL), -... (NULL, array('b', 'c')), -... (array(4, 5), array('d', 'e')), -... (array(), array()), -... (array(6, 7, 8), array('f', 'g', 'h')) -... as tab(a, b); -... """).show(truncate=False) -+------------------------+---------------+ -|result |type | -+------------------------+---------------+ -|{1 -> a} |map| -|NULL |map| -|NULL |map| -|NULL |map| -|{4 -> d, 5 -> e} |map| -|{} |map| -|{6 -> f, 7 -> g, 8 -> h}|map| -+------------------------+---------------+ - ->>> spark.sql("select map_from_arrays(NULL, NULL) as result, typeof(map_from_arrays(NULL, NULL)) as type").show(truncate=False) -+------+--------------+ -|result|type | -+------+--------------+ -|NULL |map| -+------+--------------+ diff --git a/python/pysail/tests/spark/function/test_map_from_entries.txt b/python/pysail/tests/spark/function/test_map_from_entries.txt deleted file mode 100644 index c6519bfe9..000000000 --- a/python/pysail/tests/spark/function/test_map_from_entries.txt +++ /dev/null @@ -1,51 +0,0 @@ ->>> spark.sql(""" -... SELECT map_from_entries(data) as result, typeof(map_from_entries(data)) as type -... from VALUES -... (NULL), -... ( -... array(struct(1 as a, 'b' as b), -... struct(2 as a, NULL as b), -... struct(3 as a, 'd' as b)) -... ), -... (array()), -... (cast(NULL as array>)) -... as tab(data); -... """).show(truncate=False) -+---------------------------+---------------+ -|result |type | -+---------------------------+---------------+ -|NULL |map| -|{1 -> b, 2 -> NULL, 3 -> d}|map| -|{} |map| -|NULL |map| -+---------------------------+---------------+ - ->>> spark.sql(""" -... SELECT map_from_entries(data) as result, typeof(map_from_entries(data)) as type -... from values -... (array( -... struct(1 as a, "a" as b), -... cast(NULL as struct), -... cast(NULL as struct) -... )), -... (NULL), -... (array( -... struct(2 as a, "b" as b), -... struct(3 as a, "c" as b) -... )), -... (array( -... struct(4 as a, "d" as b), -... cast(NULL as struct), -... struct(5 as a, "e" as b), -... struct(6 as a, "f" as b) -... )) -... as tab(data); -... """).show(truncate=False) -+----------------+---------------+ -|result |type | -+----------------+---------------+ -|NULL |map| -|NULL |map| -|{2 -> b, 3 -> c}|map| -|NULL |map| -+----------------+---------------+ \ No newline at end of file diff --git a/python/pysail/tests/spark/function/test_nullability_schema.py b/python/pysail/tests/spark/function/test_nullability_schema.py new file mode 100644 index 000000000..c527dbd09 --- /dev/null +++ b/python/pysail/tests/spark/function/test_nullability_schema.py @@ -0,0 +1,141 @@ +"""Output-schema nullability via the DataFrame API. + +Complements the SQL `@spark_null` feature suite with the DataFrame path: build a +DataFrame from an explicit `StructType` (so the input nullability is exactly what +we declare — a non-nullable and a nullable column), apply functions through the +`F.*` API, and assert the resulting schema's `nullable` flags. + +Spark's `nullable` flag is static and, for a deterministic function that only +transforms its arguments, follows the inputs: + - a non-null column -> non-nullable output + - a nullable column -> nullable output + +Divergences (Sail declares `nullable` differently from Spark) are marked +`xfail(strict=True)` off JVM, the same contract as the `@sail-bug` feature tag. + +NOTE — this file is a representative sample, not exhaustive. The DataFrame-API +path and the SQL path are NOT equivalent: some divergences only show up via SQL +constant-folding. For example `bround` on a decimal is tagged `@sail-bug` in the +SQL suite (the folded `SELECT bround(2.5, 0)` path) yet matches Spark through the +DataFrame API (both column and `F.lit`) — so it passes here. `round` diverges on +both paths, which is why it stays a bug. A full DataFrame-API sweep across all +functions is a worthwhile follow-up: it is the path real PySpark code takes, and +it catches (and clears) different cases than the SQL sweep. +""" + +from decimal import Decimal + +import pytest +from pyspark.sql import functions as F # noqa: N812 +from pyspark.sql import types as T # noqa: N812 + +from pysail.testing.spark.utils.common import is_jvm_spark + +sail_bug = pytest.mark.xfail(not is_jvm_spark(), reason="Known Sail nullability bug", strict=True) + + +def _frame(spark): + """One non-nullable ("nn") and one nullable ("nl") string column.""" + schema = T.StructType( + [ + T.StructField("nn", T.StringType(), nullable=False), + T.StructField("nl", T.StringType(), nullable=True), + ] + ) + return spark.createDataFrame([("hello", "world")], schema) + + +def _decimal_frame(spark): + """One non-nullable ("nn") and one nullable ("nl") decimal column.""" + schema = T.StructType( + [ + T.StructField("nn", T.DecimalType(2, 1), nullable=False), + T.StructField("nl", T.DecimalType(2, 1), nullable=True), + ] + ) + return spark.createDataFrame([(Decimal("2.5"), Decimal("2.5"))], schema) + + +def _nullable(df, column): + return df.schema[column].nullable + + +def test_input_schema_is_honored(spark): + df = _frame(spark) + assert _nullable(df, "nn") is False + assert _nullable(df, "nl") is True + + +# --- functions that correctly propagate nullability (pass on Sail and JVM) --- + + +def test_length_propagates_nullability(spark): + out = _frame(spark).select( + F.length("nn").alias("from_nn"), + F.length("nl").alias("from_nl"), + ) + assert _nullable(out, "from_nn") is False + assert _nullable(out, "from_nl") is True + + +def test_concat_propagates_nullability(spark): + out = _frame(spark).select( + F.concat("nn", "nn").alias("both_non_null"), + F.concat("nn", "nl").alias("one_nullable"), + ) + assert _nullable(out, "both_non_null") is False + assert _nullable(out, "one_nullable") is True + + +def test_coalesce_propagates_nullability(spark): + out = _frame(spark).select( + F.coalesce("nn", "nl").alias("has_non_null"), + F.coalesce("nl", "nl").alias("all_nullable"), + ) + assert _nullable(out, "has_non_null") is False + assert _nullable(out, "all_nullable") is True + + +# --- functions where Sail is over-nullable --- +# Both cases are checked so we see exactly where each is right and wrong: +# Sail gets the nullable input right (nullable -> nullable) but the non-null +# input wrong (non-null -> Sail still says nullable, Spark says non-nullable). + + +@pytest.mark.parametrize("fn", [F.upper, F.lower, F.ascii, F.initcap]) +def test_pure_string_fn_of_nullable_is_nullable(spark, fn): + # nullable input -> nullable output: Sail and Spark agree + out = _frame(spark).select(fn("nl").alias("result")) + assert _nullable(out, "result") is True + + +@pytest.mark.parametrize("fn", [F.upper, F.lower, F.ascii, F.initcap]) +@sail_bug +def test_pure_string_fn_of_non_null_is_non_nullable(spark, fn): + # non-null input -> non-nullable output: Sail diverges (says nullable=True) + out = _frame(spark).select(fn("nn").alias("result")) + assert _nullable(out, "result") is False + + +# --- the dangerous direction: Sail UNDER-nullable --- +# round on a decimal can overflow, so Spark declares it nullable=True even for a +# non-null input. Sail declares it non-nullable -> if a NULL ever appears an +# optimizer could drop null-checks, so this is a correctness risk, not just +# schema fidelity. (bround matches Spark here, kept as the passing counterpart.) + + +def test_round_of_nullable_decimal_is_nullable(spark): + out = _decimal_frame(spark).select(F.round("nl", 1).alias("result")) + assert _nullable(out, "result") is True # Sail and Spark agree + + +def test_bround_of_non_null_decimal_is_nullable(spark): + out = _decimal_frame(spark).select(F.bround("nn", 1).alias("result")) + assert _nullable(out, "result") is True # Sail and Spark agree + + +@sail_bug +def test_round_of_non_null_decimal_stays_nullable(spark): + # Spark keeps round(decimal) nullable (overflow); Sail says non-nullable. + out = _decimal_frame(spark).select(F.round("nn", 1).alias("result")) + assert _nullable(out, "result") is True diff --git a/python/pysail/tests/spark/function/test_nullifzero.txt b/python/pysail/tests/spark/function/test_nullifzero.txt deleted file mode 100644 index 5dd661f5c..000000000 --- a/python/pysail/tests/spark/function/test_nullifzero.txt +++ /dev/null @@ -1,97 +0,0 @@ ->>> spark.sql("SELECT nullifzero(0), typeof(nullifzero(0))").show() -+-------------+---------------------+ -|nullifzero(0)|typeof(nullifzero(0))| -+-------------+---------------------+ -| NULL| int| -+-------------+---------------------+ - ->>> spark.sql("SELECT nullifzero(2), typeof(nullifzero(2))").show() -+-------------+---------------------+ -|nullifzero(2)|typeof(nullifzero(2))| -+-------------+---------------------+ -| 2| int| -+-------------+---------------------+ - ->>> spark.sql("SELECT nullifzero(CAST(0 AS TINYINT)), typeof(nullifzero(CAST(0 AS TINYINT)))").show() -+------------------------------+--------------------------------------+ -|nullifzero(CAST(0 AS TINYINT))|typeof(nullifzero(CAST(0 AS TINYINT)))| -+------------------------------+--------------------------------------+ -| NULL| tinyint| -+------------------------------+--------------------------------------+ - ->>> spark.sql("SELECT nullifzero(CAST(2 AS TINYINT)), typeof(nullifzero(CAST(2 AS TINYINT)))").show() -+------------------------------+--------------------------------------+ -|nullifzero(CAST(2 AS TINYINT))|typeof(nullifzero(CAST(2 AS TINYINT)))| -+------------------------------+--------------------------------------+ -| 2| tinyint| -+------------------------------+--------------------------------------+ - ->>> spark.sql("SELECT nullifzero(CAST(0 AS SMALLINT)), typeof(nullifzero(CAST(0 AS SMALLINT)))").show() -+-------------------------------+---------------------------------------+ -|nullifzero(CAST(0 AS SMALLINT))|typeof(nullifzero(CAST(0 AS SMALLINT)))| -+-------------------------------+---------------------------------------+ -| NULL| smallint| -+-------------------------------+---------------------------------------+ - ->>> spark.sql("SELECT nullifzero(CAST(0 AS BIGINT)), typeof(nullifzero(CAST(0 AS BIGINT)))").show() -+-----------------------------+-------------------------------------+ -|nullifzero(CAST(0 AS BIGINT))|typeof(nullifzero(CAST(0 AS BIGINT)))| -+-----------------------------+-------------------------------------+ -| NULL| bigint| -+-----------------------------+-------------------------------------+ - ->>> spark.sql("SELECT nullifzero(CAST(0.0 AS FLOAT)), typeof(nullifzero(CAST(0.0 AS FLOAT)))").show() -+------------------------------+--------------------------------------+ -|nullifzero(CAST(0.0 AS FLOAT))|typeof(nullifzero(CAST(0.0 AS FLOAT)))| -+------------------------------+--------------------------------------+ -| NULL| float| -+------------------------------+--------------------------------------+ - ->>> spark.sql("SELECT nullifzero(CAST(1.5 AS FLOAT)), typeof(nullifzero(CAST(1.5 AS FLOAT)))").show() -+------------------------------+--------------------------------------+ -|nullifzero(CAST(1.5 AS FLOAT))|typeof(nullifzero(CAST(1.5 AS FLOAT)))| -+------------------------------+--------------------------------------+ -| 1.5| float| -+------------------------------+--------------------------------------+ - ->>> spark.sql("SELECT nullifzero(CAST(0.0 AS DOUBLE)), typeof(nullifzero(CAST(0.0 AS DOUBLE)))").show() -+-------------------------------+---------------------------------------+ -|nullifzero(CAST(0.0 AS DOUBLE))|typeof(nullifzero(CAST(0.0 AS DOUBLE)))| -+-------------------------------+---------------------------------------+ -| NULL| double| -+-------------------------------+---------------------------------------+ - ->>> spark.sql("SELECT nullifzero(CAST(2.5 AS DOUBLE)), typeof(nullifzero(CAST(2.5 AS DOUBLE)))").show() -+-------------------------------+---------------------------------------+ -|nullifzero(CAST(2.5 AS DOUBLE))|typeof(nullifzero(CAST(2.5 AS DOUBLE)))| -+-------------------------------+---------------------------------------+ -| 2.5| double| -+-------------------------------+---------------------------------------+ - ->>> spark.sql("SELECT nullifzero(CAST(0.00 AS DECIMAL(10,2))), typeof(nullifzero(CAST(0.00 AS DECIMAL(10,2))))").show() -+---------------------------------------+-----------------------------------------------+ -|nullifzero(CAST(0.00 AS DECIMAL(10,2)))|typeof(nullifzero(CAST(0.00 AS DECIMAL(10,2))))| -+---------------------------------------+-----------------------------------------------+ -| NULL| decimal(10,2)| -+---------------------------------------+-----------------------------------------------+ - ->>> spark.sql("SELECT nullifzero(CAST(5.25 AS DECIMAL(10,2))), typeof(nullifzero(CAST(5.25 AS DECIMAL(10,2))))").show() -+---------------------------------------+-----------------------------------------------+ -|nullifzero(CAST(5.25 AS DECIMAL(10,2)))|typeof(nullifzero(CAST(5.25 AS DECIMAL(10,2))))| -+---------------------------------------+-----------------------------------------------+ -| 5.25| decimal(10,2)| -+---------------------------------------+-----------------------------------------------+ - ->>> spark.sql("SELECT nullifzero(-1), typeof(nullifzero(-1))").show() -+-----------------+-------------------------+ -|nullifzero((- 1))|typeof(nullifzero((- 1)))| -+-----------------+-------------------------+ -| -1| int| -+-----------------+-------------------------+ - ->>> spark.sql("SELECT nullifzero(NULL), typeof(nullifzero(NULL))").show() -+----------------+------------------------+ -|nullifzero(NULL)|typeof(nullifzero(NULL))| -+----------------+------------------------+ -| NULL| int| -+----------------+------------------------+ diff --git a/python/pysail/tests/spark/function/test_parse_url.txt b/python/pysail/tests/spark/function/test_parse_url.txt deleted file mode 100644 index 68794d619..000000000 --- a/python/pysail/tests/spark/function/test_parse_url.txt +++ /dev/null @@ -1,98 +0,0 @@ ->>> spark.sql("SELECT parse_url('https://example.com/a?x=1', 'QUERY', 'x') AS result, typeof(parse_url('https://example.com/a?x=1', 'QUERY', 'x')) AS type").show(truncate=False) -+------+------+ -|result|type | -+------+------+ -|1 |string| -+------+------+ - ->>> spark.sql("SELECT parse_url('https://example.com/path#frag', 'REF') AS result, typeof(parse_url('https://example.com/path#frag', 'REF')) AS type").show(truncate=False) -+------+------+ -|result|type | -+------+------+ -|frag |string| -+------+------+ - ->>> spark.sql("SELECT parse_url('ftp://user:pwd@ftp.example.com:21/files', 'USERINFO') AS result, typeof(parse_url('ftp://user:pwd@ftp.example.com:21/files', 'USERINFO')) AS type").show(truncate=False) -+--------+------+ -|result |type | -+--------+------+ -|user:pwd|string| -+--------+------+ - ->>> spark.sql("SELECT parse_url('http://[2001:db8::2]:8080/index.html?ok=1', 'HOST') AS result, typeof(parse_url('http://[2001:db8::2]:8080/index.html?ok=1', 'HOST')) AS type").show(truncate=False) -+-------------+------+ -|result |type | -+-------------+------+ -|[2001:db8::2]|string| -+-------------+------+ - ->>> spark.sql("SELECT parse_url('https://example.com', 'PATH') AS result, typeof(parse_url('https://example.com', 'PATH')) AS type").show(truncate=False) -+------+------+ -|result|type | -+------+------+ -| |string| -+------+------+ - ->>> spark.sql("SELECT parse_url('https://example.com/a/b?x=1&y=2#frag', 'PROTOCOL') AS result, typeof(parse_url('https://example.com/a/b?x=1&y=2#frag', 'PROTOCOL')) AS type").show(truncate=False) -+------+------+ -|result|type | -+------+------+ -|https |string| -+------+------+ - ->>> spark.sql("select parse_url('http://userinfo@spark.apache.org/path?query=1#Ref', 'HOST')").show(truncate=False) -+------------------------------------------------------------------+ -|parse_url(http://userinfo@spark.apache.org/path?query=1#Ref, HOST)| -+------------------------------------------------------------------+ -|spark.apache.org | -+------------------------------------------------------------------+ - ->>> spark.sql("select parse_url('http://userinfo@spark.apache.org/path?query=1#Ref', 'PATH')").show(truncate=False) -+------------------------------------------------------------------+ -|parse_url(http://userinfo@spark.apache.org/path?query=1#Ref, PATH)| -+------------------------------------------------------------------+ -|/path | -+------------------------------------------------------------------+ - ->>> spark.sql("select parse_url('http://userinfo@spark.apache.org/path?query=1#Ref', 'QUERY')").show(truncate=False) -+-------------------------------------------------------------------+ -|parse_url(http://userinfo@spark.apache.org/path?query=1#Ref, QUERY)| -+-------------------------------------------------------------------+ -|query=1 | -+-------------------------------------------------------------------+ - ->>> spark.sql("select parse_url('http://userinfo@spark.apache.org/path?query=1#Ref', 'REF')").show(truncate=False) -+-----------------------------------------------------------------+ -|parse_url(http://userinfo@spark.apache.org/path?query=1#Ref, REF)| -+-----------------------------------------------------------------+ -|Ref | -+-----------------------------------------------------------------+ - ->>> spark.sql("select parse_url('http://userinfo@spark.apache.org/path?query=1#Ref', 'PROTOCOL')").show(truncate=False) -+----------------------------------------------------------------------+ -|parse_url(http://userinfo@spark.apache.org/path?query=1#Ref, PROTOCOL)| -+----------------------------------------------------------------------+ -|http | -+----------------------------------------------------------------------+ - ->>> spark.sql("select parse_url('http://userinfo@spark.apache.org/path?query=1#Ref', 'FILE')").show(truncate=False) -+------------------------------------------------------------------+ -|parse_url(http://userinfo@spark.apache.org/path?query=1#Ref, FILE)| -+------------------------------------------------------------------+ -|/path?query=1 | -+------------------------------------------------------------------+ - ->>> spark.sql("select parse_url('http://userinfo@spark.apache.org/path?query=1#Ref', 'AUTHORITY')").show(truncate=False) -+-----------------------------------------------------------------------+ -|parse_url(http://userinfo@spark.apache.org/path?query=1#Ref, AUTHORITY)| -+-----------------------------------------------------------------------+ -|userinfo@spark.apache.org | -+-----------------------------------------------------------------------+ - ->>> spark.sql("select parse_url('http://userinfo@spark.apache.org/path?query=1#Ref', 'USERINFO')").show(truncate=False) -+----------------------------------------------------------------------+ -|parse_url(http://userinfo@spark.apache.org/path?query=1#Ref, USERINFO)| -+----------------------------------------------------------------------+ -|userinfo | -+----------------------------------------------------------------------+ - diff --git a/python/pysail/tests/spark/function/test_schema_of_json.txt b/python/pysail/tests/spark/function/test_schema_of_json.txt deleted file mode 100644 index 842b54e8f..000000000 --- a/python/pysail/tests/spark/function/test_schema_of_json.txt +++ /dev/null @@ -1,323 +0,0 @@ ->>> from pyspark.sql.functions import lit, schema_of_json - ->>> df = spark.range(1) ->>> df.select(schema_of_json(lit('{"a": 0}')).alias("json")).collect() -[Row(json='STRUCT')] - -The `allowUnquotedFieldNames` option allows parsing JSON strings with -unquoted field names. - ->>> schema = schema_of_json('{a: 1}', {'allowUnquotedFieldNames': 'true'}) ->>> df.select(schema.alias("json")).collect() -[Row(json='STRUCT')] - -Option keys and boolean option values are case-insensitive. - ->>> schema = schema_of_json('{a: 1}', {'ALLOWUNQUOTEDFIELDNAMES': 'True'}) ->>> df.select(schema.alias("json")).collect() -[Row(json='STRUCT')] - -Quoted field names and string values are left untouched, including in -nested structures. - ->>> schema = schema_of_json( -... '{a: {"b": [1.5], c: "x: {y: 1}"}}', -... {'allowUnquotedFieldNames': 'true'}, -... ) ->>> df.select(schema.alias("json")).collect() -[Row(json='STRUCT, c: STRING>>')] - -The options are excluded from the default column name, which only contains -the JSON string argument. - ->>> df.select(schema_of_json('{a: 1}', {'allowUnquotedFieldNames': 'true'})).columns -['schema_of_json({a: 1})'] - -The option can be explicitly disabled. - ->>> schema = schema_of_json('{"a": 2}', {'allowUnquotedFieldNames': 'false'}) ->>> df.select(schema.alias("json")).collect() -[Row(json='STRUCT')] - -Invalid boolean option values are rejected. Both Sail and JVM Spark raise an -error here, but with different exception classes (Sail raises an -`AnalysisException`, while JVM Spark fails to parse the value with Scala's -`.toBoolean` and raises an `IllegalArgumentException`). Since a doctest -cannot match multiple exception classes, we catch the error explicitly so -that the test passes against both engines. - ->>> schema = schema_of_json('{"a": 1}', {'allowUnquotedFieldNames': 'yes'}) ->>> try: -... df.select(schema.alias("json")).collect() -... except Exception: -... print("error") -error - -All expected values below were captured from JVM Spark (verified identical on -Spark 3.x and 4.x). - ->>> def check(json_str, opts=None): -... col = schema_of_json(json_str) if opts is None else schema_of_json(json_str, opts) -... return df.select(col.alias("r")).collect()[0]["r"] - -Unknown options are silently ignored, matching Spark. - ->>> check('{"a": 1}', {'myImaginaryOption': 'banana'}) -'STRUCT' - -Single quotes are allowed by default (`allowSingleQuotes` defaults to true in -Spark) and can be disabled explicitly. - ->>> check('{"a": \'x\'}') -'STRUCT' ->>> check("{'a': 1}") -'STRUCT' ->>> check(r"{'a': 'it\'s'}") -'STRUCT' ->>> check('{\'a\': \'say "hi"\'}') -'STRUCT' ->>> check('{"a": "it\'s"}') -'STRUCT' ->>> try: -... check("{'a': 1}", {'allowSingleQuotes': 'false'}) -... except Exception: -... print("error") -error - -Field names that are not valid identifiers are quoted with backticks in the -DDL output, with embedded backticks doubled. - ->>> check('{"x y": 1}') -'STRUCT<`x y`: BIGINT>' ->>> check('{"x-y": 1}') -'STRUCT<`x-y`: BIGINT>' ->>> check('{"x.y": 1}') -'STRUCT<`x.y`: BIGINT>' ->>> check('{"1a": 1}') -'STRUCT<`1a`: BIGINT>' ->>> check('{"_a": 1}') -'STRUCT<_a: BIGINT>' ->>> check('{"ab1_": 1}') -'STRUCT' ->>> check('{"x`y": 1}') -'STRUCT<`x``y`: BIGINT>' ->>> check('{"café": 1}') -'STRUCT<`café`: BIGINT>' ->>> check('{"": 1}') -'STRUCT<>' - -Booleans and nulls. - ->>> check('{"a": true}') -'STRUCT' ->>> check('{"a": false}') -'STRUCT' ->>> check('{"a": null}') -'STRUCT' ->>> check('null') -'STRING' ->>> check('{"a": {"b": null}}') -'STRUCT>' ->>> check('{"a": [null]}') -'STRUCT>' ->>> check('{"a": [null, 1]}') -'STRUCT>' - -Array element types are merged to the most specific common type. - ->>> check('{"a": [1, "x"]}') -'STRUCT>' ->>> check('{"a": [1, 2.5]}') -'STRUCT>' ->>> check('{"a": [1, 2.5, "x"]}') -'STRUCT>' ->>> check('{"a": [1, true]}') -'STRUCT>' ->>> check('{"a": [{"b": 1}, {"c": 2}]}') -'STRUCT>>' ->>> check('{"a": [{"b": 1}, {"b": "x"}]}') -'STRUCT>>' ->>> check('{"a": [{"b": 1}, 2]}') -'STRUCT>' ->>> check('{"a": [[1], [2.5]]}') -'STRUCT>>' - -Empty structs are dropped during canonicalization. - ->>> check('{"a": {}}') -'STRUCT<>' ->>> check('{}') -'STRUCT<>' - -Numbers: integers that fit in a long are BIGINT, wider integral values are -DECIMAL(p,0) up to the maximum precision of 38, and everything else is -DOUBLE. `prefersDecimal` infers floating-point values as decimals. - ->>> check('{"a": 9223372036854775807}') -'STRUCT' ->>> check('{"a": 9223372036854775808}') -'STRUCT' ->>> check('{"a": 99999999999999999999999999}') -'STRUCT' ->>> check('{"a": -99999999999999999999999999}') -'STRUCT' ->>> check('{"a": ' + '9' * 39 + '}') -'STRUCT' ->>> check('{"a": 1.7976931348623157E308}') -'STRUCT' ->>> check('{"a": 1E309}') -'STRUCT' ->>> check('{"a": 12.345678901234567890123}') -'STRUCT' ->>> check('{"a": 1.5}', {'prefersDecimal': 'true'}) -'STRUCT' ->>> check('{"a": 99999999999999999999999999}', {'prefersDecimal': 'true'}) -'STRUCT' - -With `prefersDecimal`, a value with an exponent follows Java `BigDecimal` -semantics: the scale is the number of fraction digits minus the exponent, -and the precision counts the unscaled digits, floored at the scale. A -negative scale is an error on both engines. - ->>> PD = {'prefersDecimal': 'true'} ->>> check('{"a": 1.55E1}', PD) -'STRUCT' ->>> check('{"a": 1.5E-2}', PD) -'STRUCT' ->>> check('{"a": 1E-2}', PD) -'STRUCT' ->>> check('{"a": 15E-1}', PD) -'STRUCT' ->>> check('{"a": 1.500E1}', PD) -'STRUCT' ->>> check('{"a": 1.5E0}', PD) -'STRUCT' ->>> check('{"a": 1.5E2}', PD) -Traceback (most recent call last): -... -pyspark.errors.exceptions.connect.AnalysisException: Negative scale is not allowed: '-1' ->>> check('{"a": 1E2}', PD) -Traceback (most recent call last): -... -pyspark.errors.exceptions.connect.AnalysisException: Negative scale is not allowed: '-2' ->>> check('{"a": 15E2}', PD) -Traceback (most recent call last): -... -pyspark.errors.exceptions.connect.AnalysisException: Negative scale is not allowed: '-2' ->>> check('{"a": 1.5E2}') -'STRUCT' - -Non-numeric numbers are allowed by default (`allowNonNumericNumbers` -defaults to true in Spark). - ->>> check('{"a": NaN}') -'STRUCT' ->>> check('{"a": Infinity}') -'STRUCT' - -`allowNumericLeadingZeros` allows leading zeros in integral numbers, which -are rejected by default. Leading zeros do not count toward the decimal -precision. - ->>> LZ = {'allowNumericLeadingZeros': 'true'} ->>> check('{"a": 01}', LZ) -'STRUCT' ->>> check('{"a": 0001}', LZ) -'STRUCT' ->>> check('{"a": -01}', LZ) -'STRUCT' ->>> check('{"a": 007}', LZ) -'STRUCT' ->>> check('{"a": 0}', LZ) -'STRUCT' ->>> check('{"a": 00}', LZ) -'STRUCT' ->>> check('{"a": -00}', LZ) -'STRUCT' ->>> check('{"a": 01.5}', LZ) -'STRUCT' ->>> check('{"a": 00.5}', LZ) -'STRUCT' ->>> check('{"a": 01e2}', LZ) -'STRUCT' ->>> check('{"a": 009223372036854775808}', LZ) -'STRUCT' ->>> check('{"a": 00' + '9' * 38 + '}', LZ) -'STRUCT' ->>> check('{"a": 00' + '9' * 39 + '}', LZ) -'STRUCT' ->>> check('{"01b": 01, "b": "01"}', LZ) -'STRUCT<`01b`: BIGINT, b: STRING>' - -Leading zeros in the exponent part are always valid, regardless of the -option. - ->>> check('{"a": 1e02}', LZ) -'STRUCT' ->>> check('{"a": 1e02}') -'STRUCT' - -`allowNumericLeadingZeros` combines with `prefersDecimal`. - ->>> check('{"a": 00.50}', {'allowNumericLeadingZeros': 'true', 'prefersDecimal': 'true'}) -'STRUCT' ->>> check('{"a": 01.5}', {'allowNumericLeadingZeros': 'true', 'prefersDecimal': 'true'}) -'STRUCT' - -Leading zeros are rejected by default and when the option is explicitly -disabled. Both engines raise an error, with different exception classes. - ->>> try: -... check('{"a": 01}') -... except Exception: -... print("error") -error ->>> try: -... check('{"a": 01}', {'allowNumericLeadingZeros': 'false'}) -... except Exception: -... print("error") -error - -The unquoted field name character set matches Jackson: ASCII alphanumeric -characters, `_$@#*+-`, and all non-ASCII characters. A `.` is not a valid -name character. - ->>> UQ = {'allowUnquotedFieldNames': 'true'} ->>> check('{+a: 1}', UQ) -'STRUCT<`+a`: BIGINT>' ->>> check('{a@b: 1}', UQ) -'STRUCT<`a@b`: BIGINT>' ->>> check('{a#b: 1}', UQ) -'STRUCT<`a#b`: BIGINT>' ->>> check('{a*b: 1}', UQ) -'STRUCT<`a*b`: BIGINT>' ->>> check('{a$b: 1}', UQ) -'STRUCT<`a$b`: BIGINT>' ->>> check('{$a: 1}', UQ) -'STRUCT<`$a`: BIGINT>' ->>> check('{_a: 1}', UQ) -'STRUCT<_a: BIGINT>' ->>> check('{9a: 1}', UQ) -'STRUCT<`9a`: BIGINT>' ->>> check('{a9: 1}', UQ) -'STRUCT' ->>> check('{ключ: 1}', UQ) -'STRUCT<`ключ`: BIGINT>' ->>> check('{aé: 1}', UQ) -'STRUCT<`aé`: BIGINT>' ->>> try: -... check('{.a: 1}', UQ) -... except Exception: -... print("error") -error ->>> for s in ['{`a: 1}', '{a`b: 1}', '{a: 1}', '{\\a: 1}']: -... try: -... check(s, UQ) -... except Exception: -... print("error") -error -error -error -error -error -error diff --git a/python/pysail/tests/spark/function/test_sentences.txt b/python/pysail/tests/spark/function/test_sentences.txt deleted file mode 100644 index ce534cf98..000000000 --- a/python/pysail/tests/spark/function/test_sentences.txt +++ /dev/null @@ -1,55 +0,0 @@ ->>> spark.sql("SELECT sentences('Hi there! Good morning.') AS result").show(truncate=False) -+... -|result...| -+... -|[[Hi, there], [Good, morning]]...| -+... - - ->>> spark.sql("SELECT sentences('Hello world') AS result").show(truncate=False) -+... -|result...| -+... -|[[Hello, world]]...| -+... - - ->>> spark.sql("SELECT sentences('What? Yes! OK.') AS result").show(truncate=False) -+... -|result...| -+... -|[[What], [Yes], [OK]]...| -+... - - ->>> spark.sql("SELECT sentences(NULL) AS result").show(truncate=False) -+------+ -|result| -+------+ -|NULL | -+------+ - - ->>> spark.sql("SELECT sentences('') AS result").show(truncate=False) -+------+ -|result| -+------+ -|[] | -+------+ - - ->>> spark.sql("SELECT sentences('Hi there!', 'en') AS result").show(truncate=False) -+... -|result...| -+... -|[[Hi, there]]...| -+... - - ->>> spark.sql("SELECT sentences('Hi there!', 'en', 'US') AS result").show(truncate=False) -+... -|result...| -+... -|[[Hi, there]]...| -+... - diff --git a/python/pysail/tests/spark/function/test_split.txt b/python/pysail/tests/spark/function/test_split.txt deleted file mode 100644 index d64566839..000000000 --- a/python/pysail/tests/spark/function/test_split.txt +++ /dev/null @@ -1,129 +0,0 @@ ->>> spark.sql("SELECT split('oneAtwoBthreeC', '[ABC]') AS result, typeof(split('oneAtwoBthreeC', '[ABC]')) AS type").show(truncate=False) -+-------------------+-------------+ -|result |type | -+-------------------+-------------+ -|[one, two, three, ]|array| -+-------------------+-------------+ - ->>> spark.sql("SELECT split('1A2B3C', '[1-9]+') AS result, typeof(split('1A2B3C', '[1-9]+')) AS type").show(truncate=False) -+-----------+-------------+ -|result |type | -+-----------+-------------+ -|[, A, B, C]|array| -+-----------+-------------+ - ->>> spark.sql("SELECT split('aa2bb3cc4', NULL) AS result, typeof(split('aa2bb3cc4', NULL)) AS type").show(truncate=False) -+------+-------------+ -|result|type | -+------+-------------+ -|NULL |array| -+------+-------------+ - ->>> spark.sql("SELECT split(NULL, '[1-9]+') AS result, typeof(split(NULL, '[1-9]+')) AS type").show(truncate=False) -+------+-------------+ -|result|type | -+------+-------------+ -|NULL |array| -+------+-------------+ - ->>> spark.sql("SELECT split('oneAtwoBthreeC', '[ABC]', -4) AS result, typeof(split('oneAtwoBthreeC', '[ABC]', -4)) AS type").show(truncate=False) -+-------------------+-------------+ -|result |type | -+-------------------+-------------+ -|[one, two, three, ]|array| -+-------------------+-------------+ - ->>> spark.sql("SELECT split('oneAtwoBthreeC', '[ABC]', -1) AS result, typeof(split('oneAtwoBthreeC', '[ABC]', -1)) AS type").show(truncate=False) -+-------------------+-------------+ -|result |type | -+-------------------+-------------+ -|[one, two, three, ]|array| -+-------------------+-------------+ - ->>> spark.sql("SELECT split('oneAtwoBthreeC', '[ABC]', 0) AS result, typeof(split('oneAtwoBthreeC', '[ABC]', 0)) AS type").show(truncate=False) -+-------------------+-------------+ -|result |type | -+-------------------+-------------+ -|[one, two, three, ]|array| -+-------------------+-------------+ - ->>> spark.sql("SELECT split('oneAtwoBthreeC', '[ABC]', 1) AS result, typeof(split('oneAtwoBthreeC', '[ABC]', 1)) AS type").show(truncate=False) -+----------------+-------------+ -|result |type | -+----------------+-------------+ -|[oneAtwoBthreeC]|array| -+----------------+-------------+ - ->>> spark.sql("SELECT split('oneAtwoBthreeC', '[ABC]', 2) AS result, typeof(split('oneAtwoBthreeC', '[ABC]', 2)) AS type").show(truncate=False) -+-----------------+-------------+ -|result |type | -+-----------------+-------------+ -|[one, twoBthreeC]|array| -+-----------------+-------------+ - ->>> spark.sql("SELECT split('oneAtwoBthreeC', '[ABC]', 10) AS result, typeof(split('oneAtwoBthreeC', '[ABC]', 10)) AS type").show(truncate=False) -+-------------------+-------------+ -|result |type | -+-------------------+-------------+ -|[one, two, three, ]|array| -+-------------------+-------------+ - ->>> spark.sql("SELECT split('1A2B3C', '[1-9]+', 1) AS result, typeof(split('1A2B3C', '[1-9]+', 1)) AS type").show(truncate=False) -+--------+-------------+ -|result |type | -+--------+-------------+ -|[1A2B3C]|array| -+--------+-------------+ - ->>> spark.sql("SELECT split('aa2bb3cc4', '[1-9]+', -1) AS result, typeof(split('aa2bb3cc4', '[1-9]+', -1)) AS type").show(truncate=False) -+--------------+-------------+ -|result |type | -+--------------+-------------+ -|[aa, bb, cc, ]|array| -+--------------+-------------+ - ->>> spark.sql("SELECT split('aa2bb3cc4', '[1-9]+', 2) AS result, typeof(split('aa2bb3cc4', '[1-9]+', 2)) AS type").show(truncate=False) -+------------+-------------+ -|result |type | -+------------+-------------+ -|[aa, bb3cc4]|array| -+------------+-------------+ - ->>> spark.sql("SELECT split('aa2bb3cc4', '[1-9]+', NULL) AS result, typeof(split('aa2bb3cc4', '[1-9]+', NULL)) AS type").show(truncate=False) -+------+-------------+ -|result|type | -+------+-------------+ -|NULL |array| -+------+-------------+ - ->>> spark.sql("SELECT split('aa2bb3cc4', NULL, -1) AS result, typeof(split('aa2bb3cc4', NULL, -1)) AS type").show(truncate=False) -+------+-------------+ -|result|type | -+------+-------------+ -|NULL |array| -+------+-------------+ - ->>> spark.sql("SELECT split(NULL, '[1-9]+', -1) AS result, typeof(split(NULL, '[1-9]+', -1)) AS type").show(truncate=False) -+------+-------------+ -|result|type | -+------+-------------+ -|NULL |array| -+------+-------------+ - ->>> spark.sql("SELECT split(s, p, l) AS result, typeof(split(s, p, l)) AS type FROM VALUES ('oneAtwoBthreeC', '[ABC]', -4), ('oneAtwoBthreeC', '[ABC]', -1), ('oneAtwoBthreeC', '[ABC]', 0), ('oneAtwoBthreeC', '[ABC]', 1), ('oneAtwoBthreeC', '[ABC]', 2), ('oneAtwoBthreeC', '[ABC]', 10), ('1A2B3C', '[1-9]+', 1), ('aa2bb3cc4', '[1-9]+', -1), ('aa2bb3cc4', '[1-9]+', 2), ('aa2bb3cc4', '[1-9]+', NULL), ('aa2bb3cc4', NULL, -1), (NULL, '[1-9]+', -1) AS T(s, p, l)").show(truncate=False) -+-------------------+-------------+ -|result |type | -+-------------------+-------------+ -|[one, two, three, ]|array| -|[one, two, three, ]|array| -|[one, two, three, ]|array| -|[oneAtwoBthreeC] |array| -|[one, twoBthreeC] |array| -|[one, two, three, ]|array| -|[1A2B3C] |array| -|[aa, bb, cc, ] |array| -|[aa, bb3cc4] |array| -|NULL |array| -|NULL |array| -|NULL |array| -+-------------------+-------------+ diff --git a/python/pysail/tests/spark/function/test_to_char.txt b/python/pysail/tests/spark/function/test_to_char.txt deleted file mode 100644 index 6b1b133b4..000000000 --- a/python/pysail/tests/spark/function/test_to_char.txt +++ /dev/null @@ -1,388 +0,0 @@ -Tests for `to_char`. - -All expected values below were captured from JVM Spark. Numeric cases are -verified identical across Spark 3.x and 4.x. Datetime and binary inputs are only -supported by JVM Spark 4.x (Spark 3.x requires a DECIMAL input), so those examples -are marked SAIL_ONLY; their expected values were captured from JVM Spark -4.x. - ->>> from pyspark.sql.functions import lit, to_char - ->>> df = spark.createDataFrame([(78.12,)], ["e"]) ->>> df.select(to_char(df.e, lit("$99.99")).alias("r")).collect() -[Row(r='$78.12')] - -The helpers below evaluate a SQL expression and return the result value, or -print "error" if the query fails. Error cases use the helper (instead of -matching the exception text) because Sail and the two JVM Spark versions -raise errors with different classes and messages. - ->>> def c(expr): -... return spark.sql(f"SELECT {expr} AS r").collect()[0]["r"] ->>> def err(expr): -... try: -... spark.sql(f"SELECT {expr} AS r").collect() -... print("no error") -... except Exception: -... print("error") - -Digit sequences: '9' pads with spaces, '0' pads with zeros. A sequence -starting with '0' before the decimal point zero-pads the whole sequence. - ->>> c("to_char(454, '999')") -'454' ->>> c("to_char(454, '9999')") -' 454' ->>> c("to_char(454, '99999')") -' 454' ->>> c("to_char(454, '0999')") -'0454' ->>> c("to_char(454, '00999')") -'00454' ->>> c("to_char(454, '0000')") -'0454' ->>> c("to_char(454, '90999')") -' 454' ->>> c("to_char(454, '99099')") -' 454' ->>> c("to_char(1, '9')") -'1' ->>> c("to_char(1, '0')") -'1' - -Values with more digits than the format are replaced by '#' characters. - ->>> c("to_char(454, '99')") -'##' ->>> c("to_char(454, '00')") -'##' ->>> c("to_char(123.456, '999D9')") -'###.#' ->>> c("to_char(78.123, '99.9')") -'##.#' ->>> c("to_char(12345, '9,999')") -'# ###' ->>> c("to_char(0.05, '9.9')") -'#.#' - -Zero values. - ->>> c("to_char(0, '9')") -' ' ->>> c("to_char(0, '99')") -' ' ->>> c("to_char(0, '0')") -'0' ->>> c("to_char(0, '00')") -'00' ->>> c("to_char(0, '99.99')") -' 0.00' ->>> c("to_char(0, '00.00')") -'00.00' ->>> c("to_char(0, 'S99')") -' +' ->>> c("to_char(0, '99S')") -'+ ' ->>> c("to_char(0, '$99.99')") -'$ 0.00' ->>> c("to_char(0.0, '9.9')") -'0.0' - -Decimal point ('.' or 'D'). - ->>> c("to_char(78, '99.99')") -'78.00' ->>> c("to_char(78, '99.')") -'78 ' ->>> c("to_char(78, '99D')") -'78 ' ->>> c("to_char(0.45, '.99')") -'.45' ->>> c("to_char(0.45, 'D99')") -'.45' ->>> c("to_char(0.45, '9.99')") -'0.45' ->>> c("to_char(0.45, '99.99')") -' 0.45' - -Grouping separators (',' or 'G'). - ->>> c("to_char(12454, '99G999')") -'12,454' ->>> c("to_char(1234, '9,999')") -'1,234' ->>> c("to_char(1234, '9G999')") -'1,234' ->>> c("to_char(45, '9,999')") -' 45' ->>> c("to_char(45, '0,000')") -'0,045' ->>> c("to_char(1234567, '9,999,999')") -'1,234,567' ->>> c("to_char(1234567, '9999,999')") -'1234,567' ->>> c("to_char(12454, '99G99,9')") -'12,45,4' ->>> c("to_char(1234.56, '9,999.99')") -'1,234.56' - -Currency sign. - ->>> c("to_char(78.12, '$99.99')") -'$78.12' ->>> c("to_char(78.12, '$00.00')") -'$78.12' ->>> c("to_char(-78.12, '$99.99')") -'$78.12' ->>> c("to_char(0.12, '$.99')") -'$.12' - -'S' prints '+' or '-'; 'MI' prints '-' or a space. Signs stay adjacent to -the digits when the digit sequence is space-padded. - ->>> c("to_char(454, 'S999')") -'+454' ->>> c("to_char(-454, 'S999')") -'-454' ->>> c("to_char(454, '999S')") -'454+' ->>> c("to_char(-454, '999S')") -'454-' ->>> c("to_char(-12454.8, '99G999D9S')") -'12,454.8-' ->>> c("to_char(-1, 'S9999')") -' -1' ->>> c("to_char(1, 'S9999')") -' +1' ->>> c("to_char(-1, 'S0000')") -'-0001' ->>> c("to_char(-454, 'S$999')") -'-$454' ->>> c("to_char(-78.12, 'S$99.99')") -'-$78.12' ->>> c("to_char(-78.12, '$99.99S')") -'$78.12-' ->>> c("to_char(454, 'MI999')") -' 454' ->>> c("to_char(-454, 'MI999')") -'-454' ->>> c("to_char(454, '999MI')") -'454 ' ->>> c("to_char(-454, '999MI')") -'454-' ->>> c("to_char(-1, 'MI9999')") -' -1' ->>> c("to_char(-1, '9999MI')") -' 1-' - -'PR' wraps negative values in angle brackets; positive values get two -trailing spaces. - ->>> c("to_char(454, '999PR')") -'454 ' ->>> c("to_char(-454, '999PR')") -'<454>' ->>> c("to_char(-454, '9999PR')") -' <454>' ->>> c("to_char(-4, '9999PR')") -' <4>' ->>> c("to_char(-78.12, '$99.99PR')") -'<$78.12>' ->>> c("to_char(-454, 'S999PR')") -'<-454>' - -Sign and decimal point interaction. A decimal point left dangling at the -end of the digits is replaced by a space. - ->>> c("to_char(-0.1, 'S9.9')") -' -.1' ->>> c("to_char(-0.1, '9.9S')") -'0.1-' ->>> c("to_char(-0.1, '9.9MI')") -'0.1-' ->>> c("to_char(0.1, '9.9S')") -'0.1+' ->>> c("to_char(78, '99.S')") -'78+ ' ->>> c("to_char(-78, '99.S')") -'78- ' ->>> c("to_char(-78, '99.PR')") -'<78> ' - -Without a sign token, the sign of a negative value is dropped. - ->>> c("to_char(-454, '999')") -'454' ->>> c("to_char(-12.34, '99.99')") -'12.34' - -Trailing zeros of the decimal value beyond its significant digits are -dropped before formatting, and the format pads with zeros after the -decimal point. - ->>> c("to_char(CAST(78.12 AS DECIMAL(10,4)), '99.99')") -'78.12' ->>> c("to_char(CAST(78.1 AS DECIMAL(10,4)), '99.99')") -'78.10' ->>> c("to_char(CAST(78 AS DECIMAL(10,4)), '99.99')") -'78.00' ->>> c("to_char(CAST(0.5 AS DECIMAL(10,4)), '99.99')") -' 0.50' ->>> c("to_char(454.00, '000D00')") -'454.00' ->>> c("to_char(CAST(12454.8 AS DECIMAL(20,10)), '99G999D9S')") -'12,454.8+' - -Floating-point inputs are converted like Spark casts them to decimals: -DOUBLE to DECIMAL(30, 15) and FLOAT to DECIMAL(14, 7), based on the -shortest decimal representation of the (widened) double value. Note that -`CAST(78.12 AS FLOAT)` widens to the double 78.12000274658203, whose seven -fractional digits 1200027 overflow the two fractional digits in the format. - ->>> c("to_char(78.12D, '$99.99')") -'$78.12' ->>> c("to_char(CAST(0.001 AS DOUBLE), '9.999')") -'0.001' ->>> c("to_char(CAST(0.1 AS DOUBLE), '9.9')") -'0.1' ->>> c("to_char(CAST(-12454.8 AS DOUBLE), '99G999D9S')") -'12,454.8-' ->>> c("to_char(CAST(0.12345678901234567 AS DOUBLE), '9.99999999999999999')") -'0.12345678901234600' ->>> c("to_char(CAST(1.5 AS FLOAT), '9.9')") -'1.5' ->>> c("to_char(CAST(78.12 AS FLOAT), '99.99')") -'##.##' - -Integer inputs. - ->>> c("to_char(CAST(127 AS TINYINT), '999')") -'127' ->>> c("to_char(CAST(-32768 AS SMALLINT), 'S99999')") -'-32768' ->>> c("to_char(CAST(2147483647 AS INT), '9999999999')") -'2147483647' ->>> c("to_char(CAST(9223372036854775807 AS BIGINT), '9999999999999999999')") -'9223372036854775807' - -String inputs are parsed as decimals. - ->>> c("to_char('78.12', '99.99')") -'78.12' ->>> c("to_char('454', '999')") -'454' ->>> c("to_char('-454', 'S999')") -'-454' - -NULL handling: a NULL value or a NULL format yields NULL. - ->>> c("to_char(CAST(NULL AS DECIMAL(5,2)), '999')") is None -True ->>> c("to_char(CAST(NULL AS INT), '999')") is None -True ->>> c("to_char(454, CAST(NULL AS STRING))") is None -True - -Large decimals beyond 64-bit unscaled values. - ->>> c("to_char(CAST('12345678901234567890123456789012345678' AS DECIMAL(38,0)), '99999999999999999999999999999999999999')") -'12345678901234567890123456789012345678' ->>> c("to_char(CAST('1234567890.0987654321' AS DECIMAL(38,18)), '9999999999.999999999999999999')") -'1234567890.098765432100000000' - -The format string is case-insensitive. - ->>> c("to_char(-12454.8, '99g999d9s')") -'12,454.8-' ->>> c("to_char(-454, '999pr')") -'<454>' ->>> c("to_char(-454, 'mi999')") -'-454' ->>> c("to_char(78.12, '$99d99')") -'$78.12' - -Invalid number formats raise an error. - ->>> err("to_char(454, '')") -error ->>> err("to_char(454, 'SS999')") -error ->>> err("to_char(454, '9.9.9')") -error ->>> err("to_char(454, '$$999')") -error ->>> err("to_char(454, '9$9')") -error ->>> err("to_char(454, '.99$')") -error ->>> err("to_char(78.12, '99.99$')") -error ->>> err("to_char(454, ',999')") -error ->>> err("to_char(454, '999,')") -error ->>> err("to_char(454, '9,,99')") -error ->>> err("to_char(454, '99.9,9')") -error ->>> err("to_char(454, 'PR999')") -error ->>> err("to_char(454, 'MI999MI')") -error ->>> err("to_char(454, 'x99')") -error ->>> err("to_char(454, '99x')") -error ->>> err("to_char(12.34, '99v99')") -error ->>> err("to_char(454, 'S')") -error ->>> err("to_char(454, '$')") -error ->>> err("to_char(454, 'MI')") -error ->>> err("to_char(454, 'PR')") -error ->>> err("to_char(454, '999PRMI')") -error - -Datetime inputs use the Spark datetime pattern syntax. JVM Spark only -supports this for 4.x, so these examples are skipped on JVM Spark; the -expected values match JVM Spark 4.x. - ->>> c("to_char(date'2016-04-08', 'y')") # doctest: +SAIL_ONLY -'2016' ->>> c("to_char(date'2022-04-08', 'yyyy-MM-dd')") # doctest: +SAIL_ONLY -'2022-04-08' ->>> c("to_char(timestamp'2016-04-08 13:14:15', 'yyyy-MM-dd HH:mm:ss')") # doctest: +SAIL_ONLY -'2016-04-08 13:14:15' ->>> c("to_char(date'2022-04-08', 'MMM')") # doctest: +SAIL_ONLY -'Apr' ->>> c("to_char(date'2022-04-08', 'E')") # doctest: +SAIL_ONLY -'Fri' ->>> c("to_char(timestamp_ntz'2016-04-08 13:14:15', 'yyyy MM dd')") # doctest: +SAIL_ONLY -'2016 04 08' ->>> c("to_char(CAST(NULL AS DATE), 'yyyy')") is None # doctest: +SAIL_ONLY -True - -Binary inputs support the 'base64', 'hex', and 'utf-8' formats -(case-insensitive, surrounding whitespace ignored). JVM Spark only -supports this for 4.x, so these examples are skipped on JVM Spark; the -expected values match JVM Spark 4.x. - ->>> c("to_char(encode('Spark SQL', 'utf-8'), 'base64')") # doctest: +SAIL_ONLY -'U3BhcmsgU1FM' ->>> c("to_char(encode('Spark SQL', 'utf-8'), 'hex')") # doctest: +SAIL_ONLY -'537061726B2053514C' ->>> c("to_char(encode('abc', 'utf-8'), 'utf-8')") # doctest: +SAIL_ONLY -'abc' ->>> c("to_char(encode('Spark SQL', 'utf-8'), 'BASE64')") # doctest: +SAIL_ONLY -'U3BhcmsgU1FM' ->>> c("to_char(encode('Spark SQL', 'utf-8'), ' hex ')") # doctest: +SAIL_ONLY -'537061726B2053514C' ->>> c("to_char(encode('abc', 'utf-8'), 'UTF-8')") # doctest: +SAIL_ONLY -'abc' ->>> err("to_char(encode('abc', 'utf-8'), 'foo')") # doctest: +SAIL_ONLY -error ->>> err("to_char(encode('abc', 'utf-8'), CAST(NULL AS STRING))") # doctest: +SAIL_ONLY -error diff --git a/python/pysail/tests/spark/function/test_to_date.txt b/python/pysail/tests/spark/function/test_to_date.txt deleted file mode 100644 index 549d72ea9..000000000 --- a/python/pysail/tests/spark/function/test_to_date.txt +++ /dev/null @@ -1,37 +0,0 @@ -# Timestamp args not covered by Spark tests ->>> from pyspark.sql import functions as sf ->>> spark.conf.set("spark.sql.session.timeZone", "UTC") ->>> spark.sql("SELECT to_date(TIMESTAMP_NTZ '2025-11-02 23:30:45.123456')").show(truncate=False) -+---------------------------------------------------+ -|to_date(TIMESTAMP_NTZ '2025-11-02 23:30:45.123456')| -+---------------------------------------------------+ -|2025-11-02 | -+---------------------------------------------------+ ->>> spark.sql("SELECT to_date(TIMESTAMP_LTZ '2025-11-02 23:30:45.123456')").show(truncate=False) -+-----------------------------------------------+ -|to_date(TIMESTAMP '2025-11-02 23:30:45.123456')| -+-----------------------------------------------+ -|2025-11-02 | -+-----------------------------------------------+ ->>> spark.sql("SELECT to_date(TIMESTAMP_LTZ '2025-11-02 23:30:45.123456 America/New_York')").show(truncate=False) -+-----------------------------------------------+ -|to_date(TIMESTAMP '2025-11-03 04:30:45.123456')| -+-----------------------------------------------+ -|2025-11-03 | -+-----------------------------------------------+ ->>> spark.sql("SELECT to_date(TIMESTAMP '2025-11-03 23:30:45.123456', 'invalid_format')").show(truncate=False) -+---------------------------------------------------------------+ -|to_date(TIMESTAMP '2025-11-03 23:30:45.123456', invalid_format)| -+---------------------------------------------------------------+ -|2025-11-03 | -+---------------------------------------------------------------+ ->>> df = spark.createDataFrame([("2025-11-02 23:30:45.123456",), ("2025-11-02 23:30:45.123456-08:00",), ("2025-11-02 23:30:45.123456+01:00",)], ["ts"]) ->>> df = df.select("ts", sf.expr("CAST(ts AS TIMESTAMP_NTZ)").alias("ts_ntz"), sf.expr("CAST(ts AS TIMESTAMP_LTZ)").alias("ts_ltz")) ->>> df.select("ts", "ts_ntz", "ts_ltz", sf.to_date("ts_ntz").alias("date_ntz"), sf.to_date("ts_ltz").alias("date_ltz")).show(truncate=False) -+--------------------------------+--------------------------+--------------------------+----------+----------+ -|ts |ts_ntz |ts_ltz |date_ntz |date_ltz | -+--------------------------------+--------------------------+--------------------------+----------+----------+ -|2025-11-02 23:30:45.123456 |2025-11-02 23:30:45.123456|2025-11-02 23:30:45.123456|2025-11-02|2025-11-02| -|2025-11-02 23:30:45.123456-08:00|2025-11-02 23:30:45.123456|2025-11-03 07:30:45.123456|2025-11-02|2025-11-03| -|2025-11-02 23:30:45.123456+01:00|2025-11-02 23:30:45.123456|2025-11-02 22:30:45.123456|2025-11-02|2025-11-02| -+--------------------------------+--------------------------+--------------------------+----------+----------+ diff --git a/python/pysail/tests/spark/function/test_to_number.txt b/python/pysail/tests/spark/function/test_to_number.txt deleted file mode 100644 index a9b226cd0..000000000 --- a/python/pysail/tests/spark/function/test_to_number.txt +++ /dev/null @@ -1,83 +0,0 @@ ->>> spark.sql("SELECT to_number('<-$12,345.67>', 'S$999,099.99PR') AS result, typeof(to_number('<-$12,345.67>', 'S$999,099.99PR')) AS type").show(truncate=False) -+--------+------------+ -|result |type | -+--------+------------+ -|12345.67|decimal(8,2)| -+--------+------------+ - ->>> spark.sql("SELECT to_number('<$1,212,345.67>', 'S$0,000,000.99PR') AS result, typeof(to_number('<$1,212,345.67>', 'S$0,000,000.99PR')) AS type").show(truncate=False) -+-----------+------------+ -|result |type | -+-----------+------------+ -|-1212345.67|decimal(9,2)| -+-----------+------------+ - ->>> spark.sql("SELECT to_number('$345', 'S$999,099.99') AS result, typeof(to_number('$345', 'S$999,099.99')) AS type").show(truncate=False) -+------+------------+ -|result|type | -+------+------------+ -|345.00|decimal(8,2)| -+------+------------+ - ->>> spark.sql("SELECT to_number('$045', 'S$999,099.99') AS result, typeof(to_number('$045', 'S$999,099.99')) AS type").show(truncate=False) -+------+------------+ -|result|type | -+------+------------+ -|45.00 |decimal(8,2)| -+------+------------+ - ->>> spark.sql("SELECT to_number('<1234>', '999999PR') AS result, typeof(to_number('<1234>', '999999PR')) AS type").show(truncate=False) -+------+------------+ -|result|type | -+------+------------+ -|-1234 |decimal(6,0)| -+------+------------+ - ->>> spark.sql("SELECT to_number('12,454.8-', '99,999.9S') AS result, typeof(to_number('12,454.8-', '99,999.9S')) AS type").show(truncate=False) -+--------+------------+ -|result |type | -+--------+------------+ -|-12454.8|decimal(6,1)| -+--------+------------+ - ->>> spark.sql("SELECT to_number('<-$123,456.32>', 'S$999,999.999999PR') AS result, typeof(to_number('<-$123,456.32>', 'S$999,999.999999PR')) AS type").show(truncate=False) -+-------------+-------------+ -|result |type | -+-------------+-------------+ -|123456.320000|decimal(12,6)| -+-------------+-------------+ - ->>> spark.sql("SELECT to_number('$123,456.32', 'MI$999,999.99S') AS result, typeof(to_number('$123,456.32', 'MI$999,999.99S')) AS type").show(truncate=False) -+---------+------------+ -|result |type | -+---------+------------+ -|123456.32|decimal(8,2)| -+---------+------------+ - ->>> spark.sql("SELECT to_number('$123,456.32-', 'MI$999,999.99S') AS result, typeof(to_number('$123,456.32-', 'MI$999,999.99S')) AS type").show(truncate=False) -+----------+------------+ -|result |type | -+----------+------------+ -|-123456.32|decimal(8,2)| -+----------+------------+ - ->>> spark.sql("SELECT to_number('-$123,456.32-', 'MI$999,999.99S') AS result, typeof(to_number('-$123,456.32-', 'MI$999,999.99S')) AS type").show(truncate=False) -+---------+------------+ -|result |type | -+---------+------------+ -|123456.32|decimal(8,2)| -+---------+------------+ - ->>> spark.sql("SELECT to_number('-$123,456.32+', 'MI$999,999.99S') AS result, typeof(to_number('-$123,456.32+', 'MI$999,999.99S')) AS type").show(truncate=False) -+----------+------------+ -|result |type | -+----------+------------+ -|-123456.32|decimal(8,2)| -+----------+------------+ - ->>> spark.sql("SELECT to_number('-$123,456.32', 'MI$999,999.99S') AS result, typeof(to_number('-$123,456.32', 'MI$999,999.99S')) AS type").show(truncate=False) -+----------+------------+ -|result |type | -+----------+------------+ -|-123456.32|decimal(8,2)| -+----------+------------+ diff --git a/python/pysail/tests/spark/function/test_to_timestamp_ntz.txt b/python/pysail/tests/spark/function/test_to_timestamp_ntz.txt deleted file mode 100644 index cffefe979..000000000 --- a/python/pysail/tests/spark/function/test_to_timestamp_ntz.txt +++ /dev/null @@ -1,32 +0,0 @@ ->>> from datetime import datetime ->>> from zoneinfo import ZoneInfo ->>> from pyspark.sql import functions as F, types as T ->>> ->>> spark.conf.set("spark.sql.session.timeZone", "Europe/Amsterdam") ->>> ->>> df = spark.createDataFrame( -... [ -... (datetime(2023, 1, 1, 10, tzinfo=ZoneInfo("Europe/Amsterdam")),), -... (datetime(2023, 1, 1, 10, tzinfo=ZoneInfo("Asia/Shanghai")),) -... ], -... T.StructType([T.StructField("ts", T.TimestampType(), True)]), -... ) ->>> df.show() -+-------------------+ -| ts| -+-------------------+ -|2023-01-01 10:00:00| -|2023-01-01 03:00:00| -+-------------------+ - ->>> df.select(F.to_timestamp_ntz("ts")).show() -+--------------------+ -|to_timestamp_ntz(ts)| -+--------------------+ -| 2023-01-01 10:00:00| -| 2023-01-01 03:00:00| -+--------------------+ - ->>> df.select(F.to_timestamp_ntz("ts")).printSchema() -root - |-- to_timestamp_ntz(ts): timestamp_ntz (nullable = true) diff --git a/python/pysail/tests/spark/function/test_to_varchar.txt b/python/pysail/tests/spark/function/test_to_varchar.txt deleted file mode 100644 index 906217b1a..000000000 --- a/python/pysail/tests/spark/function/test_to_varchar.txt +++ /dev/null @@ -1,29 +0,0 @@ -Tests for `to_varchar`. - -`to_varchar` follows the same formatting behavior as the underlying formatter. - ->>> from pyspark.sql.functions import lit, to_varchar - ->>> df = spark.createDataFrame([(78.12,)], ["e"]) ->>> df.select(to_varchar(df.e, lit("$99.99")).alias("r")).collect() -[Row(r='$78.12')] - ->>> def c(expr): -... return spark.sql(f"SELECT {expr} AS r").collect()[0]["r"] ->>> def err(expr): -... try: -... spark.sql(f"SELECT {expr} AS r").collect() -... print("no error") -... except Exception: -... print("error") - ->>> c("to_varchar(454, '999')") -'454' ->>> c("to_varchar(78.12, '$99.99')") -'$78.12' ->>> c("to_varchar(-12454.8, '99G999D9S')") -'12,454.8-' ->>> err("to_varchar(454, 'PR999')") -error ->>> c("to_varchar(encode('Spark SQL', 'utf-8'), 'hex')") # doctest: +SAIL_ONLY -'537061726B2053514C' diff --git a/python/pysail/tests/spark/function/test_try_add.txt b/python/pysail/tests/spark/function/test_try_add.txt deleted file mode 100644 index a5ee04d4d..000000000 --- a/python/pysail/tests/spark/function/test_try_add.txt +++ /dev/null @@ -1,73 +0,0 @@ ->>> from pyspark.sql.types import StructType, StructField, IntegerType ->>> import pyspark.sql.functions as sf - ->>> schema = StructType([StructField("birth", IntegerType(), True),StructField("age", IntegerType(), True),]) ->>> df = spark.createDataFrame([(1982, 15),(1990, 2),(None, 10),(2147483647, 1),(-2147483648,-1),],schema=schema) ->>> df.select("*", sf.try_add("birth", "age").alias("birth_plus_age")).show() -+-----------+---+--------------+ -| birth|age|birth_plus_age| -+-----------+---+--------------+ -| 1982| 15| 1997| -| 1990| 2| 1992| -| NULL| 10| NULL| -| 2147483647| 1| NULL| -|-2147483648| -1| NULL| -+-----------+---+--------------+ - ->>> df = spark.sql(""" -... SELECT -... try_add(DATE '2015-09-30', 1) as d1, -... try_add(DATE '2000-01-01', 366) as d2, -... try_add(DATE '2021-01-01', 1) as d3, -... try_add(NULL, 100) as d4 -... """) ->>> df.show() -+----------+----------+----------+----+ -| d1| d2| d3| d4| -+----------+----------+----------+----+ -|2015-10-01|2001-01-01|2021-01-02|NULL| -+----------+----------+----------+----+ - ->>> df = spark.sql(""" -... SELECT -... try_add(DATE '2015-01-31', INTERVAL 1 MONTH) as d1, -... try_add(DATE '2020-02-29', INTERVAL 12 MONTH) as d2, -... try_add(NULL, INTERVAL 3 MONTH) as d3 -... """) ->>> df.show() -+----------+----------+----+ -| d1| d2| d3| -+----------+----------+----+ -|2015-02-28|2021-02-28|NULL| -+----------+----------+----+ - ->>> df = spark.sql(""" -... SELECT -... try_add(DATE '2000-07-31', INTERVAL -1 MONTH) as d1, -... try_add(DATE '2021-01-31', INTERVAL -1 MONTH) as d2 -... """) ->>> df.show() -+----------+----------+ -| d1| d2| -+----------+----------+ -|2000-06-30|2020-12-31| -+----------+----------+ - ->>> df = spark.sql(""" -... SELECT -... try_add(INTERVAL '1' YEAR, INTERVAL '2' YEAR) as result -... """) ->>> # INTERVAL '3' YEAR <-- sail ->>> # INTERVAL '3-0' YEAR TO MONTH <-- PySpark - ->>> spark.conf.set("spark.sql.session.timeZone", "UTC") ->>> df = spark.sql(""" -... SELECT -... try_add(TIMESTAMP '2021-01-01 00:00:00', INTERVAL 1 DAY) as result -... """) ->>> df.show(truncate=False) -+-------------------+ -|result | -+-------------------+ -|2021-01-02 00:00:00| -+-------------------+ diff --git a/python/pysail/tests/spark/function/test_try_divide.txt b/python/pysail/tests/spark/function/test_try_divide.txt deleted file mode 100644 index dfd644a30..000000000 --- a/python/pysail/tests/spark/function/test_try_divide.txt +++ /dev/null @@ -1,119 +0,0 @@ ->>> from pyspark.sql import functions as sf - ->>> df = spark.createDataFrame([(6000, 15), (1990, 2)], ["a", "b"]) ->>> df.select(sf.try_divide(df.a, df.b).alias('r')).show() -+-----+ -| r| -+-----+ -|400.0| -|995.0| -+-----+ - ->>> spark.range(1).select(sf.try_divide(sf.make_interval(days=sf.lit(1)), sf.lit(2))).show() -+-------------------------------------------------+ -|try_divide(make_interval(0, 0, 0, 1, 0, 0, 0), 2)| -+-------------------------------------------------+ -| 12 hours| -+-------------------------------------------------+ - ->>> spark.range(1).select( -... sf.try_divide(sf.make_interval(hours=sf.lit(10)), sf.lit(4)).alias("result") -... ).show(truncate=False) -+------------------+ -|result | -+------------------+ -|2 hours 30 minutes| -+------------------+ - ->>> spark.range(1).select( -... sf.try_divide(sf.make_interval(days=sf.lit(3), hours=sf.lit(12)), sf.lit(3)).alias("result") -... ).show(truncate=False) -+--------------+ -|result | -+--------------+ -|1 days 4 hours| -+--------------+ - ->>> spark.range(1).select( -... sf.try_divide(sf.make_interval(secs=sf.lit(10.5)), sf.lit(2)).alias("result") -... ).show(truncate=False) -+------------+ -|result | -+------------+ -|5.25 seconds| -+------------+ - ->>> spark.range(1).select( -... sf.try_divide(sf.make_interval(weeks=sf.lit(1)), sf.lit(2)).alias("result") -... ).show(truncate=False) -+---------------+ -|result | -+---------------+ -|3 days 12 hours| -+---------------+ - ->>> spark.range(1).select( -... sf.try_divide(sf.make_interval(days=sf.lit(1)), sf.lit(None).cast("int")).alias("result") -... ).show(truncate=False) -+------+ -|result| -+------+ -|NULL | -+------+ - ->>> spark.range(1).select( -... sf.try_divide(sf.make_interval(days=sf.lit(1)), sf.lit(0)).alias("result") -... ).show(truncate=False) -+------+ -|result| -+------+ -|NULL | -+------+ - ->>> spark.range(1).select( -... sf.try_divide(sf.make_interval(days=sf.lit(1)), sf.lit(2)).alias("result") -... ).show(truncate=False) -+--------+ -|result | -+--------+ -|12 hours| -+--------+ - ->>> spark.range(1).select( -... sf.try_divide(sf.make_interval(days=sf.lit(-1)), sf.lit(2)).alias("result") -... ).show(truncate=False) -+---------+ -|result | -+---------+ -|-12 hours| -+---------+ - ->>> spark.range(1).select( -... sf.try_divide(sf.make_interval(days=sf.lit(1), mins=sf.lit(90)), sf.lit(2)).alias("result") -... ).show(truncate=False) -+-------------------+ -|result | -+-------------------+ -|12 hours 45 minutes| -+-------------------+ - ->>> spark.sql("SELECT try_divide(make_ym_interval(1, 6), 2) AS result").show(truncate=False) -+----------------------------+ -|result | -+----------------------------+ -|INTERVAL '0-9' YEAR TO MONTH| -+----------------------------+ - ->>> spark.range(1).selectExpr("try_divide(make_ym_interval(1, 6), 2) AS result").show(truncate=False) -+----------------------------+ -|result | -+----------------------------+ -|INTERVAL '0-9' YEAR TO MONTH| -+----------------------------+ - ->>> spark.range(1).select(sf.try_divide(sf.make_interval(secs=sf.lit(1)), sf.lit(2))).show() -+-------------------------------------------------+ -|try_divide(make_interval(0, 0, 0, 0, 0, 0, 1), 2)| -+-------------------------------------------------+ -| 0.5 seconds| -+-------------------------------------------------+ diff --git a/python/pysail/tests/spark/function/test_try_mod.txt b/python/pysail/tests/spark/function/test_try_mod.txt deleted file mode 100644 index fdb952681..000000000 --- a/python/pysail/tests/spark/function/test_try_mod.txt +++ /dev/null @@ -1,188 +0,0 @@ ->>> spark.sql("SELECT try_mod(CAST(3.0 AS DECIMAL(2,1)), CAST(2.00 AS DECIMAL(3,2))) AS r").show() -+----+ -| r| -+----+ -|1.00| -+----+ - ->>> spark.sql("SELECT try_mod(CAST(12.34 AS DECIMAL(4,2)), CAST(5.67 AS DECIMAL(3,2))) AS r").show() -+----+ -| r| -+----+ -|1.00| -+----+ - ->>> spark.sql("SELECT try_mod(CAST(1.2 AS DECIMAL(2,1)), CAST(0.03 AS DECIMAL(3,2))) AS r").show() -+----+ -| r| -+----+ -|0.00| -+----+ - ->>> spark.sql("SELECT try_mod(CAST(1.23 AS DECIMAL(3,2)), CAST(0.00 AS DECIMAL(3,2))) AS r").show() -+----+ -| r| -+----+ -|NULL| -+----+ - ->>> spark.sql("SELECT try_mod(CAST(0.00 AS DECIMAL(3,2)), CAST(2.50 AS DECIMAL(3,2))) AS r").show() -+----+ -| r| -+----+ -|0.00| -+----+ - ->>> spark.sql("SELECT try_mod(CAST(1.23 AS DECIMAL(3,2)), CAST(9.99 AS DECIMAL(3,2))) AS r").show() -+----+ -| r| -+----+ -|1.23| -+----+ - ->>> spark.sql("SELECT try_mod(CAST(5.00 AS DECIMAL(3,2)), CAST(5.00 AS DECIMAL(3,2))) AS r").show() -+----+ -| r| -+----+ -|0.00| -+----+ - ->>> spark.sql("SELECT try_mod(CAST(9.9 AS DECIMAL(38,1)), CAST(0.000000000000000001 AS DECIMAL(18,18))) AS r").show() -+--------------------+ -| r| -+--------------------+ -|0.000000000000000000| -+--------------------+ - ->>> spark.sql("SELECT try_mod(CAST(99999999999999999999999999999999999999 AS DECIMAL(38,0)), CAST(10 AS DECIMAL(2,0))) AS r").show() -+---+ -| r| -+---+ -| 9| -+---+ - ->>> spark.sql("SELECT try_mod(3, CAST(2.00 AS DECIMAL(3,2))) AS r").show() -+----+ -| r| -+----+ -|1.00| -+----+ - ->>> spark.sql("SELECT try_mod(CAST(3.00 AS DECIMAL(3,2)), 2) AS r").show() -+----+ -| r| -+----+ -|1.00| -+----+ - ->>> spark.sql("SELECT try_mod(NULL, CAST(2.00 AS DECIMAL(3,2))) AS r").show() -+----+ -| r| -+----+ -|NULL| -+----+ - ->>> spark.sql("SELECT try_mod(CAST(3.00 AS DECIMAL(3,2)), NULL) AS r").show() -+----+ -| r| -+----+ -|NULL| -+----+ - ->>> spark.sql("SELECT try_mod(CAST(1.23 AS DECIMAL(3,2)), CAST(0.000000000001 AS DECIMAL(13,12))) AS r").show() -+--------------+ -| r| -+--------------+ -|0.000000000000| -+--------------+ - ->>> spark.sql("SELECT try_mod(CAST(10.00 AS DECIMAL(4,2)), CAST(2.50 AS DECIMAL(3,2))) AS r").show() -+----+ -| r| -+----+ -|0.00| -+----+ - ->>> spark.sql("""SELECT -... try_mod(CAST( 7.5 AS DECIMAL(3,1)), CAST( 2.0 AS DECIMAL(2,1))) AS p_p, -... try_mod(CAST(-7.5 AS DECIMAL(3,1)), CAST( 2.0 AS DECIMAL(2,1))) AS n_p, -... try_mod(CAST( 7.5 AS DECIMAL(3,1)), CAST(-2.0 AS DECIMAL(2,1))) AS p_n, -... try_mod(CAST(-7.5 AS DECIMAL(3,1)), CAST(-2.0 AS DECIMAL(2,1))) AS n_n -... """).show() -+---+----+---+----+ -|p_p| n_p|p_n| n_n| -+---+----+---+----+ -|1.5|-1.5|1.5|-1.5| -+---+----+---+----+ - ->>> spark.sql("SELECT try_mod(CAST(3.0 AS DECIMAL(2,1)), CAST(2.00 AS DECIMAL(3,2))) AS r").printSchema() -root - |-- r: decimal(3,2) (nullable = true) - ->>> spark.sql("SELECT try_mod(CAST(12.34 AS DECIMAL(4,2)), CAST(5.67 AS DECIMAL(3,2))) AS r").printSchema() -root - |-- r: decimal(3,2) (nullable = true) - ->>> spark.sql("SELECT try_mod(CAST(1.2 AS DECIMAL(2,1)), CAST(0.03 AS DECIMAL(3,2))) AS r").printSchema() -root - |-- r: decimal(3,2) (nullable = true) - ->>> spark.sql("SELECT try_mod(CAST(1.23 AS DECIMAL(3,2)), CAST(0.00 AS DECIMAL(3,2))) AS r").printSchema() -root - |-- r: decimal(3,2) (nullable = true) - ->>> spark.sql("SELECT try_mod(CAST(0.00 AS DECIMAL(3,2)), CAST(2.50 AS DECIMAL(3,2))) AS r").printSchema() -root - |-- r: decimal(3,2) (nullable = true) - ->>> spark.sql("SELECT try_mod(CAST(1.23 AS DECIMAL(3,2)), CAST(9.99 AS DECIMAL(3,2))) AS r").printSchema() -root - |-- r: decimal(3,2) (nullable = true) - ->>> spark.sql("SELECT try_mod(CAST(5.00 AS DECIMAL(3,2)), CAST(5.00 AS DECIMAL(3,2))) AS r").printSchema() -root - |-- r: decimal(3,2) (nullable = true) - ->>> spark.sql("SELECT try_mod(CAST(9.9 AS DECIMAL(38,1)), CAST(0.000000000000000001 AS DECIMAL(18,18))) AS r").printSchema() -root - |-- r: decimal(18,18) (nullable = true) - ->>> spark.sql("SELECT try_mod(CAST(99999999999999999999999999999999999999 AS DECIMAL(38,0)), CAST(10 AS DECIMAL(2,0))) AS r").printSchema() -root - |-- r: decimal(2,0) (nullable = true) - ->>> spark.sql("SELECT try_mod(3, CAST(2.00 AS DECIMAL(3,2))) AS r").printSchema() -root - |-- r: decimal(3,2) (nullable = true) - ->>> spark.sql("SELECT try_mod(CAST(3.00 AS DECIMAL(3,2)), 2) AS r").printSchema() -root - |-- r: decimal(3,2) (nullable = true) - ->>> spark.sql("SELECT try_mod(NULL, CAST(2.00 AS DECIMAL(3,2))) AS r").printSchema() -root - |-- r: decimal(3,2) (nullable = true) - ->>> spark.sql("SELECT try_mod(CAST(3.00 AS DECIMAL(3,2)), NULL) AS r").printSchema() -root - |-- r: decimal(3,2) (nullable = true) - ->>> spark.sql("SELECT try_mod(CAST(1.23 AS DECIMAL(3,2)), CAST(0.000000000001 AS DECIMAL(13,12))) AS r").printSchema() -root - |-- r: decimal(13,12) (nullable = true) - ->>> spark.sql("SELECT try_mod(CAST(10.00 AS DECIMAL(4,2)), CAST(2.50 AS DECIMAL(3,2))) AS r").printSchema() -root - |-- r: decimal(3,2) (nullable = true) - ->>> spark.sql("""SELECT -... try_mod(CAST( 7.5 AS DECIMAL(3,1)), CAST( 2.0 AS DECIMAL(2,1))) AS p_p, -... try_mod(CAST(-7.5 AS DECIMAL(3,1)), CAST( 2.0 AS DECIMAL(2,1))) AS n_p, -... try_mod(CAST( 7.5 AS DECIMAL(3,1)), CAST(-2.0 AS DECIMAL(2,1))) AS p_n, -... try_mod(CAST(-7.5 AS DECIMAL(3,1)), CAST(-2.0 AS DECIMAL(2,1))) AS n_n -... """).printSchema() -root - |-- p_p: decimal(2,1) (nullable = true) - |-- n_p: decimal(2,1) (nullable = true) - |-- p_n: decimal(2,1) (nullable = true) - |-- n_n: decimal(2,1) (nullable = true) diff --git a/python/pysail/tests/spark/function/test_try_multiply.txt b/python/pysail/tests/spark/function/test_try_multiply.txt deleted file mode 100644 index 6437ad75d..000000000 --- a/python/pysail/tests/spark/function/test_try_multiply.txt +++ /dev/null @@ -1,96 +0,0 @@ ->>> from pyspark.sql import functions as sf - ->>> spark.range(1).select( -... sf.try_multiply(sf.make_interval(days=sf.lit(1)), sf.lit(2)).alias("result") -... ).show(truncate=False) -+------+ -|result| -+------+ -|2 days| -+------+ - ->>> spark.range(1).select( -... sf.try_multiply(sf.make_interval(hours=sf.lit(10)), sf.lit(4)).alias("result") -... ).show(truncate=False) -+--------+ -|result | -+--------+ -|40 hours| -+--------+ - ->>> spark.range(1).select( -... sf.try_multiply(sf.make_interval(days=sf.lit(3), hours=sf.lit(12)), sf.lit(3)).alias("result") -... ).show(truncate=False) -+---------------+ -|result | -+---------------+ -|9 days 36 hours| -+---------------+ - ->>> spark.range(1).select( -... sf.try_multiply(sf.make_interval(secs=sf.lit(10.5)), sf.lit(2)).alias("result") -... ).show(truncate=False) -+----------+ -|result | -+----------+ -|21 seconds| -+----------+ - ->>> spark.range(1).select( -... sf.try_multiply(sf.make_interval(weeks=sf.lit(1)), sf.lit(2)).alias("result") -... ).show(truncate=False) -+-------+ -|result | -+-------+ -|14 days| -+-------+ - ->>> spark.range(1).select( -... sf.try_multiply(sf.make_interval(days=sf.lit(1)), sf.lit(None).cast("int")).alias("result") -... ).show(truncate=False) -+------+ -|result| -+------+ -|NULL | -+------+ - ->>> spark.range(1).select( -... sf.try_multiply(sf.make_interval(days=sf.lit(1)), sf.lit(0)).alias("result") -... ).show(truncate=False) -+---------+ -|result | -+---------+ -|0 seconds| -+---------+ - ->>> spark.range(1).select( -... sf.try_multiply(sf.make_interval(days=sf.lit(-1)), sf.lit(2)).alias("result") -... ).show(truncate=False) -+-------+ -|result | -+-------+ -|-2 days| -+-------+ - ->>> spark.range(1).select( -... sf.try_multiply(sf.make_interval(days=sf.lit(1), mins=sf.lit(90)), sf.lit(2)).alias("result") -... ).show(truncate=False) -+--------------+ -|result | -+--------------+ -|2 days 3 hours| -+--------------+ - ->>> spark.sql("SELECT try_multiply(make_ym_interval(1, 6), 2) AS result").show(truncate=False) -+----------------------------+ -|result | -+----------------------------+ -|INTERVAL '3-0' YEAR TO MONTH| -+----------------------------+ - ->>> spark.range(1).selectExpr("try_multiply(make_ym_interval(1, 6), 2) AS result").show(truncate=False) -+----------------------------+ -|result | -+----------------------------+ -|INTERVAL '3-0' YEAR TO MONTH| -+----------------------------+ diff --git a/python/pysail/tests/spark/function/test_try_parse_url.txt b/python/pysail/tests/spark/function/test_try_parse_url.txt deleted file mode 100644 index 7b0d720e6..000000000 --- a/python/pysail/tests/spark/function/test_try_parse_url.txt +++ /dev/null @@ -1,69 +0,0 @@ ->>> spark.sql("SELECT try_parse_url('https://example.com/a?x=1', 'QUERY', 'x') AS result, typeof(try_parse_url('https://example.com/a?x=1', 'QUERY', 'x')) AS type").show(truncate=False) -+------+------+ -|result|type | -+------+------+ -|1 |string| -+------+------+ - ->>> spark.sql("SELECT try_parse_url('www.example.com/path?x=1', 'HOST') AS result, typeof(try_parse_url('www.example.com/path?x=1', 'HOST')) AS type").show(truncate=False) -+------+------+ -|result|type | -+------+------+ -|NULL |string| -+------+------+ - ->>> spark.sql("SELECT try_parse_url('https://example.com/?a=1', 'QUERY', 'b') AS result, typeof(try_parse_url('https://example.com/?a=1', 'QUERY', 'b')) AS type").show(truncate=False) -+------+------+ -|result|type | -+------+------+ -|NULL |string| -+------+------+ - ->>> spark.sql("SELECT try_parse_url('https://example.com/path#frag', 'REF') AS result, typeof(try_parse_url('https://example.com/path#frag', 'REF')) AS type").show(truncate=False) -+------+------+ -|result|type | -+------+------+ -|frag |string| -+------+------+ - ->>> spark.sql("SELECT try_parse_url('ftp://user:pwd@ftp.example.com:21/files', 'USERINFO') AS result, typeof(try_parse_url('ftp://user:pwd@ftp.example.com:21/files', 'USERINFO')) AS type").show(truncate=False) -+--------+------+ -|result |type | -+--------+------+ -|user:pwd|string| -+--------+------+ - ->>> spark.sql("SELECT try_parse_url('http://[2001:db8::2]:8080/index.html?ok=1', 'HOST') AS result, typeof(try_parse_url('http://[2001:db8::2]:8080/index.html?ok=1', 'HOST')) AS type").show(truncate=False) -+-------------+------+ -|result |type | -+-------------+------+ -|[2001:db8::2]|string| -+-------------+------+ - ->>> spark.sql("SELECT try_parse_url('notaurl', 'HOST') AS result, typeof(try_parse_url('notaurl', 'HOST')) AS type").show(truncate=False) -+------+------+ -|result|type | -+------+------+ -|NULL |string| -+------+------+ - ->>> spark.sql("SELECT try_parse_url('https://example.com', 'PATH') AS result, typeof(try_parse_url('https://example.com', 'PATH')) AS type").show(truncate=False) -+------+------+ -|result|type | -+------+------+ -| |string| -+------+------+ - ->>> spark.sql("SELECT try_parse_url('https://example.com/a/b?x=1&y=2#frag', 'PROTOCOL') AS result, typeof(try_parse_url('https://example.com/a/b?x=1&y=2#frag', 'PROTOCOL')) AS type").show(truncate=False) -+------+------+ -|result|type | -+------+------+ -|https |string| -+------+------+ - ->>> spark.sql("SELECT try_parse_url('https://ex.com/?Tag=ok', 'QUERY', 'tag') AS result, typeof(try_parse_url('https://ex.com/?Tag=ok', 'QUERY', 'tag')) AS type").show(truncate=False) -+------+------+ -|result|type | -+------+------+ -|NULL |string| -+------+------+ \ No newline at end of file diff --git a/python/pysail/tests/spark/function/test_try_to_number.txt b/python/pysail/tests/spark/function/test_try_to_number.txt deleted file mode 100644 index ab978ce31..000000000 --- a/python/pysail/tests/spark/function/test_try_to_number.txt +++ /dev/null @@ -1,153 +0,0 @@ ->>> spark.sql("SELECT try_to_number('<-$12,345.67>', 'S$999,099.99PR') AS result, typeof(try_to_number('<-$12,345.67>', 'S$999,099.99PR')) AS type").show(truncate=False) -+--------+------------+ -|result |type | -+--------+------------+ -|12345.67|decimal(8,2)| -+--------+------------+ - ->>> spark.sql("SELECT try_to_number('<$1,212,345.67>', 'S$0,000,000.99PR') AS result, typeof(try_to_number('<$1,212,345.67>', 'S$0,000,000.99PR')) AS type").show(truncate=False) -+-----------+------------+ -|result |type | -+-----------+------------+ -|-1212345.67|decimal(9,2)| -+-----------+------------+ - ->>> spark.sql("SELECT try_to_number('$345', 'S$999,099.99') AS result, typeof(try_to_number('$345', 'S$999,099.99')) AS type").show(truncate=False) -+------+------------+ -|result|type | -+------+------------+ -|345.00|decimal(8,2)| -+------+------------+ - ->>> spark.sql("SELECT try_to_number('$045', 'S$999,099.99') AS result, typeof(try_to_number('$045', 'S$999,099.99')) AS type").show(truncate=False) -+------+------------+ -|result|type | -+------+------------+ -|45.00 |decimal(8,2)| -+------+------------+ - ->>> spark.sql("SELECT try_to_number('<1234>', '999999PR') AS result, typeof(try_to_number('<1234>', '999999PR')) AS type").show(truncate=False) -+------+------------+ -|result|type | -+------+------------+ -|-1234 |decimal(6,0)| -+------+------------+ - ->>> spark.sql("SELECT try_to_number('12,454.8-', '99,999.9S') AS result, typeof(try_to_number('12,454.8-', '99,999.9S')) AS type").show(truncate=False) -+--------+------------+ -|result |type | -+--------+------------+ -|-12454.8|decimal(6,1)| -+--------+------------+ - ->>> spark.sql("SELECT try_to_number('<-$123,456.32>', 'S$999,999.999999PR') AS result, typeof(try_to_number('<-$123,456.32>', 'S$999,999.999999PR')) AS type").show(truncate=False) -+-------------+-------------+ -|result |type | -+-------------+-------------+ -|123456.320000|decimal(12,6)| -+-------------+-------------+ - ->>> spark.sql("SELECT try_to_number('$123,456.32', 'MI$999,999.99S') AS result, typeof(try_to_number('$123,456.32', 'MI$999,999.99S')) AS type").show(truncate=False) -+---------+------------+ -|result |type | -+---------+------------+ -|123456.32|decimal(8,2)| -+---------+------------+ - ->>> spark.sql("SELECT try_to_number('$123,456.32-', 'MI$999,999.99S') AS result, typeof(try_to_number('$123,456.32-', 'MI$999,999.99S')) AS type").show(truncate=False) -+----------+------------+ -|result |type | -+----------+------------+ -|-123456.32|decimal(8,2)| -+----------+------------+ - ->>> spark.sql("SELECT try_to_number('-$123,456.32-', 'MI$999,999.99S') AS result, typeof(try_to_number('-$123,456.32-', 'MI$999,999.99S')) AS type").show(truncate=False) -+---------+------------+ -|result |type | -+---------+------------+ -|123456.32|decimal(8,2)| -+---------+------------+ - ->>> spark.sql("SELECT try_to_number('-$123,456.32+', 'MI$999,999.99S') AS result, typeof(try_to_number('-$123,456.32+', 'MI$999,999.99S')) AS type").show(truncate=False) -+----------+------------+ -|result |type | -+----------+------------+ -|-123456.32|decimal(8,2)| -+----------+------------+ - ->>> spark.sql("SELECT try_to_number('-$123,456.32', 'MI$999,999.99S') AS result, typeof(try_to_number('-$123,456.32', 'MI$999,999.99S')) AS type").show(truncate=False) -+----------+------------+ -|result |type | -+----------+------------+ -|-123456.32|decimal(8,2)| -+----------+------------+ - ->>> spark.sql("SELECT try_to_number('045', 'S$999,099.99') AS result, typeof(try_to_number('045', 'S$999,099.99')) AS type").show(truncate=False) -+------+------------+ -|result|type | -+------+------------+ -|NULL |decimal(8,2)| -+------+------------+ - ->>> spark.sql("SELECT try_to_number('1234', '999999PR') AS result, typeof(try_to_number('1234', '999999PR')) AS type").show(truncate=False) -+------+------------+ -|result|type | -+------+------------+ -|1234 |decimal(6,0)| -+------+------------+ - ->>> spark.sql("SELECT try_to_number('<1234>', '999999PR') AS result, typeof(try_to_number('>1234>', '999999PR')) AS type").show(truncate=False) -+------+------------+ -|result|type | -+------+------------+ -|-1234 |decimal(6,0)| -+------+------------+ - ->>> spark.sql("SELECT try_to_number('1234>', '999999PR') AS result, typeof(try_to_number('1234>', '999999PR')) AS type").show(truncate=False) -+------+------------+ -|result|type | -+------+------------+ -|NULL |decimal(6,0)| -+------+------------+ - ->>> spark.sql("SELECT try_to_number('12454.8-', '99,999.9S') AS result, typeof(try_to_number('12454.8-', '99,999.9S')) AS type").show(truncate=False) -+------+------------+ -|result|type | -+------+------------+ -|NULL |decimal(6,1)| -+------+------------+ - ->>> spark.sql("SELECT try_to_number('<$1212345.67>', 'S$0,000,000.99PR') AS result, typeof(try_to_number('<$1,212,345.67>', 'S$0,000,000.99PR')) AS type").show(truncate=False) -+------+------------+ -|result|type | -+------+------------+ -|NULL |decimal(9,2)| -+------+------------+ - ->>> spark.sql("SELECT try_to_number('-$123,456.32', 'S$999,999.999999PR') AS result, typeof(try_to_number('-$123,456.32', 'S$999,999.999999PR')) AS type").show(truncate=False) -+--------------+-------------+ -|result |type | -+--------------+-------------+ -|-123456.320000|decimal(12,6)| -+--------------+-------------+ - ->>> spark.sql("SELECT try_to_number('-123,456.32-', 'MI$999,999.99S') AS result, typeof(try_to_number('-123,456.32-', 'MI$999,999.99S')) AS type").show(truncate=False) -+------+------------+ -|result|type | -+------+------------+ -|NULL |decimal(8,2)| -+------+------------+ - ->>> spark.sql("SELECT try_to_number('+<$123,456.32+', 'MI$999,999.99S') AS result, typeof(try_to_number('+<$123,456.32+', 'MI$999,999.99S')) AS type").show(truncate=False) -+------+------------+ -|result|type | -+------+------------+ -|NULL |decimal(8,2)| -+------+------------+ - ->>> spark.sql("SELECT try_to_number('-123,456.32', 'MI$999,999.99S') AS result, typeof(try_to_number('-123,456.32', 'MI$999,999.99S')) AS type").show(truncate=False) -+------+------------+ -|result|type | -+------+------------+ -|NULL |decimal(8,2)| -+------+------------+ diff --git a/python/pysail/tests/spark/function/test_try_to_timestamp.txt b/python/pysail/tests/spark/function/test_try_to_timestamp.txt deleted file mode 100644 index 93e3001eb..000000000 --- a/python/pysail/tests/spark/function/test_try_to_timestamp.txt +++ /dev/null @@ -1,48 +0,0 @@ ->>> from datetime import datetime ->>> from zoneinfo import ZoneInfo ->>> from pyspark.sql import functions as F, types as T ->>> ->>> spark.conf.set("spark.sql.session.timeZone", "Europe/Amsterdam") ->>> ->>> df = spark.createDataFrame( -... [ -... (datetime(2023, 1, 1, 10, tzinfo=ZoneInfo("Europe/Amsterdam")),), -... (datetime(2023, 1, 1, 10, tzinfo=ZoneInfo("Asia/Shanghai")),) -... ], -... T.StructType([T.StructField("ts", T.TimestampType(), True)]), -... ) ->>> df.show() -+-------------------+ -| ts| -+-------------------+ -|2023-01-01 10:00:00| -|2023-01-01 03:00:00| -+-------------------+ - ->>> spark.conf.set("spark.sql.timestampType", "TIMESTAMP_LTZ") ->>> ->>> df.select(F.try_to_timestamp("ts")).show() -+--------------------+ -|try_to_timestamp(ts)| -+--------------------+ -| 2023-01-01 10:00:00| -| 2023-01-01 03:00:00| -+--------------------+ - ->>> df.select(F.try_to_timestamp("ts")).printSchema() -root - |-- try_to_timestamp(ts): timestamp (nullable = true) - ->>> spark.conf.set("spark.sql.timestampType", "TIMESTAMP_NTZ") ->>> ->>> df.select(F.try_to_timestamp("ts")).show() -+--------------------+ -|try_to_timestamp(ts)| -+--------------------+ -| 2023-01-01 10:00:00| -| 2023-01-01 03:00:00| -+--------------------+ - ->>> df.select(F.try_to_timestamp("ts")).printSchema() -root - |-- try_to_timestamp(ts): timestamp_ntz (nullable = true) diff --git a/python/pysail/tests/spark/function/test_url_decode.txt b/python/pysail/tests/spark/function/test_url_decode.txt deleted file mode 100644 index 5fb84535f..000000000 --- a/python/pysail/tests/spark/function/test_url_decode.txt +++ /dev/null @@ -1,20 +0,0 @@ ->>> spark.sql("select url_decode('https%3A%2F%2Fspark.apache.org')").show(truncate=False) -+------------------------------------------+ -|url_decode(https%3A%2F%2Fspark.apache.org)| -+------------------------------------------+ -|https://spark.apache.org | -+------------------------------------------+ - ->>> spark.sql("select url_decode('inva lid://user:pass@host/file\\;param?query\\;p2')").show(truncate=False) -+---------------------------------------------------------+ -|url_decode(inva lid://user:pass@host/file;param?query;p2)| -+---------------------------------------------------------+ -|inva lid://user:pass@host/file;param?query;p2 | -+---------------------------------------------------------+ - ->>> spark.sql("select url_decode(null)").show(truncate=False) -+----------------+ -|url_decode(NULL)| -+----------------+ -|NULL | -+----------------+ diff --git a/python/pysail/tests/spark/function/test_url_encode.txt b/python/pysail/tests/spark/function/test_url_encode.txt deleted file mode 100644 index c14ce7867..000000000 --- a/python/pysail/tests/spark/function/test_url_encode.txt +++ /dev/null @@ -1,21 +0,0 @@ ->>> spark.sql("select url_encode('https://spark.apache.org')").show(truncate=False) -+------------------------------------+ -|url_encode(https://spark.apache.org)| -+------------------------------------+ -|https%3A%2F%2Fspark.apache.org | -+------------------------------------+ - ->>> spark.sql("select url_encode('inva lid://user:pass@host/file\\;param?query\\;p2')").show(truncate=False) -+---------------------------------------------------------------+ -|url_encode(inva lid://user:pass@host/file;param?query;p2) | -+---------------------------------------------------------------+ -|inva+lid%3A%2F%2Fuser%3Apass%40host%2Ffile%3Bparam%3Fquery%3Bp2| -+---------------------------------------------------------------+ - ->>> spark.sql("select url_encode(null)").show(truncate=False) -+----------------+ -|url_encode(NULL)| -+----------------+ -|NULL | -+----------------+ - diff --git a/python/pysail/tests/spark/function/test_width_bucket.txt b/python/pysail/tests/spark/function/test_width_bucket.txt deleted file mode 100644 index 6e080605d..000000000 --- a/python/pysail/tests/spark/function/test_width_bucket.txt +++ /dev/null @@ -1,48 +0,0 @@ ->>> from decimal import Decimal ->>> from pyspark.sql.types import StructType, StructField, DecimalType, IntegerType ->>> from pyspark.sql.functions import width_bucket ->>> schema = StructType([ -... StructField("v", DecimalType(10, 2)), -... StructField("min", DecimalType(10, 2)), -... StructField("max", DecimalType(10, 2)), -... StructField("n", IntegerType()), -... ]) ->>> rows = [ -... (Decimal("0.00"), Decimal("0.00"), Decimal("10.00"), 5), # v==min → 1 -... (Decimal("2.00"), Decimal("0.00"), Decimal("10.00"), 5), # width=2 → 2 -... (Decimal("10.00"), Decimal("0.00"), Decimal("10.00"), 5), # v==max → 5 -... (Decimal("10.01"), Decimal("0.00"), Decimal("10.00"), 5), # v>max → 6 -... (Decimal("-0.01"), Decimal("0.00"), Decimal("10.00"), 5), # v>> df = spark.createDataFrame(rows, schema) ->>> df.select(width_bucket("v", "min", "max", "n")).show() -+----------------------------+ -|width_bucket(v, min, max, n)| -+----------------------------+ -| 1| -| 2| -| 6| -| 6| -| 0| -+----------------------------+ - ->>> spark.sql("SELECT width_bucket(0.0, 10.0, 0.0, 5)").show() -+-------------------------------+ -|width_bucket(0.0, 10.0, 0.0, 5)| -+-------------------------------+ -| 6| -+-------------------------------+ - ->>> spark.sql("SELECT width_bucket(10.0, 0.0, 10.0, 5)").show() -+--------------------------------+ -|width_bucket(10.0, 0.0, 10.0, 5)| -+--------------------------------+ -| 6| -+--------------------------------+ - ->>> spark.sql("SELECT width_bucket(10.0, 0.0, 0.0, 5)").show() -+-------------------------------+ -|width_bucket(10.0, 0.0, 0.0, 5)| -+-------------------------------+ -| NULL| -+-------------------------------+ diff --git a/python/pysail/tests/spark/function/test_zeroifnull.txt b/python/pysail/tests/spark/function/test_zeroifnull.txt deleted file mode 100644 index bb6824287..000000000 --- a/python/pysail/tests/spark/function/test_zeroifnull.txt +++ /dev/null @@ -1,97 +0,0 @@ ->>> spark.sql("SELECT zeroifnull(NULL), typeof(zeroifnull(NULL))").show() -+----------------+------------------------+ -|zeroifnull(NULL)|typeof(zeroifnull(NULL))| -+----------------+------------------------+ -| 0| int| -+----------------+------------------------+ - ->>> spark.sql("SELECT zeroifnull(2), typeof(zeroifnull(2))").show() -+-------------+---------------------+ -|zeroifnull(2)|typeof(zeroifnull(2))| -+-------------+---------------------+ -| 2| int| -+-------------+---------------------+ - ->>> spark.sql("SELECT zeroifnull(CAST(NULL AS TINYINT)), typeof(zeroifnull(CAST(NULL AS TINYINT)))").show() -+---------------------------------+-----------------------------------------+ -|zeroifnull(CAST(NULL AS TINYINT))|typeof(zeroifnull(CAST(NULL AS TINYINT)))| -+---------------------------------+-----------------------------------------+ -| 0| tinyint| -+---------------------------------+-----------------------------------------+ - ->>> spark.sql("SELECT zeroifnull(CAST(5 AS TINYINT)), typeof(zeroifnull(CAST(5 AS TINYINT)))").show() -+------------------------------+--------------------------------------+ -|zeroifnull(CAST(5 AS TINYINT))|typeof(zeroifnull(CAST(5 AS TINYINT)))| -+------------------------------+--------------------------------------+ -| 5| tinyint| -+------------------------------+--------------------------------------+ - ->>> spark.sql("SELECT zeroifnull(CAST(NULL AS SMALLINT)), typeof(zeroifnull(CAST(NULL AS SMALLINT)))").show() -+----------------------------------+------------------------------------------+ -|zeroifnull(CAST(NULL AS SMALLINT))|typeof(zeroifnull(CAST(NULL AS SMALLINT)))| -+----------------------------------+------------------------------------------+ -| 0| smallint| -+----------------------------------+------------------------------------------+ - ->>> spark.sql("SELECT zeroifnull(CAST(NULL AS BIGINT)), typeof(zeroifnull(CAST(NULL AS BIGINT)))").show() -+--------------------------------+----------------------------------------+ -|zeroifnull(CAST(NULL AS BIGINT))|typeof(zeroifnull(CAST(NULL AS BIGINT)))| -+--------------------------------+----------------------------------------+ -| 0| bigint| -+--------------------------------+----------------------------------------+ - ->>> spark.sql("SELECT zeroifnull(CAST(NULL AS FLOAT)), typeof(zeroifnull(CAST(NULL AS FLOAT)))").show() -+-------------------------------+---------------------------------------+ -|zeroifnull(CAST(NULL AS FLOAT))|typeof(zeroifnull(CAST(NULL AS FLOAT)))| -+-------------------------------+---------------------------------------+ -| 0.0| float| -+-------------------------------+---------------------------------------+ - ->>> spark.sql("SELECT zeroifnull(CAST(3.14 AS FLOAT)), typeof(zeroifnull(CAST(3.14 AS FLOAT)))").show() -+-------------------------------+---------------------------------------+ -|zeroifnull(CAST(3.14 AS FLOAT))|typeof(zeroifnull(CAST(3.14 AS FLOAT)))| -+-------------------------------+---------------------------------------+ -| 3.14| float| -+-------------------------------+---------------------------------------+ - ->>> spark.sql("SELECT zeroifnull(CAST(NULL AS DOUBLE)), typeof(zeroifnull(CAST(NULL AS DOUBLE)))").show() -+--------------------------------+----------------------------------------+ -|zeroifnull(CAST(NULL AS DOUBLE))|typeof(zeroifnull(CAST(NULL AS DOUBLE)))| -+--------------------------------+----------------------------------------+ -| 0.0| double| -+--------------------------------+----------------------------------------+ - ->>> spark.sql("SELECT zeroifnull(CAST(2.718 AS DOUBLE)), typeof(zeroifnull(CAST(2.718 AS DOUBLE)))").show() -+---------------------------------+-----------------------------------------+ -|zeroifnull(CAST(2.718 AS DOUBLE))|typeof(zeroifnull(CAST(2.718 AS DOUBLE)))| -+---------------------------------+-----------------------------------------+ -| 2.718| double| -+---------------------------------+-----------------------------------------+ - ->>> spark.sql("SELECT zeroifnull(CAST(NULL AS DECIMAL(10,2))), typeof(zeroifnull(CAST(NULL AS DECIMAL(10,2))))").show() -+---------------------------------------+-----------------------------------------------+ -|zeroifnull(CAST(NULL AS DECIMAL(10,2)))|typeof(zeroifnull(CAST(NULL AS DECIMAL(10,2))))| -+---------------------------------------+-----------------------------------------------+ -| 0.0| double| -+---------------------------------------+-----------------------------------------------+ - ->>> spark.sql("SELECT zeroifnull(CAST(42.99 AS DECIMAL(10,2))), typeof(zeroifnull(CAST(42.99 AS DECIMAL(10,2))))").show() -+----------------------------------------+------------------------------------------------+ -|zeroifnull(CAST(42.99 AS DECIMAL(10,2)))|typeof(zeroifnull(CAST(42.99 AS DECIMAL(10,2))))| -+----------------------------------------+------------------------------------------------+ -| 42.99| double| -+----------------------------------------+------------------------------------------------+ - ->>> spark.sql("SELECT zeroifnull(0), typeof(zeroifnull(0))").show() -+-------------+---------------------+ -|zeroifnull(0)|typeof(zeroifnull(0))| -+-------------+---------------------+ -| 0| int| -+-------------+---------------------+ - ->>> spark.sql("SELECT zeroifnull(-5), typeof(zeroifnull(-5))").show() -+-----------------+-------------------------+ -|zeroifnull((- 5))|typeof(zeroifnull((- 5)))| -+-----------------+-------------------------+ -| -5| int| -+-----------------+-------------------------+ From 5abb58353d5e3a12e712c7606d89b0b2f86b9d52 Mon Sep 17 00:00:00 2001 From: Xiuli Wei <542472913@qq.com> Date: Tue, 28 Jul 2026 18:29:06 +0800 Subject: [PATCH 090/131] feat: support Aliyun OSS object storage (#2283) --- crates/sail-data-source/src/url.rs | 5 ++ crates/sail-object-store/src/registry.rs | 46 ++++++++++++++- crates/sail-object-store/src/s3.rs | 74 +++++++++++++++++++++++- docs/guide/storage/index.md | 3 +- docs/guide/storage/s3.md | 31 ++++++++-- 5 files changed, 150 insertions(+), 9 deletions(-) diff --git a/crates/sail-data-source/src/url.rs b/crates/sail-data-source/src/url.rs index 36dc17f56..3f54e4580 100644 --- a/crates/sail-data-source/src/url.rs +++ b/crates/sail-data-source/src/url.rs @@ -818,6 +818,11 @@ mod tests { &[("https://example.com/path/?foo#bar", Some("**/*.txt"))], ); test("s3://bucket/path", &[("s3://bucket/path", None)]); + test("oss://bucket/path", &[("oss://bucket/path", None)]); + test( + "oss://bucket/path/*.txt", + &[("oss://bucket/path/", Some("*.txt"))], + ); test( "s3://bucket/path/*.txt", &[("s3://bucket/path/", Some("*.txt"))], diff --git a/crates/sail-object-store/src/registry.rs b/crates/sail-object-store/src/registry.rs index 05ec4499c..1fec6a974 100644 --- a/crates/sail-object-store/src/registry.rs +++ b/crates/sail-object-store/src/registry.rs @@ -135,6 +135,22 @@ fn get_dynamic_object_store(url: &Url) -> object_store::Result { + let url = url.clone(); + let store = LazyObjectStore::new(move || { + let url = url.clone(); + async move { get_s3_object_store(&url).await } + }); + Arc::new(store) + } + _ if is_aliyun_oss_url(url) => { + let url = url.clone(); + let store = LazyObjectStore::new(move || { + let url = url.clone(); + async move { get_s3_object_store(&url).await } + }); + Arc::new(store) + } _ => { let (scheme, _path) = ObjectStoreScheme::parse(url)?; let store: Arc = match scheme { @@ -187,6 +203,21 @@ fn get_dynamic_object_store(url: &Url) -> object_store::Result bool { + if !matches!(url.scheme(), "http" | "https") { + return false; + } + let Some(host) = url.host_str() else { + return false; + }; + let labels = host.split('.').collect::>(); + matches!( + labels.as_slice(), + [_, "s3", region, "aliyuncs", "com"] | ["s3", region, "aliyuncs", "com"] + if region.starts_with("oss-") + ) +} + // The following implementations are basic for now just to get preliminary functionality. pub async fn get_azure_object_store(url: &Url) -> object_store::Result { MicrosoftAzureBuilder::from_env() @@ -224,7 +255,7 @@ mod tests { use tokio::runtime::Handle; use url::Url; - use super::{DynamicObjectStoreRegistry, ObjectStoreKey}; + use super::{DynamicObjectStoreRegistry, ObjectStoreKey, is_aliyun_oss_url}; #[test] fn object_store_key_separates_session_fingerprints() { @@ -266,4 +297,17 @@ mod tests { let default_store = registry.get_store(&url).unwrap(); assert!(Arc::ptr_eq(&fallback, &default_store)); } + + #[test] + fn aliyun_oss_url_detection_matches_s3_compatible_endpoints() { + assert!(is_aliyun_oss_url( + &Url::parse("https://bucket.s3.oss-cn-hangzhou.aliyuncs.com/path").unwrap() + )); + assert!(is_aliyun_oss_url( + &Url::parse("https://s3.oss-cn-hangzhou.aliyuncs.com/bucket/path").unwrap() + )); + assert!(!is_aliyun_oss_url( + &Url::parse("https://bucket.oss-cn-hangzhou.aliyuncs.com/path").unwrap() + )); + } } diff --git a/crates/sail-object-store/src/s3.rs b/crates/sail-object-store/src/s3.rs index b9292f1d8..ad0037e31 100644 --- a/crates/sail-object-store/src/s3.rs +++ b/crates/sail-object-store/src/s3.rs @@ -161,7 +161,7 @@ pub fn parse_s3_url( ); match scheme { - "s3" | "s3a" => { + "s3" | "s3a" | "oss" => { builder = builder.with_bucket_name(host); if let Some(bucket_prefix) = host.strip_suffix("--x-s3") && let Some(_bucket_az) = bucket_prefix.rsplit_once("--") @@ -173,6 +173,7 @@ pub fn parse_s3_url( if scheme == "http" { builder = builder.with_allow_http(true); } + let endpoint = || url[..url::Position::BeforePath].to_string(); match host.split('.').collect::>()[..] { // Support for path-style continues for buckets created on/before Sept. 30, 2020: // https://aws.amazon.com/blogs/aws/amazon-s3-path-deprecation-plan-the-rest-of-the-story/ @@ -219,6 +220,20 @@ pub fn parse_s3_url( builder = builder.with_bucket_name(bucket); } } + [bucket, "s3", region, "aliyuncs", "com"] if region.starts_with("oss-") => { + builder = builder.with_bucket_name(bucket); + builder = builder.with_region(region.trim_start_matches("oss-")); + builder = builder.with_endpoint(endpoint()); + builder = builder.with_virtual_hosted_style_request(true); + } + ["s3", region, "aliyuncs", "com"] if region.starts_with("oss-") => { + builder = builder.with_region(region.trim_start_matches("oss-")); + builder = builder.with_endpoint(endpoint()); + builder = builder.with_virtual_hosted_style_request(false); + if let Some(bucket) = first_path_segment { + builder = builder.with_bucket_name(bucket); + } + } [bucket, _s3express_zone_id, region, "amazonaws", "com"] => { builder = builder.with_bucket_name(bucket); builder = builder.with_region(region); @@ -246,3 +261,60 @@ pub fn parse_s3_url( Ok(builder) } + +#[cfg(test)] +#[expect(clippy::unwrap_used)] +mod tests { + use super::*; + + #[test] + fn parse_oss_url_sets_bucket() { + let url = Url::parse("oss://bucket/path/to/data").unwrap(); + let builder = parse_s3_url(AmazonS3Builder::from_env(), &url).unwrap(); + + assert_eq!( + builder.get_config_value(&AmazonS3ConfigKey::Bucket), + Some("bucket".to_string()) + ); + } + + #[test] + fn parse_aliyun_oss_virtual_hosted_endpoint() { + let url = + Url::parse("https://bucket.s3.oss-cn-hangzhou.aliyuncs.com:9443/path/to/data").unwrap(); + let builder = parse_s3_url(AmazonS3Builder::from_env(), &url).unwrap(); + + assert_eq!( + builder.get_config_value(&AmazonS3ConfigKey::Bucket), + Some("bucket".to_string()) + ); + assert_eq!( + builder.get_config_value(&AmazonS3ConfigKey::Region), + Some("cn-hangzhou".to_string()) + ); + assert_eq!( + builder.get_config_value(&AmazonS3ConfigKey::Endpoint), + Some("https://bucket.s3.oss-cn-hangzhou.aliyuncs.com:9443".to_string()) + ); + } + + #[test] + fn parse_aliyun_oss_path_style_endpoint() { + let url = + Url::parse("https://s3.oss-cn-hangzhou.aliyuncs.com:9443/bucket/path/to/data").unwrap(); + let builder = parse_s3_url(AmazonS3Builder::from_env(), &url).unwrap(); + + assert_eq!( + builder.get_config_value(&AmazonS3ConfigKey::Bucket), + Some("bucket".to_string()) + ); + assert_eq!( + builder.get_config_value(&AmazonS3ConfigKey::Region), + Some("cn-hangzhou".to_string()) + ); + assert_eq!( + builder.get_config_value(&AmazonS3ConfigKey::Endpoint), + Some("https://s3.oss-cn-hangzhou.aliyuncs.com:9443".to_string()) + ); + } +} diff --git a/docs/guide/storage/index.md b/docs/guide/storage/index.md index 605ecd3df..1223fc3a2 100644 --- a/docs/guide/storage/index.md +++ b/docs/guide/storage/index.md @@ -45,6 +45,7 @@ Here is a summary of the supported (:white_check_mark:) and unsupported (:x:) st | [Memory](./memory) | :white_check_mark: | :white_check_mark: | | [AWS S3](./s3) | :white_check_mark: | :white_check_mark: | | [Cloudflare R2](./s3) | :white_check_mark: | :white_check_mark: | +| [Aliyun OSS](./s3) | :white_check_mark: | :white_check_mark: | | [Azure Data Lake Storage (ADLS)](./azure) | :white_check_mark: | :white_check_mark: | | [Azure Blob Storage](./azure) | :white_check_mark: | :white_check_mark: | | [Google Cloud Storage](./gcs) | :white_check_mark: | :white_check_mark: | @@ -57,7 +58,7 @@ Here is a summary of the supported (:white_check_mark:) and unsupported (:x:) st Some HTTPS URLs are automatically recognized as cloud storage: -- **S3**: URLs containing `amazonaws.com` or `r2.cloudflarestorage.com`. +- **S3**: URLs containing `amazonaws.com`, `r2.cloudflarestorage.com`, or Aliyun OSS S3-compatible endpoints such as `.s3.oss-.aliyuncs.com` and `s3.oss-.aliyuncs.com`. - **Azure**: URLs containing `dfs.core.windows.net`, `blob.core.windows.net`, `dfs.fabric.microsoft.com`, or `blob.fabric.microsoft.com`. These URLs will use the appropriate cloud storage backend instead of the generic HTTP store. diff --git a/docs/guide/storage/s3.md b/docs/guide/storage/s3.md index 93058b09c..c71bbf360 100644 --- a/docs/guide/storage/s3.md +++ b/docs/guide/storage/s3.md @@ -5,7 +5,7 @@ rank: 2 # AWS S3 -Sail supports reading and writing data to AWS S3 and S3-compatible object storage services (such as [MinIO](https://min.io/) or Cloudflare R2) using the `s3://`, `s3a://`, or `https://` URI schemes. +Sail supports reading and writing data to AWS S3 and S3-compatible object storage services (such as [MinIO](https://min.io/) or Cloudflare R2) using the `s3://`, `s3a://`, `oss://`, or `https://` URI schemes. For example, `s3://bucket/path/to/data` refers to the `path/to/data` path in the `bucket` bucket. Sail determines whether the path refers to a single object or a key prefix. @@ -17,7 +17,7 @@ Sail supports the following URL formats for S3 and S3-compatible object storage
  • - s3 and s3a protocols + s3, s3a, and oss protocols
    • @@ -25,6 +25,9 @@ Sail supports the following URL formats for S3 and S3-compatible object storage
      • +
    • + +
  • @@ -59,6 +62,23 @@ Sail supports the following URL formats for S3 and S3-compatible object storage
+
  • + HTTPS endpoints for Aliyun OSS S3-compatible requests +
      +
    • + +
      • +
    • + +
      • +
    +
  • HTTPS endpoint for AWS S3 Express One Zone
      @@ -193,16 +213,15 @@ export AWS_VIRTUAL_HOSTED_STYLE_REQUEST="true" You can use either long-lived or temporary credentials for Aliyun OSS. -Aliyun OSS only supports virtual-hosted-style requests, so you must include the bucket name in the endpoint URL. The region in the endpoint URL must match the region specified in the `AWS_REGION` environment variable. +Aliyun OSS S3-compatible endpoints support virtual-hosted-style and path-style URL formats. When you configure `AWS_ENDPOINT`, set `AWS_VIRTUAL_HOSTED_STYLE_REQUEST` to match the endpoint format. The region in the endpoint URL must match the region specified in the `AWS_REGION` environment variable. -The S3-compatible endpoint supports read, write (including multipart upload), list, and delete operations for objects stored in Aliyun OSS. Access the data with the `s3://` URI scheme instead of `oss://`. +The S3-compatible endpoint supports read, write (including multipart upload), list, and delete operations for objects stored in Aliyun OSS. Access the data with the `oss://`, `s3://`, or HTTPS URI formats. ::: warning -There are a few limitations when using the `s3://` URI scheme to access Aliyun OSS. +There are a few limitations when using Aliyun OSS S3-compatible access. - Credential rotation is not supported. -- Because the bucket name is included in the endpoint URL, each Spark Connect server instance can access only one bucket, and you cannot switch buckets at runtime. - The S3-compatible endpoint does not support headers such as `If-None-Match` for object uploads. Some lakehouse formats, such as Delta Lake, may not work because they rely on these headers for atomic operations. ::: From 0c8c90ad41932a941dcea77801969f00e7f55a37 Mon Sep 17 00:00:00 2001 From: Xiaolong Liang Date: Tue, 28 Jul 2026 21:26:47 +0800 Subject: [PATCH 091/131] chore: bump datafusion to 54.1.0 (#2293) --- Cargo.lock | 168 +++++++++--------- Cargo.toml | 26 +-- .../proto/sail/plan/physical.proto | 1 + crates/sail-execution/src/proto/codec.rs | 45 +++++ .../src/scalar/array/spark_array_compact.rs | 3 +- .../spark/__snapshots__/test_tpcds.plan.yaml | 55 +++--- .../features/array/array_compact.feature | 15 +- .../features/math/width_bucket.feature | 11 +- 8 files changed, 185 insertions(+), 139 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 729c5742a..260af0fee 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -144,7 +144,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -155,7 +155,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -1884,9 +1884,9 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.18" +version = "0.9.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" dependencies = [ "crossbeam-utils", ] @@ -2044,9 +2044,9 @@ dependencies = [ [[package]] name = "datafusion" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "997a31e15872606a49478e670c58302094c97cb96abb0a7d60720f8e92170040" +checksum = "754ef4e8f073922a26f5b23133b9db4829342362b09be0bc94309cf261c2f098" dependencies = [ "arrow", "arrow-schema", @@ -2099,9 +2099,9 @@ dependencies = [ [[package]] name = "datafusion-catalog" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7dd61161508f8f5fa1107774ea687bd753c22d83a32eebf963549f89de14139" +checksum = "06afd1e38dd27bbb1258685a1fc6524df6aff4e07b25b393a47de59635178d99" dependencies = [ "arrow", "async-trait", @@ -2124,9 +2124,9 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "897c70f871277f9ce99aa38347be0d679bbe3e617156c4d2a8378cec8a2a0891" +checksum = "f0668fb32c12065ec242be0e5b4bc62bd7a06a0be3ecd83791ef877e4be67e02" dependencies = [ "arrow", "async-trait", @@ -2147,9 +2147,9 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "121c9ded5d87d9172319e006f2afdb9928d72dbacd6a90a458d8acb1e3b43a65" +checksum = "ca43b263cdff57042cfa8fb817fb3469f4878933380dccff25f5e793580abbf9" dependencies = [ "arrow", "arrow-ipc", @@ -2173,9 +2173,9 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "981b9dae74f78ee3d9f714fb49b01919eab975461b56149510c3ba9ea11287d1" +checksum = "05f0ba2b864792bdca4d76c59a1de0ab6e1b61946596b9936888dbd6360035f2" dependencies = [ "futures", "log", @@ -2184,9 +2184,9 @@ dependencies = [ [[package]] name = "datafusion-datasource" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffd7d295b2ec7c00d8a56562f41ed41062cf0af75549ed891c12a0a09eddfefe" +checksum = "b840a8bce0bcbf5afad02946d438591e7c373f7afccaf3d874c04485772514dd" dependencies = [ "arrow", "async-compression", @@ -2220,9 +2220,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-arrow" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "552b0b3f342f7ec41b3fbd70f6339dc82a30cfd0349e7f280e7852528085349f" +checksum = "a24cc0b9cf6e367f27f27406eff13abf48a11b72446aaa40b3105c0ded5c17d9" dependencies = [ "arrow", "arrow-ipc", @@ -2244,9 +2244,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-avro" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb517d08967d536284ce70afb5fe8583133779249f2d7b90587d339741a7f195" +checksum = "f26a312bb06528c17b4bb86b798b0c4321e7656d85e0ef65c2e9adf07b0a518d" dependencies = [ "arrow", "arrow-avro", @@ -2263,9 +2263,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68850aa426b897e879c8b87e512ea8124f1d0a2869a4e51808ddaaddf1bc0ada" +checksum = "e1abe56b2a7a2d1d6de5117dd1a203181e267f28529faa5da546947621b697d7" dependencies = [ "arrow", "async-trait", @@ -2286,9 +2286,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "402f93242ae08ef99139ee2c528a49d087efe88d5c7b2c3ff5480855a40ce54f" +checksum = "9c3e467f0611ad7bdd5aad17c63c9bb6182d04e5282e5496d897ea2b49c024ba" dependencies = [ "arrow", "async-trait", @@ -2309,9 +2309,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-parquet" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffd2499c1bee0eeccf6a57156105700eeeb17bc701899ac719183c4e74231450" +checksum = "4cc35b92cd560082155e80d9c826929c852d3c51543f4affd3a51c464a0aab3a" dependencies = [ "arrow", "async-trait", @@ -2340,15 +2340,15 @@ dependencies = [ [[package]] name = "datafusion-doc" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb9e7e5d11130c48c8bd4e80c79a9772dd28ce6dc330baca9246205d245b9e2e" +checksum = "d69bb69d8769e34f76839c960dbde24c1ac0c885a79b6c3c2287bdc56ec67891" [[package]] name = "datafusion-execution" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37a8643ab852eb68864e1b72ae789e8066282dce48eea6347ffb0aee33d1ccc0" +checksum = "d8eac0a09bc8d263f52025cad9e001da4d8138d633fa288edda4d06b1772eae6" dependencies = [ "arrow", "arrow-buffer", @@ -2368,9 +2368,9 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6932f4d71eed9c8d9341476a2b845aadfabde5495d08dbcd8fc23881f49fa7a0" +checksum = "eeb14d374767ee0fc62dc79a5ba8bcf8a63c14e993c7d992d0e63adfa23d77d3" dependencies = [ "arrow", "arrow-schema", @@ -2391,9 +2391,9 @@ dependencies = [ [[package]] name = "datafusion-expr-common" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0225491839a31b1f7d2cb8092c2d50792e2fe1c1724e4e6d08e011f5feaf4ed2" +checksum = "b7b19a8c95522bee8cbb313d74263b85e355d2b52f42e67ef5694bf5de9e9356" dependencies = [ "arrow", "datafusion-common", @@ -2403,9 +2403,9 @@ dependencies = [ [[package]] name = "datafusion-functions" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14872c47bfc3d21e53ec82f57074e6987a15941c1e2f43cde4ac6ae2746634e3" +checksum = "5f64c983bbbdcb729d921a2b2ac3375598719b5cc0c30345ad664936f3176fc7" dependencies = [ "arrow", "arrow-buffer", @@ -2435,9 +2435,9 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75a2ca14e1b609be21e657e2d3130b2f446456b08393b377bb721a33952d2e09" +checksum = "89bc17041e424a47ed062f43df24d84aab8b57c4c3221e5c1a5eef46d6c5718b" dependencies = [ "arrow", "datafusion-common", @@ -2456,9 +2456,9 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ece74ba09092d2ef9c9b54a38445450aea292a1f8b04faf531936b723a24b3c" +checksum = "97dd2a9e865c6108059f5b37b77934f84b50bfb108f837bd0e5c9536e03f0545" dependencies = [ "arrow", "datafusion-common", @@ -2468,9 +2468,9 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f3e3f9ee8ca59bf70518802107de6f1b88a9509efdc629fadc5de9d6b2d5ef5" +checksum = "75f0bdfeef16d96417b9632ef855645376b242e9006a126dfd0bedfc54a93f5f" dependencies = [ "arrow", "arrow-ord", @@ -2493,9 +2493,9 @@ dependencies = [ [[package]] name = "datafusion-functions-table" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89161dffc22cf2b50f9f4b1bee83b5221d3b4ed7c2e37fd7aa2b22a5297b3a26" +checksum = "f4e4941673c917819616877e9993da4503e4f4739812be0bc32c5356184c6383" dependencies = [ "arrow", "async-trait", @@ -2509,9 +2509,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7339345b226b3874037708bf5023ba1c2de705128f8457a095aae5ae9cb9c78" +checksum = "12dd2e16c12b84b6f6b41b19f55b366dd1c46876bb35b86896c6349067379e8d" dependencies = [ "arrow", "datafusion-common", @@ -2526,9 +2526,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa84836dc2392df6f43d6a29d37fb56a8ebdc8b3f4e10ae8dc15861fd20278fb" +checksum = "4cdc5e4b6f8b6ef823cc1c761f85088ad4c884fe8df64df3cbcc6b2b84698441" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -2536,9 +2536,9 @@ dependencies = [ [[package]] name = "datafusion-macros" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "587164e03ad68732aa9e7bfe5686e3f25970d4c64fd4bd80790749840892dae5" +checksum = "1a3614234dd93578c92428cb4f408e020874f0d2b7e6c90c928d9d28b5df2ceb" dependencies = [ "datafusion-doc", "quote", @@ -2547,9 +2547,9 @@ dependencies = [ [[package]] name = "datafusion-optimizer" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77f20e8cf9e8654d92f4c16b24c487353ee5bf153ffc12d5772cd399ab8cd281" +checksum = "a0635620b050b81bb92764e99250868f654e2cd5ad1bece413283b3f73c83179" dependencies = [ "arrow", "chrono", @@ -2567,9 +2567,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f015a4a82f6f7ff7e1d8d4bf3870a936752fa38b17705dfcc14adef95aa8922c" +checksum = "8cabf7a86eb70b816729e33c81bf7767c936ee1226f607a114f5dac2decac8d0" dependencies = [ "arrow", "datafusion-common", @@ -2589,9 +2589,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-adapter" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51e6ffff8acdfe54e0ea15ccf38115c4a9184433b0439f42907637928d00a235" +checksum = "de222e04f7e6744501555a54ab0abe26bfdfebee380af79a9bdc175704246859" dependencies = [ "arrow", "datafusion-common", @@ -2604,9 +2604,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7967a3e171c6a4bf09474b3f7a14f1a3db13ed1714ba12156f33fcce2bba54e8" +checksum = "72d0d0057fc5a502d45c870cb6d47c66eb7bdd5edb1bd71ad6f3f724975ac2a8" dependencies = [ "arrow", "chrono", @@ -2621,9 +2621,9 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59ff803e2a96054cb6d83f35f9e60fd4f42eac515e1932bd1b2dbc91d5fcbf36" +checksum = "86046eed10950c5f9aaed9acfd148e9bd2e1dfdfe4f9aef607d1447b271e4183" dependencies = [ "arrow", "datafusion-common", @@ -2640,9 +2640,9 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "776ee54d47d15bdb126452f9ca17b03761e3b004682914beaedd3f86eb507fbc" +checksum = "9bc84da934c903407ba297971ebcc020c4c1a38aafd765d6c144c76eff3fa6a1" dependencies = [ "arrow", "arrow-data", @@ -2673,9 +2673,9 @@ dependencies = [ [[package]] name = "datafusion-proto" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dd15a1ba5d3af93808241065c6c44dbca8296a189845e8a587c45c07bf0ffae" +checksum = "67791dcfacd142a9d95f8f73c2a161094cc936d231d80c235f5017dacf24e84e" dependencies = [ "arrow", "chrono", @@ -2700,9 +2700,9 @@ dependencies = [ [[package]] name = "datafusion-proto-common" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90042982cf9462eb06a0b81f92efa4188dae871e7ea3ab8dc61aa9c9349b2530" +checksum = "b8cd9e80d637891645d074db0f6c650b591117367247deb313fdfb78dff559cb" dependencies = [ "arrow", "datafusion-common", @@ -2711,9 +2711,9 @@ dependencies = [ [[package]] name = "datafusion-pruning" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5fb9e5774660aa69c3ba93c610f175f75b65cb8c3776edb3626de8f3a4f4ee3" +checksum = "eb63eeac6de19be40f487b65dd84e546195f783c5a9928618e0c4f2a3569b0d7" dependencies = [ "arrow", "datafusion-common", @@ -2727,9 +2727,9 @@ dependencies = [ [[package]] name = "datafusion-session" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15ce715fa2a61f4623cc234bcc14a3ef6a91f189128d5b14b468a6a17cdfc417" +checksum = "5f961d209177f91bd014db5cbb2c33b7d28a2597b9003e77f17aeb712964315a" dependencies = [ "async-trait", "datafusion-common", @@ -2741,9 +2741,9 @@ dependencies = [ [[package]] name = "datafusion-spark" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "390bb0bf37cb2b95ffd65eacd66f60df50793d1f94097799e416f39477a51957" +checksum = "a1ccd16a6949503e56c084df1b90c8889db826ec9347d2f0f51a7837d6fa011e" dependencies = [ "arrow", "bigdecimal", @@ -2771,9 +2771,9 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6094ad36a3ed6d7ac87b20b479b2d0b118250f66cf997603829fdc65b44a7099" +checksum = "b1d71cb454da682b2af7488e1fc1ddd72ee1b28f19297b8ccad73f0a21ee9a69" dependencies = [ "arrow", "bigdecimal", @@ -2892,7 +2892,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -3057,7 +3057,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -4605,7 +4605,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -5381,7 +5381,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -7264,7 +7264,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -8356,7 +8356,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b55fb86dfd3a2f5f76ea78310a88f96c4ea21a3031f8d212443d56123fd0521" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -8664,7 +8664,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -8744,7 +8744,7 @@ dependencies = [ "cfg-if", "libc", "psm", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -8934,7 +8934,7 @@ dependencies = [ "getrandom 0.4.3", "once_cell", "rustix 1.1.4", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -9997,7 +9997,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 1d96fb7e7..afc0a871a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -150,7 +150,7 @@ prost-types = "0.14" # The `axum` version must match the one used in `tonic` (replace `RELEASE` with the release we are using): # https://github.com/hyperium/tonic/blob/vRELEASE/tonic/Cargo.toml axum = "0.8.9" -datafusion = { version = "54.0.0", default-features = false, features = [ +datafusion = { version = "54.1.0", default-features = false, features = [ "nested_expressions", "crypto_expressions", "datetime_expressions", @@ -164,18 +164,18 @@ datafusion = { version = "54.0.0", default-features = false, features = [ "avro", "recursive_protection", ] } -datafusion-common = { version = "54.0.0", features = ["object_store"] } -datafusion-datasource = { version = "54.0.0" } -datafusion-datasource-avro = { version = "54.0.0" } -datafusion-datasource-json = { version = "54.0.0" } -datafusion-expr = { version = "54.0.0", default-features = false } -datafusion-expr-common = { version = "54.0.0" } -datafusion-proto = { version = "54.0.0" } -datafusion-functions = { version = "54.0.0" } -datafusion-functions-nested = { version = "54.0.0", default-features = false } -datafusion-physical-expr = { version = "54.0.0" } -datafusion-session = { version = "54.0.0" } -datafusion-spark = { version = "54.0.0", features = ["core"] } +datafusion-common = { version = "54.1.0", features = ["object_store"] } +datafusion-datasource = { version = "54.1.0" } +datafusion-datasource-avro = { version = "54.1.0" } +datafusion-datasource-json = { version = "54.1.0" } +datafusion-expr = { version = "54.1.0", default-features = false } +datafusion-expr-common = { version = "54.1.0" } +datafusion-proto = { version = "54.1.0" } +datafusion-functions = { version = "54.1.0" } +datafusion-functions-nested = { version = "54.1.0", default-features = false } +datafusion-physical-expr = { version = "54.1.0" } +datafusion-session = { version = "54.1.0" } +datafusion-spark = { version = "54.1.0", features = ["core"] } pyo3 = { version = "0.29.0", features = ["serde"] } jiter = { version = "0.15.0", default-features = false } arrow = { version = "58.3.0", features = ["chrono-tz"] } diff --git a/crates/sail-execution/proto/sail/plan/physical.proto b/crates/sail-execution/proto/sail/plan/physical.proto index 6254b100a..f57d08b62 100644 --- a/crates/sail-execution/proto/sail/plan/physical.proto +++ b/crates/sail-execution/proto/sail/plan/physical.proto @@ -657,6 +657,7 @@ message RecursiveQueryExecNode { bytes static_term = 2; bytes recursive_term = 3; bool is_distinct = 4; + bytes output_schema = 5; } message SortMergeJoinExecNode { diff --git a/crates/sail-execution/src/proto/codec.rs b/crates/sail-execution/src/proto/codec.rs index 9d252fe44..ab5624d92 100644 --- a/crates/sail-execution/src/proto/codec.rs +++ b/crates/sail-execution/src/proto/codec.rs @@ -616,11 +616,14 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { static_term, recursive_term, is_distinct, + output_schema, }) => { let static_term = try_decode_physical_plan(ctx, self, &static_term)?; let recursive_term = try_decode_physical_plan(ctx, self, &recursive_term)?; + let output_schema = Arc::new(try_decode_schema(&output_schema)?); Ok(Arc::new(RecursiveQueryExec::try_new( name, + output_schema, static_term, recursive_term, is_distinct, @@ -1530,11 +1533,13 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { let recursive_term = try_encode_physical_plan(self, recursive_query.recursive_term().clone())?; let is_distinct = recursive_query.is_distinct(); + let output_schema = try_encode_schema(recursive_query.schema().as_ref())?; NodeKind::RecursiveQuery(r#gen::RecursiveQueryExecNode { name, static_term, recursive_term, is_distinct, + output_schema, }) } else if let Some(sort_merge_join) = node.downcast_ref::() { let left = try_encode_physical_plan(self, sort_merge_join.left().clone())?; @@ -4421,6 +4426,46 @@ mod tests { codec.try_decode_udwf(&name, &buf) } + #[test] + fn test_round_trip_recursive_query_preserves_output_schema() -> Result<()> { + use datafusion::arrow::datatypes::{DataType, Field}; + use datafusion::physical_plan::empty::EmptyExec; + + let static_schema = Arc::new(Schema::new(vec![Field::new( + "value", + DataType::Int32, + false, + )])); + let output_schema = Arc::new(Schema::new(vec![Field::new( + "value", + DataType::Int32, + true, + )])); + let static_term: Arc = + Arc::new(EmptyExec::new(Arc::clone(&static_schema))); + let recursive_term: Arc = + Arc::new(EmptyExec::new(Arc::clone(&output_schema))); + let plan = Arc::new(RecursiveQueryExec::try_new( + "numbers".to_string(), + Arc::clone(&output_schema), + static_term, + recursive_term, + false, + )?); + + let codec = RemoteExecutionCodec; + let bytes = try_encode_physical_plan(&codec, plan)?; + let decoded = try_decode_physical_plan(&TaskContext::default(), &codec, &bytes)?; + let recursive_query = decoded + .downcast_ref::() + .ok_or_else(|| plan_datafusion_err!("decoded plan is not a RecursiveQueryExec"))?; + + assert_eq!(recursive_query.schema(), output_schema); + assert_eq!(recursive_query.static_term().schema(), output_schema); + assert_eq!(recursive_query.recursive_term().schema(), output_schema); + Ok(()) + } + #[test] fn test_struct_field_matching_proto_round_trip_and_default() -> Result<()> { for mode in [ diff --git a/crates/sail-function/src/scalar/array/spark_array_compact.rs b/crates/sail-function/src/scalar/array/spark_array_compact.rs index 435539d18..54a490ef6 100644 --- a/crates/sail-function/src/scalar/array/spark_array_compact.rs +++ b/crates/sail-function/src/scalar/array/spark_array_compact.rs @@ -84,6 +84,7 @@ fn array_compact_inner(args: &[ArrayRef]) -> Result { fn array_compact_generic(list: &GenericListArray) -> Result { let values = list.values(); + let values_nulls = values.logical_nulls(); let offsets = list.offsets(); // Build a boolean filter mask for the flattened values array. @@ -106,7 +107,7 @@ fn array_compact_generic(list: &GenericListArray) -> Resu } else { // Non-null row: keep only non-null values. for j in start..end { - let keep_value = values.is_valid(j); + let keep_value = values_nulls.as_ref().is_none_or(|nulls| nulls.is_valid(j)); keep.push(keep_value); if keep_value { total_kept += 1; diff --git a/python/pysail/tests/spark/__snapshots__/test_tpcds.plan.yaml b/python/pysail/tests/spark/__snapshots__/test_tpcds.plan.yaml index 5a414a05c..57e4cc0cc 100644 --- a/python/pysail/tests/spark/__snapshots__/test_tpcds.plan.yaml +++ b/python/pysail/tests/spark/__snapshots__/test_tpcds.plan.yaml @@ -342,36 +342,35 @@ data: AggregateExec: mode=FinalPartitioned, gby=[#8@0 as #8], aggr=[count(Int64(1))] RepartitionExec: partitioning=Hash([#8@0], 4), input_partitions=4 AggregateExec: mode=Partial, gby=[#8@0 as #8], aggr=[count(Int64(1))] - FilterExec: CAST(#87@1 AS Decimal128(11, 7)) > Some(12),2,1 * #155@2, projection=[#8@0] - HashJoinExec: mode=CollectLeft, join_type=Left, on=[(#94@2, #145@1)], projection=[#8@0, #87@1, #155@3] - CoalescePartitionsExec - HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(#82@0, #33@1)], projection=[#8@3, #87@1, #94@2] - ProjectionExec: expr=[_0@0 as #82, _5@1 as #87, _12@2 as #94] - DataSourceExec: partitions=1, partition_sizes=[1] - HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(#31@1, CAST(d.#54 AS Float64)@1)], projection=[#8@0, #33@2] - CoalescePartitionsExec - HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(c.#13 AS Float64)@2, #34@2)], projection=[#8@0, #31@3, #33@4] - CoalescePartitionsExec - ProjectionExec: expr=[#8@0 as #8, #13@1 as #13, CAST(#13@1 AS Float64) as CAST(c.#13 AS Float64)] - RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 - HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(#17@1, #0@0)], projection=[#8@3, #13@0] - ProjectionExec: expr=[_0@0 as #13, _4@1 as #17] - DataSourceExec: partitions=1, partition_sizes=[1] - ProjectionExec: expr=[_0@0 as #0, _8@1 as #8] - DataSourceExec: partitions=1, partition_sizes=[1] - RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 - ProjectionExec: expr=[_0@0 as #31, _2@1 as #33, _3@2 as #34] - DataSourceExec: partitions=1, partition_sizes=[1] - ProjectionExec: expr=[_0@0 as #54, CAST(_0@0 AS Float64) as CAST(d.#54 AS Float64)] - FilterExec: _3@1 = scalar_subquery(), projection=[_0@0] - RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 + HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(#94@2, #145@1)], filter=CAST(#87@0 AS Decimal128(11, 7)) > Some(12),2,1 * #155@1, projection=[#8@0] + CoalescePartitionsExec + HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(#82@0, #33@1)], projection=[#8@3, #87@1, #94@2] + ProjectionExec: expr=[_0@0 as #82, _5@1 as #87, _12@2 as #94] + DataSourceExec: partitions=1, partition_sizes=[1] + HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(#31@1, CAST(d.#54 AS Float64)@1)], projection=[#8@0, #33@2] + CoalescePartitionsExec + HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(c.#13 AS Float64)@2, #34@2)], projection=[#8@0, #31@3, #33@4] + CoalescePartitionsExec + ProjectionExec: expr=[#8@0 as #8, #13@1 as #13, CAST(#13@1 AS Float64) as CAST(c.#13 AS Float64)] + RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 + HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(#17@1, #0@0)], projection=[#8@3, #13@0] + ProjectionExec: expr=[_0@0 as #13, _4@1 as #17] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[_0@0 as #0, _8@1 as #8] + DataSourceExec: partitions=1, partition_sizes=[1] + RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 + ProjectionExec: expr=[_0@0 as #31, _2@1 as #33, _3@2 as #34] DataSourceExec: partitions=1, partition_sizes=[1] - ProjectionExec: expr=[avg(j.#138)@1 as #155, #145@0 as #145] - AggregateExec: mode=FinalPartitioned, gby=[#145@0 as #145], aggr=[avg(j.#138)] - RepartitionExec: partitioning=Hash([#145@0], 4), input_partitions=1 - AggregateExec: mode=Partial, gby=[#145@1 as #145], aggr=[avg(j.#138)] - ProjectionExec: expr=[_5@0 as #138, _12@1 as #145] + ProjectionExec: expr=[_0@0 as #54, CAST(_0@0 AS Float64) as CAST(d.#54 AS Float64)] + FilterExec: _3@1 = scalar_subquery(), projection=[_0@0] + RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[avg(j.#138)@1 as #155, #145@0 as #145] + AggregateExec: mode=FinalPartitioned, gby=[#145@0 as #145], aggr=[avg(j.#138)] + RepartitionExec: partitioning=Hash([#145@0], 4), input_partitions=1 + AggregateExec: mode=Partial, gby=[#145@1 as #145], aggr=[avg(j.#138)] + ProjectionExec: expr=[_5@0 as #138, _12@1 as #145] + DataSourceExec: partitions=1, partition_sizes=[1] AggregateExec: mode=FinalPartitioned, gby=[#132@0 as #132], aggr=[] RepartitionExec: partitioning=Hash([#132@0], 4), input_partitions=4 AggregateExec: mode=Partial, gby=[#132@0 as #132], aggr=[] diff --git a/python/pysail/tests/spark/function/features/array/array_compact.feature b/python/pysail/tests/spark/function/features/array/array_compact.feature index c9fecd320..4f93d7c66 100644 --- a/python/pysail/tests/spark/function/features/array/array_compact.feature +++ b/python/pysail/tests/spark/function/features/array/array_compact.feature @@ -13,13 +13,14 @@ Feature: array_compact() removes null values from an array | | Examples: - | case | arr | result | - | removes null values from integer array | array(1, NULL, 2, NULL, 3) | [1, 2, 3] | - | removes null values from string array | array('a', NULL, 'b', NULL, 'c') | [a, b, c] | - | with no null values returns same array | array(1, 2, 3) | [1, 2, 3] | - | with all null values returns empty array | array(CAST(NULL AS INT), CAST(NULL AS INT)) | [] | - | with null at beginning | array(NULL, 1, 2, 3) | [1, 2, 3] | - | with null at end | array(1, 2, 3, NULL) | [1, 2, 3] | + | case | arr | result | + | removes null values from integer array | array(1, NULL, 2, NULL, 3) | [1, 2, 3] | + | removes null values from string array | array('a', NULL, 'b', NULL, 'c') | [a, b, c] | + | with no null values returns same array | array(1, 2, 3) | [1, 2, 3] | + | with all null values returns empty array | array(CAST(NULL AS INT), CAST(NULL AS INT)) | [] | + | with untyped null values returns empty array | array(NULL, NULL) | [] | + | with null at beginning | array(NULL, 1, 2, 3) | [1, 2, 3] | + | with null at end | array(1, 2, 3, NULL) | [1, 2, 3] | Rule: Empty array handling diff --git a/python/pysail/tests/spark/function/features/math/width_bucket.feature b/python/pysail/tests/spark/function/features/math/width_bucket.feature index 76ce7961f..a9e0e9378 100644 --- a/python/pysail/tests/spark/function/features/math/width_bucket.feature +++ b/python/pysail/tests/spark/function/features/math/width_bucket.feature @@ -4,7 +4,6 @@ Feature: width_bucket output schema @spark_null Rule: Output schema - @sail-bug Scenario: a non-null literal input to width_bucket yields the schema Spark declares When query """ @@ -16,7 +15,6 @@ Feature: width_bucket output schema |-- result: long (nullable = true) """ - @sail-bug Scenario: a nullable column input to width_bucket stays nullable When query """ @@ -53,7 +51,8 @@ Feature: width_bucket output schema | | Examples: - | case | args | result | - | width_bucket doctest #2 (result) | 0.0, 10.0, 0.0, 5 | 6 | - | width_bucket doctest #3 (result) | 10.0, 0.0, 10.0, 5 | 6 | - | width_bucket doctest #4 (result) | 10.0, 0.0, 0.0, 5 | NULL | + | case | args | result | + | width_bucket returns middle bucket | 5.0, 0.0, 10.0, 5 | 3 | + | width_bucket doctest #2 (result) | 0.0, 10.0, 0.0, 5 | 6 | + | width_bucket doctest #3 (result) | 10.0, 0.0, 10.0, 5 | 6 | + | width_bucket doctest #4 (result) | 10.0, 0.0, 0.0, 5 | NULL | From 23a3653b433b3f07b2e098241080e90dba114f92 Mon Sep 17 00:00:00 2001 From: Xiaolong Liang Date: Wed, 29 Jul 2026 12:34:22 +0800 Subject: [PATCH 092/131] feat: add basic checkpoint support (#2270) --- Cargo.lock | 16 + Cargo.toml | 1 + crates/sail-cache/Cargo.toml | 4 + crates/sail-cache/src/lib.rs | 1 + crates/sail-cache/src/remote_checkpoint.rs | 223 ++++ crates/sail-common/src/config/application.rs | 11 + .../sail-common/src/config/application.yaml | 11 + crates/sail-common/src/spec/plan.rs | 4 + .../proto/sail/plan/physical.proto | 22 + .../sail-execution/src/job_graph/planner.rs | 33 + crates/sail-execution/src/proto/codec.rs | 230 +++- crates/sail-logical-plan/src/lib.rs | 1 + .../src/remote_checkpoint.rs | 134 +++ crates/sail-object-store/src/lib.rs | 4 + crates/sail-object-store/src/path.rs | 114 ++ crates/sail-physical-plan/Cargo.toml | 8 + crates/sail-physical-plan/src/lib.rs | 1 + .../src/remote_checkpoint.rs | 1050 +++++++++++++++++ crates/sail-plan/Cargo.toml | 1 + .../src/resolver/command/checkpoint.rs | 36 + crates/sail-plan/src/resolver/command/mod.rs | 5 + crates/sail-plan/src/resolver/query/mod.rs | 32 +- crates/sail-session/Cargo.toml | 3 + crates/sail-session/src/planner.rs | 504 +++++++- .../src/session_factory/server.rs | 4 + .../src/session_manager/actor/handler.rs | 14 +- crates/sail-spark-connect/src/server.rs | 5 +- .../src/service/plan_executor.rs | 70 +- python/pysail/testing/spark/steps/plan.py | 14 + .../__snapshots__/test_checkpoint.plan.yaml | 59 + .../tests/spark/dataframe/test_checkpoint.py | 169 +++ 31 files changed, 2758 insertions(+), 26 deletions(-) create mode 100644 crates/sail-cache/src/remote_checkpoint.rs create mode 100644 crates/sail-logical-plan/src/remote_checkpoint.rs create mode 100644 crates/sail-object-store/src/path.rs create mode 100644 crates/sail-physical-plan/src/remote_checkpoint.rs create mode 100644 crates/sail-plan/src/resolver/command/checkpoint.rs create mode 100644 python/pysail/tests/spark/dataframe/__snapshots__/test_checkpoint.plan.yaml create mode 100644 python/pysail/tests/spark/dataframe/test_checkpoint.py diff --git a/Cargo.lock b/Cargo.lock index 260af0fee..959fef93e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7359,11 +7359,15 @@ version = "0.6.6" dependencies = [ "chrono", "datafusion", + "datafusion-common", "log", "moka", "object_store", "sail-common", + "sail-common-datafusion", + "sail-object-store", "thiserror 2.0.19", + "uuid", ] [[package]] @@ -7930,14 +7934,22 @@ version = "0.6.6" dependencies = [ "datafusion", "datafusion-common", + "datafusion-datasource-parquet", "datafusion-expr", "futures", "lazy_static", + "log", + "object_store", + "parquet", + "sail-cache", "sail-catalog", "sail-common-datafusion", "sail-logical-plan", + "serde", + "serde_json", "tokio", "tokio-stream", + "uuid", ] [[package]] @@ -7968,6 +7980,7 @@ dependencies = [ "rand 0.10.2", "regex", "ryu", + "sail-cache", "sail-catalog", "sail-catalog-memory", "sail-common", @@ -8056,12 +8069,14 @@ dependencies = [ "chrono", "datafusion", "datafusion-common", + "datafusion-datasource", "datafusion-expr", "datafusion-physical-expr", "fastrace", "futures", "indexmap 2.14.0", "log", + "object_store", "readonly", "sail-cache", "sail-catalog", @@ -8091,6 +8106,7 @@ dependencies = [ "thiserror 2.0.19", "tokio", "tonic", + "uuid", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index afc0a871a..7d52713fd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -168,6 +168,7 @@ datafusion-common = { version = "54.1.0", features = ["object_store"] } datafusion-datasource = { version = "54.1.0" } datafusion-datasource-avro = { version = "54.1.0" } datafusion-datasource-json = { version = "54.1.0" } +datafusion-datasource-parquet = { version = "54.1.0" } datafusion-expr = { version = "54.1.0", default-features = false } datafusion-expr-common = { version = "54.1.0" } datafusion-proto = { version = "54.1.0" } diff --git a/crates/sail-cache/Cargo.toml b/crates/sail-cache/Cargo.toml index cd6b240ee..e9ced3ad3 100644 --- a/crates/sail-cache/Cargo.toml +++ b/crates/sail-cache/Cargo.toml @@ -8,10 +8,14 @@ workspace = true [dependencies] sail-common = { path = "../sail-common" } +sail-common-datafusion = { path = "../sail-common-datafusion" } +sail-object-store = { path = "../sail-object-store" } thiserror = { workspace = true } object_store = { workspace = true } datafusion = { workspace = true } +datafusion-common = { workspace = true } chrono = { workspace = true } log = { workspace = true } moka = { workspace = true } +uuid = { workspace = true } diff --git a/crates/sail-cache/src/lib.rs b/crates/sail-cache/src/lib.rs index a8fe1223a..422e2fb1a 100644 --- a/crates/sail-cache/src/lib.rs +++ b/crates/sail-cache/src/lib.rs @@ -4,6 +4,7 @@ pub mod error; pub mod file_listing_cache; pub mod file_metadata_cache; pub mod file_statistics_cache; +pub mod remote_checkpoint; #[expect(dead_code)] pub(crate) fn try_parse_memory_limit(limit: &str) -> Option { diff --git a/crates/sail-cache/src/remote_checkpoint.rs b/crates/sail-cache/src/remote_checkpoint.rs new file mode 100644 index 000000000..7ff9355aa --- /dev/null +++ b/crates/sail-cache/src/remote_checkpoint.rs @@ -0,0 +1,223 @@ +use std::collections::HashMap; +use std::collections::hash_map::Entry; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, RwLock}; + +use datafusion::arrow::datatypes::SchemaRef; +use datafusion::execution::object_store::ObjectStoreUrl; +use datafusion::execution::runtime_env::RuntimeEnv; +use datafusion::physical_expr::{LexOrdering, Partitioning}; +use datafusion_common::{DataFusionError, Result, internal_datafusion_err}; +use object_store::path::Path; +use sail_common_datafusion::extension::SessionExtension; +use sail_object_store::{ + ResolvedObjectStorePath, delete_object_store_prefix_objects, resolve_object_store_path, +}; +use uuid::Uuid; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RemoteCheckpointFile { + pub location: Path, + pub size: u64, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RemoteCheckpointPartition { + pub partition: usize, + pub row_count: u64, + pub file: Option, +} + +#[derive(Clone, Debug)] +pub struct RemoteCheckpointDescriptor { + pub relation_id: String, + pub object_store_url: ObjectStoreUrl, + pub prefix: Path, + pub logical_schema: SchemaRef, + pub storage_schema: SchemaRef, + pub output_partitioning: Partitioning, + pub output_ordering: Option, + pub partitions: Vec, +} + +/// Session-local publication index for complete checkpoints; object-store contents are not +/// reloaded after a process restart. +#[derive(Debug)] +pub struct RemoteCheckpointRegistry { + path: Option, + session_namespace: String, + closed: AtomicBool, + relations: RwLock>>, +} + +impl RemoteCheckpointRegistry { + pub fn new(path: Option) -> Self { + Self { + path, + session_namespace: Uuid::new_v4().to_string(), + closed: AtomicBool::new(false), + relations: RwLock::new(HashMap::new()), + } + } + + pub fn resolve_relation( + &self, + runtime_env: &RuntimeEnv, + relation_id: &str, + ) -> Result<(ObjectStoreUrl, Path)> { + if self.closed.load(Ordering::Acquire) { + return Err(DataFusionError::Plan( + "checkpoint session is stopping".to_string(), + )); + } + let root = self.resolve_root(runtime_env)?; + let prefix = self.session_prefix(root.prefix()).join(relation_id); + Ok((root.object_store_url().clone(), prefix)) + } + + pub fn insert(&self, descriptor: RemoteCheckpointDescriptor) -> Result<()> { + if self.closed.load(Ordering::Acquire) { + return Err(DataFusionError::Plan( + "checkpoint session is stopping".to_string(), + )); + } + let relation_id = descriptor.relation_id.clone(); + let mut relations = self.relations.write().map_err(|error| { + internal_datafusion_err!("checkpoint registry is poisoned: {error}") + })?; + match relations.entry(relation_id.clone()) { + Entry::Occupied(_) => { + return Err(internal_datafusion_err!( + "checkpoint relation already exists: {relation_id}" + )); + } + Entry::Vacant(entry) => { + entry.insert(Arc::new(descriptor)); + } + } + Ok(()) + } + + pub fn get(&self, relation_id: &str) -> Result>> { + let relations = self.relations.read().map_err(|error| { + internal_datafusion_err!("checkpoint registry is poisoned: {error}") + })?; + Ok(relations.get(relation_id).cloned()) + } + + pub async fn cleanup_relation( + &self, + runtime_env: &RuntimeEnv, + object_store_url: &ObjectStoreUrl, + prefix: &Path, + ) -> Result<()> { + let store = runtime_env.object_store(object_store_url)?; + delete_object_store_prefix_objects(store.as_ref(), prefix).await + } + + pub async fn cleanup_session(&self, runtime_env: &RuntimeEnv) -> Result<()> { + self.closed.store(true, Ordering::Release); + let storage_cleanup = if self.path.is_some() { + match self.resolve_root(runtime_env) { + Ok(root) => { + delete_object_store_prefix_objects( + root.store().as_ref(), + &self.session_prefix(root.prefix()), + ) + .await + } + Err(error) => Err(error), + } + } else { + Ok(()) + }; + let registry_cleanup = self.clear(); + match (storage_cleanup, registry_cleanup) { + (Ok(()), Ok(())) => Ok(()), + (Err(error), Ok(())) | (Ok(()), Err(error)) => Err(error), + (Err(storage_error), Err(registry_error)) => Err(internal_datafusion_err!( + "checkpoint storage cleanup failed: {storage_error}; registry cleanup failed: {registry_error}" + )), + } + } + + fn resolve_root(&self, runtime_env: &RuntimeEnv) -> Result { + let path = self.path.as_deref().ok_or_else(|| { + DataFusionError::Plan( + "checkpoint object-store path is not configured; set execution.checkpoint.path" + .to_string(), + ) + })?; + resolve_object_store_path(runtime_env, path) + } + + fn session_prefix(&self, root: &Path) -> Path { + root.clone().join(self.session_namespace.as_str()) + } + + fn clear(&self) -> Result<()> { + let mut relations = self.relations.write().map_err(|error| { + internal_datafusion_err!("checkpoint registry is poisoned: {error}") + })?; + relations.clear(); + Ok(()) + } +} + +impl Default for RemoteCheckpointRegistry { + fn default() -> Self { + Self::new(None) + } +} + +impl SessionExtension for RemoteCheckpointRegistry { + fn name() -> &'static str { + "remote checkpoint registry" + } +} + +#[cfg(test)] +#[expect(clippy::expect_used, clippy::unwrap_used)] +mod tests { + use datafusion::arrow::datatypes::Schema; + + use super::*; + + fn descriptor(relation_id: &str) -> RemoteCheckpointDescriptor { + let schema = Arc::new(Schema::empty()); + RemoteCheckpointDescriptor { + relation_id: relation_id.to_string(), + object_store_url: ObjectStoreUrl::parse("memory://").unwrap(), + prefix: Path::from("checkpoint"), + logical_schema: Arc::clone(&schema), + storage_schema: schema, + output_partitioning: Partitioning::UnknownPartitioning(1), + output_ordering: None, + partitions: vec![], + } + } + + #[test] + fn registry_owns_only_ready_descriptors() -> Result<()> { + let registry = RemoteCheckpointRegistry::default(); + registry.insert(descriptor("relation"))?; + assert_eq!( + registry + .get("relation")? + .expect("descriptor must be present") + .relation_id, + "relation" + ); + let original = registry + .get("relation")? + .expect("descriptor must be present"); + assert!(registry.insert(descriptor("relation")).is_err()); + let retained = registry + .get("relation")? + .expect("descriptor must remain present"); + assert!(Arc::ptr_eq(&original, &retained)); + registry.clear()?; + assert!(registry.get("relation")?.is_none()); + Ok(()) + } +} diff --git a/crates/sail-common/src/config/application.rs b/crates/sail-common/src/config/application.rs index ae9af1a14..8c811cd67 100644 --- a/crates/sail-common/src/config/application.rs +++ b/crates/sail-common/src/config/application.rs @@ -38,6 +38,16 @@ pub struct AppConfig { pub internal: (), } +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CheckpointConfig { + #[serde( + serialize_with = "serialize_non_empty_string", + deserialize_with = "deserialize_non_empty_string" + )] + pub path: Option, +} + /// A configuration provider that injects placeholder internal configuration. struct InternalConfigPlaceholder; @@ -292,6 +302,7 @@ pub struct ExecutionConfig { pub collect_statistics: bool, pub use_row_number_estimates_to_optimize_partitioning: bool, pub file_listing_cache: FileListingCacheConfig, + pub checkpoint: CheckpointConfig, } #[derive(Debug, Clone, Serialize, Deserialize)] diff --git a/crates/sail-common/src/config/application.yaml b/crates/sail-common/src/config/application.yaml index cd59a557f..a5e72742a 100644 --- a/crates/sail-common/src/config/application.yaml +++ b/crates/sail-common/src/config/application.yaml @@ -317,6 +317,17 @@ This setting can only be configured at startup and cannot be changed at runtime. experimental: true +- key: execution.checkpoint.path + type: string + default: "" + description: | + The object storage base path for DataFrame checkpoints managed by the server. The path must be + readable and writable by the driver and every worker. Checkpoint operations are rejected when + this value is empty. Sail does not fall back to driver memory or local worker storage. Sail + removes each session namespace on shutdown. An object storage lifecycle policy can clean up + data left by an ungraceful process exit. + experimental: true + - key: parquet.enable_page_index type: boolean default: "true" diff --git a/crates/sail-common/src/spec/plan.rs b/crates/sail-common/src/spec/plan.rs index 29ffb6abd..052a2166c 100644 --- a/crates/sail-common/src/spec/plan.rs +++ b/crates/sail-common/src/spec/plan.rs @@ -447,6 +447,10 @@ pub enum CommandNode { is_global: bool, if_exists: bool, }, + RemoteCheckpoint { + relation_id: String, + input: Box, + }, Write(Write), WriteTo(WriteTo), WriteStream(WriteStream), diff --git a/crates/sail-execution/proto/sail/plan/physical.proto b/crates/sail-execution/proto/sail/plan/physical.proto index f57d08b62..077c5f325 100644 --- a/crates/sail-execution/proto/sail/plan/physical.proto +++ b/crates/sail-execution/proto/sail/plan/physical.proto @@ -67,9 +67,29 @@ message ExtendedPhysicalPlanNode { NoopSinkExecNode noop_sink = 52; CsvExecNode csv = 53; ParquetExecNode parquet = 54; + RemoteCheckpointWriteExecNode remote_checkpoint_write = 55; + RemoteCheckpointCommitExecNode remote_checkpoint_commit = 56; } } +message RemoteCheckpointWriteExecNode { + bytes input = 1; + string object_store_url = 2; + string prefix = 3; + bytes storage_schema = 4; +} + +message RemoteCheckpointCommitExecNode { + bytes input = 1; + string relation_id = 2; + string object_store_url = 3; + string prefix = 4; + bytes logical_schema = 5; + bytes storage_schema = 6; + bytes output_partitioning = 7; + LexOrdering output_ordering = 8; +} + message ExtendedPhysicalExprNode { oneof ExprKind { CastColumnExprNode schema_evolution_cast = 1; @@ -605,6 +625,7 @@ message MemoryExecNode { bool show_sizes = 4; repeated LexOrdering sort_information = 5; optional uint64 limit = 6; + bytes output_partitioning = 7; } message ValuesExecNode { @@ -626,6 +647,7 @@ message ParquetExecNode { bytes base_config = 1; bytes options = 2; optional bytes predicate = 3; + bytes output_partitioning = 4; } message ArrowExecNode { diff --git a/crates/sail-execution/src/job_graph/planner.rs b/crates/sail-execution/src/job_graph/planner.rs index 30ab4f58d..8cd58c790 100644 --- a/crates/sail-execution/src/job_graph/planner.rs +++ b/crates/sail-execution/src/job_graph/planner.rs @@ -30,6 +30,7 @@ use sail_iceberg::physical_plan::IcebergCommitExec; use sail_physical_plan::barrier::BarrierExec; use sail_physical_plan::catalog_command::CatalogCommandExec; use sail_physical_plan::coalesce::CoalesceExec; +use sail_physical_plan::remote_checkpoint::RemoteCheckpointCommitExec; use sail_physical_plan::repartition::ExplicitRepartitionExec; use crate::error::{ExecutionError, ExecutionResult}; @@ -461,6 +462,7 @@ fn plan_job_graph_stages( || subtree.plan.is::() || subtree.plan.is::() || subtree.plan.is::() + || subtree.plan.is::() { if matches!(driver_stage_handling, DriverStageHandling::PreserveRoot) { subtree.into_planned_subtree() @@ -611,6 +613,7 @@ fn is_driver_stage_plan(plan: &Arc) -> bool { || plan.is::() || plan.is::() || plan.is::() + || plan.is::() } fn wrap_pending_scalar_subqueries( @@ -912,6 +915,7 @@ mod tests { use std::sync::Arc; use datafusion::arrow::datatypes::{DataType, Field, Schema, SchemaRef}; + use datafusion::execution::object_store::ObjectStoreUrl; use datafusion::functions_aggregate::sum::sum_udaf; use datafusion::logical_expr::Operator; use datafusion::logical_expr::execution_props::{ScalarSubqueryResults, SubqueryIndex}; @@ -931,6 +935,7 @@ mod tests { use sail_physical_plan::barrier::BarrierExec; use sail_physical_plan::catalog_command::CatalogCommandExec; use sail_physical_plan::coalesce::CoalesceExec; + use sail_physical_plan::remote_checkpoint::RemoteCheckpointCommitExec; use sail_physical_plan::repartition::ExplicitRepartitionExec; use super::JobGraph; @@ -1008,6 +1013,34 @@ mod tests { )); } + #[test] + fn test_job_graph_places_remote_checkpoint_commit_on_driver() { + let schema = schema(); + let commit = Arc::new(RemoteCheckpointCommitExec::new( + empty_plan(), + "relation".to_string(), + ObjectStoreUrl::parse("memory://").unwrap(), + object_store::path::Path::from("checkpoint/session/relation"), + Arc::clone(&schema), + schema, + Partitioning::UnknownPartitioning(1), + None, + )); + + let graph = JobGraph::try_new(commit).unwrap(); + + assert_eq!(graph.stages().len(), 2); + assert_eq!(graph.stages()[0].placement, TaskPlacement::Driver); + assert!(graph.stages()[0].plan.is::()); + assert!(matches!( + graph.stages()[1].inputs.as_slice(), + [StageInput { + stage: 0, + mode: InputMode::Forward, + }] + )); + } + #[test] fn test_job_graph_keeps_driver_actual_plan_inside_barrier() { let precondition = Arc::new( diff --git a/crates/sail-execution/src/proto/codec.rs b/crates/sail-execution/src/proto/codec.rs index ab5624d92..e1a395482 100644 --- a/crates/sail-execution/src/proto/codec.rs +++ b/crates/sail-execution/src/proto/codec.rs @@ -261,6 +261,9 @@ use sail_physical_plan::map_partitions::MapPartitionsExec; use sail_physical_plan::merge_cardinality_check::MergeCardinalityCheckExec; use sail_physical_plan::monotonic_id::MonotonicIdExec; use sail_physical_plan::range::RangeExec; +use sail_physical_plan::remote_checkpoint::{ + CheckpointDataSource, RemoteCheckpointCommitExec, RemoteCheckpointWriteExec, +}; use sail_physical_plan::schema_pivot::SchemaPivotExec; use sail_physical_plan::show_string::ShowStringExec; use sail_physical_plan::spark_partition_id::SparkPartitionIdExec; @@ -439,6 +442,7 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { show_sizes, sort_information, limit, + output_partitioning, }) => { let schema = try_decode_schema(&schema)?; let partitions = partitions @@ -454,7 +458,67 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { .with_show_sizes(show_sizes) .try_with_sort_information(sort_information)? .with_limit(limit.map(|x| x as usize)); - Ok(Arc::new(DataSourceExec::new(Arc::new(source)))) + let scan = DataSourceExec::new(Arc::new(source)); + let scan = if output_partitioning.is_empty() { + scan + } else { + let output_partitioning = + self.try_decode_partitioning(&output_partitioning, &scan.schema(), ctx)?; + scan.with_partitioning(output_partitioning) + }; + Ok(Arc::new(scan)) + } + NodeKind::RemoteCheckpointWrite(r#gen::RemoteCheckpointWriteExecNode { + input, + object_store_url, + prefix, + storage_schema, + }) => { + let input = try_decode_physical_plan(ctx, self, &input)?; + let object_store_url = + datafusion::execution::object_store::ObjectStoreUrl::parse(object_store_url)?; + let prefix = object_store::path::Path::parse(prefix) + .map_err(|error| plan_datafusion_err!("invalid checkpoint prefix: {error}"))?; + let storage_schema = Arc::new(try_decode_schema(&storage_schema)?); + Ok(Arc::new(RemoteCheckpointWriteExec::try_new( + input, + object_store_url, + prefix, + storage_schema, + )?)) + } + NodeKind::RemoteCheckpointCommit(r#gen::RemoteCheckpointCommitExecNode { + input, + relation_id, + object_store_url, + prefix, + logical_schema, + storage_schema, + output_partitioning, + output_ordering, + }) => { + let input = try_decode_physical_plan(ctx, self, &input)?; + let object_store_url = + datafusion::execution::object_store::ObjectStoreUrl::parse(object_store_url)?; + let prefix = object_store::path::Path::parse(prefix) + .map_err(|error| plan_datafusion_err!("invalid checkpoint prefix: {error}"))?; + let logical_schema = Arc::new(try_decode_schema(&logical_schema)?); + let storage_schema = Arc::new(try_decode_schema(&storage_schema)?); + let output_partitioning = + self.try_decode_partitioning(&output_partitioning, &storage_schema, ctx)?; + let output_ordering = output_ordering + .map(|ordering| self.try_decode_lex_ordering(&ordering, &storage_schema, ctx)) + .transpose()?; + Ok(Arc::new(RemoteCheckpointCommitExec::new( + input, + relation_id, + object_store_url, + prefix, + logical_schema, + storage_schema, + output_partitioning, + output_ordering, + ))) } NodeKind::Values(r#gen::ValuesExecNode { data, schema }) => { let schema = try_decode_schema(&schema)?; @@ -506,6 +570,7 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { base_config, options, predicate, + output_partitioning, }) => { let base_config = try_decode_message(&base_config)?; let predicate_schema = parse_protobuf_file_scan_schema(&base_config)?; @@ -540,7 +605,15 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { &RemotePhysicalProtoConverter {}, Arc::new(source), )?; - Ok(Arc::new(DataSourceExec::new(Arc::new(source)))) + let scan = DataSourceExec::new(Arc::new(source)); + let scan = if output_partitioning.is_empty() { + scan + } else { + let output_partitioning = + self.try_decode_partitioning(&output_partitioning, &scan.schema(), ctx)?; + scan.with_partitioning(output_partitioning) + }; + Ok(Arc::new(scan)) } NodeKind::Arrow(r#gen::ArrowExecNode { base_config }) => { let base_config = try_decode_message(&base_config)?; @@ -1522,6 +1595,28 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { udf: Some(udf), schema, }) + } else if let Some(checkpoint) = node.downcast_ref::() { + NodeKind::RemoteCheckpointWrite(r#gen::RemoteCheckpointWriteExecNode { + input: try_encode_physical_plan(self, checkpoint.input().clone())?, + object_store_url: checkpoint.object_store_url().as_str().to_string(), + prefix: checkpoint.prefix().to_string(), + storage_schema: try_encode_schema(checkpoint.storage_schema().as_ref())?, + }) + } else if let Some(checkpoint) = node.downcast_ref::() { + NodeKind::RemoteCheckpointCommit(r#gen::RemoteCheckpointCommitExecNode { + input: try_encode_physical_plan(self, checkpoint.input().clone())?, + relation_id: checkpoint.relation_id().to_string(), + object_store_url: checkpoint.object_store_url().as_str().to_string(), + prefix: checkpoint.prefix().to_string(), + logical_schema: try_encode_schema(checkpoint.logical_schema().as_ref())?, + storage_schema: try_encode_schema(checkpoint.storage_schema().as_ref())?, + output_partitioning: self + .try_encode_partitioning(checkpoint.checkpoint_partitioning())?, + output_ordering: checkpoint + .checkpoint_ordering() + .map(|ordering| self.try_encode_lex_ordering(ordering)) + .transpose()?, + }) } else if let Some(work_table) = node.downcast_ref::() { let name = work_table.name().to_string(); let schema = try_encode_schema(work_table.schema().as_ref())?; @@ -1609,7 +1704,16 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { common_prefix_length, }) } else if let Some(data_source) = node.downcast_ref::() { - let source = data_source.data_source(); + let data_source = data_source.data_source(); + let (source, output_partitioning) = + if let Some(checkpoint) = data_source.downcast_ref::() { + ( + checkpoint.source(), + self.try_encode_partitioning(checkpoint.checkpoint_partitioning())?, + ) + } else { + (data_source, Vec::new()) + }; if let Some(file_scan) = source.downcast_ref::() { let file_source = file_scan.file_source(); if let Some(text_source) = file_source.downcast_ref::() { @@ -1676,6 +1780,7 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { base_config, options, predicate, + output_partitioning, }) } else if file_source.is::() { let base_config = try_encode_message(serialize_file_scan_config( @@ -1730,6 +1835,7 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { show_sizes: memory.show_sizes(), sort_information, limit: memory.fetch().map(|x| x as u64), + output_partitioning, }) } else { return plan_err!("unsupported data source node: {data_source:?}"); @@ -4917,6 +5023,124 @@ mod tests { assert_same_result(&physical, decoded_expr, schema_ref, vec![Arc::new(list)]) } + #[test] + fn test_round_trip_remote_checkpoint_write_plan() -> Result<()> { + use datafusion::physical_plan::empty::EmptyExec; + + let input_schema = Arc::new(Schema::new(vec![Field::new( + "logical_name", + DataType::Int64, + true, + )])); + let storage_schema = Arc::new(Schema::new(vec![Field::new("_c0", DataType::Int64, true)])); + let checkpoint = RemoteCheckpointWriteExec::try_new( + Arc::new(EmptyExec::new(input_schema)), + datafusion::execution::object_store::ObjectStoreUrl::parse("s3://checkpoint-bucket")?, + object_store::path::Path::from("checkpoints/session/relation"), + Arc::clone(&storage_schema), + )?; + let codec = RemoteExecutionCodec; + + let bytes = try_encode_physical_plan(&codec, Arc::new(checkpoint))?; + let decoded = try_decode_physical_plan(&TaskContext::default(), &codec, &bytes)?; + let decoded = decoded + .downcast_ref::() + .ok_or_else(|| plan_datafusion_err!("decoded plan is not a checkpoint writer"))?; + + assert_eq!( + decoded.object_store_url().as_str(), + "s3://checkpoint-bucket/" + ); + assert_eq!( + decoded.prefix(), + &object_store::path::Path::from("checkpoints/session/relation") + ); + assert_eq!(decoded.storage_schema(), &storage_schema); + assert_eq!(decoded.input().schema().field(0).name(), "logical_name"); + Ok(()) + } + + #[test] + fn test_round_trip_remote_checkpoint_commit_plan() -> Result<()> { + use datafusion::physical_expr::expressions::Column; + use datafusion::physical_plan::empty::EmptyExec; + + let metadata_schema = Arc::new(Schema::new(vec![Field::new( + "metadata", + DataType::Utf8, + false, + )])); + let logical_schema = Arc::new(Schema::new(vec![Field::new( + "logical_name", + DataType::Int64, + true, + )])); + let storage_schema = Arc::new(Schema::new(vec![Field::new("_c0", DataType::Int64, true)])); + let key = Arc::new(Column::new("_c0", 0)) as Arc; + let ordering = LexOrdering::new([PhysicalSortExpr::new_default(Arc::clone(&key))]) + .ok_or_else(|| plan_datafusion_err!("expected non-empty checkpoint ordering"))?; + let checkpoint = RemoteCheckpointCommitExec::new( + Arc::new(EmptyExec::new(metadata_schema)), + "relation".to_string(), + datafusion::execution::object_store::ObjectStoreUrl::parse("s3://checkpoint-bucket")?, + object_store::path::Path::from("checkpoints/session/relation"), + Arc::clone(&logical_schema), + Arc::clone(&storage_schema), + Partitioning::Hash(vec![key], 4), + Some(ordering), + ); + let codec = RemoteExecutionCodec; + + let bytes = try_encode_physical_plan(&codec, Arc::new(checkpoint))?; + let decoded = try_decode_physical_plan(&TaskContext::default(), &codec, &bytes)?; + let decoded = decoded + .downcast_ref::() + .ok_or_else(|| plan_datafusion_err!("decoded plan is not a checkpoint commit"))?; + + assert_eq!(decoded.relation_id(), "relation"); + assert_eq!(decoded.logical_schema(), &logical_schema); + assert_eq!(decoded.storage_schema(), &storage_schema); + let Partitioning::Hash(expressions, 4) = decoded.checkpoint_partitioning() else { + return plan_err!("expected checkpoint hash partitioning"); + }; + let partition_column = expressions[0] + .downcast_ref::() + .ok_or_else(|| plan_datafusion_err!("checkpoint hash key is not a column"))?; + assert_eq!(partition_column.name(), "_c0"); + assert_eq!(partition_column.index(), 0); + assert!(decoded.checkpoint_ordering().is_some()); + Ok(()) + } + + #[test] + fn test_round_trip_checkpoint_data_source_partitioning() -> Result<()> { + use datafusion::datasource::memory::MemorySourceConfig; + use datafusion::datasource::source::DataSourceExec; + use datafusion::physical_expr::expressions::Column; + use datafusion::physical_plan::ExecutionPlanProperties; + + let schema = Arc::new(Schema::new(vec![Field::new("key", DataType::Int64, false)])); + let source = MemorySourceConfig::try_new(&[vec![], vec![]], Arc::clone(&schema), None)?; + let key = Arc::new(Column::new("key", 0)) as Arc; + let source = Arc::new(CheckpointDataSource::new( + Arc::new(source), + Partitioning::Hash(vec![key], 2), + )); + let scan = DataSourceExec::new(source); + let remote_scan = RemoteDataSourceExec::new(&scan); + let codec = RemoteExecutionCodec; + + let bytes = try_encode_physical_plan(&codec, Arc::new(remote_scan))?; + let decoded = try_decode_physical_plan(&TaskContext::default(), &codec, &bytes)?; + + assert!(decoded.is::()); + assert!(matches!( + decoded.output_partitioning(), + Partitioning::Hash(_, 2) + )); + Ok(()) + } + #[test] fn test_hash_output_partitioning_decodes_higher_order_key() -> Result<()> { use crate::task::definition::{TaskOutput, TaskOutputDistribution, TaskOutputLocator}; diff --git a/crates/sail-logical-plan/src/lib.rs b/crates/sail-logical-plan/src/lib.rs index fa6d6a904..26cf741e1 100644 --- a/crates/sail-logical-plan/src/lib.rs +++ b/crates/sail-logical-plan/src/lib.rs @@ -4,6 +4,7 @@ pub mod map_partitions; pub mod merge; pub mod monotonic_id; pub mod range; +pub mod remote_checkpoint; pub mod repartition; pub mod schema_pivot; pub mod show_string; diff --git a/crates/sail-logical-plan/src/remote_checkpoint.rs b/crates/sail-logical-plan/src/remote_checkpoint.rs new file mode 100644 index 000000000..93274a2f7 --- /dev/null +++ b/crates/sail-logical-plan/src/remote_checkpoint.rs @@ -0,0 +1,134 @@ +use std::fmt::Formatter; +use std::sync::Arc; + +use datafusion::arrow::datatypes::SchemaRef; +use datafusion_common::{DFSchema, DFSchemaRef, Result, internal_datafusion_err}; +use datafusion_expr::{Expr, LogicalPlan, UserDefinedLogicalNodeCore}; +use educe::Educe; + +#[derive(Clone, Debug, Eq, PartialEq, Hash, Educe)] +#[educe(PartialOrd)] +pub struct RemoteCheckpointRelationNode { + relation_id: String, + #[educe(PartialOrd(ignore))] + schema: DFSchemaRef, +} + +impl RemoteCheckpointRelationNode { + pub fn try_new(relation_id: String, schema: SchemaRef) -> Result { + Ok(Self { + relation_id, + schema: Arc::new(DFSchema::try_from(schema.as_ref().clone())?), + }) + } + + pub fn relation_id(&self) -> &str { + &self.relation_id + } +} + +impl UserDefinedLogicalNodeCore for RemoteCheckpointRelationNode { + fn name(&self) -> &str { + "RemoteCheckpointRelation" + } + + fn inputs(&self) -> Vec<&LogicalPlan> { + vec![] + } + + fn schema(&self) -> &DFSchemaRef { + &self.schema + } + + fn expressions(&self) -> Vec { + vec![] + } + + fn fmt_for_explain(&self, f: &mut Formatter) -> std::fmt::Result { + write!( + f, + "RemoteCheckpointRelation: relation_id={}", + self.relation_id + ) + } + + fn with_exprs_and_inputs(&self, exprs: Vec, inputs: Vec) -> Result { + if !exprs.is_empty() || !inputs.is_empty() { + return Err(internal_datafusion_err!( + "RemoteCheckpointRelation does not accept expressions or inputs" + )); + } + Ok(self.clone()) + } +} + +#[derive(Clone, Debug, Eq, PartialEq, Hash, Educe)] +#[educe(PartialOrd)] +pub struct RemoteCheckpointCommandNode { + relation_id: String, + input: Arc, + #[educe(PartialOrd(ignore))] + logical_schema: SchemaRef, +} + +impl RemoteCheckpointCommandNode { + pub fn new(relation_id: String, input: Arc, logical_schema: SchemaRef) -> Self { + Self { + relation_id, + input, + logical_schema, + } + } + + pub fn relation_id(&self) -> &str { + &self.relation_id + } + + pub fn logical_schema(&self) -> &SchemaRef { + &self.logical_schema + } +} + +impl UserDefinedLogicalNodeCore for RemoteCheckpointCommandNode { + fn name(&self) -> &str { + "RemoteCheckpointCommand" + } + + fn inputs(&self) -> Vec<&LogicalPlan> { + vec![self.input.as_ref()] + } + + fn schema(&self) -> &DFSchemaRef { + DFSchema::empty_ref() + } + + fn expressions(&self) -> Vec { + vec![] + } + + fn fmt_for_explain(&self, f: &mut Formatter) -> std::fmt::Result { + write!( + f, + "RemoteCheckpointCommand: relation_id={}", + self.relation_id + ) + } + + fn with_exprs_and_inputs(&self, exprs: Vec, inputs: Vec) -> Result { + if !exprs.is_empty() || inputs.len() != 1 { + return Err(internal_datafusion_err!( + "RemoteCheckpointCommand requires one input and no expressions" + )); + } + Ok(Self { + input: Arc::new(inputs.into_iter().next().ok_or_else(|| { + internal_datafusion_err!("RemoteCheckpointCommand input is missing") + })?), + ..self.clone() + }) + } + + fn necessary_children_exprs(&self, _output_columns: &[usize]) -> Option>> { + Some(vec![(0..self.input.schema().fields().len()).collect()]) + } +} diff --git a/crates/sail-object-store/src/lib.rs b/crates/sail-object-store/src/lib.rs index c9bc0541a..deb0b1647 100644 --- a/crates/sail-object-store/src/lib.rs +++ b/crates/sail-object-store/src/lib.rs @@ -1,6 +1,10 @@ mod hugging_face; mod layers; +mod path; mod registry; mod s3; +pub use path::{ + ResolvedObjectStorePath, delete_object_store_prefix_objects, resolve_object_store_path, +}; pub use registry::DynamicObjectStoreRegistry; diff --git a/crates/sail-object-store/src/path.rs b/crates/sail-object-store/src/path.rs new file mode 100644 index 000000000..ca4d01a9c --- /dev/null +++ b/crates/sail-object-store/src/path.rs @@ -0,0 +1,114 @@ +use std::sync::Arc; + +use datafusion::datasource::listing::ListingTableUrl; +use datafusion::execution::object_store::ObjectStoreUrl; +use datafusion::execution::runtime_env::RuntimeEnv; +use datafusion_common::{DataFusionError, Result}; +use futures::StreamExt; +use object_store::ObjectStore; +use object_store::path::Path; + +#[derive(Clone)] +pub struct ResolvedObjectStorePath { + object_store_url: ObjectStoreUrl, + prefix: Path, + store: Arc, +} + +impl ResolvedObjectStorePath { + pub fn object_store_url(&self) -> &ObjectStoreUrl { + &self.object_store_url + } + + pub fn prefix(&self) -> &Path { + &self.prefix + } + + pub fn store(&self) -> &Arc { + &self.store + } +} + +pub async fn delete_object_store_prefix_objects( + store: &dyn ObjectStore, + prefix: &Path, +) -> Result<()> { + let locations = store + .list(Some(prefix)) + .map(|result| result.map(|object| object.location)); + let mut deleted = store.delete_stream(Box::pin(locations)); + while let Some(result) = deleted.next().await { + match result { + Ok(_) | Err(object_store::Error::NotFound { .. }) => {} + Err(error) => return Err(DataFusionError::ObjectStore(Box::new(error))), + } + } + Ok(()) +} + +pub fn resolve_object_store_path( + runtime_env: &RuntimeEnv, + path: &str, +) -> Result { + let directory = format!("{}/", path.trim_end_matches('/')); + let url = ListingTableUrl::parse(&directory)?; + let object_store_url = url.object_store(); + let store = runtime_env.object_store(&object_store_url)?; + Ok(ResolvedObjectStorePath { + object_store_url, + prefix: url.prefix().clone(), + store, + }) +} + +#[cfg(test)] +mod tests { + use datafusion::execution::runtime_env::RuntimeEnv; + use object_store::memory::InMemory; + use object_store::{ObjectStoreExt, PutPayload}; + use url::Url; + + use super::*; + + #[tokio::test] + async fn resolved_path_puts_and_deletes_only_its_prefix() -> Result<()> { + let runtime_env = RuntimeEnv::default(); + runtime_env.register_object_store( + &Url::parse("memory:///") + .map_err(|error| DataFusionError::External(Box::new(error)))?, + Arc::new(InMemory::new()), + ); + let resolved = resolve_object_store_path(&runtime_env, "memory:///checkpoint")?; + let first = resolved.prefix().clone().join("part-00000.parquet"); + let nested = resolved + .prefix() + .clone() + .join("nested") + .join("part-00001.parquet"); + let outside = Path::from("outside.parquet"); + + for (location, bytes) in [ + (&first, b"first".as_slice()), + (&nested, b"nested".as_slice()), + (&outside, b"outside".as_slice()), + ] { + resolved + .store() + .put(location, PutPayload::from(bytes.to_vec())) + .await + .map_err(|error| DataFusionError::ObjectStore(Box::new(error)))?; + } + delete_object_store_prefix_objects(resolved.store().as_ref(), resolved.prefix()).await?; + + assert!(matches!( + resolved.store().head(&first).await, + Err(object_store::Error::NotFound { .. }) + )); + assert!(matches!( + resolved.store().head(&nested).await, + Err(object_store::Error::NotFound { .. }) + )); + assert!(resolved.store().head(&outside).await.is_ok()); + Ok(()) + } +} diff --git a/crates/sail-physical-plan/Cargo.toml b/crates/sail-physical-plan/Cargo.toml index 5a76c127c..668f64bc0 100644 --- a/crates/sail-physical-plan/Cargo.toml +++ b/crates/sail-physical-plan/Cargo.toml @@ -8,13 +8,21 @@ workspace = true [dependencies] sail-catalog = { path = "../sail-catalog" } +sail-cache = { path = "../sail-cache" } sail-common-datafusion = { path = "../sail-common-datafusion" } sail-logical-plan = { path = "../sail-logical-plan" } datafusion = { workspace = true } datafusion-common = { workspace = true } +datafusion-datasource-parquet = { workspace = true } datafusion-expr = { workspace = true } futures = { workspace = true } +object_store = { workspace = true } +parquet = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } tokio = { workspace = true } tokio-stream = { workspace = true } lazy_static = { workspace = true } +log = { workspace = true } +uuid = { workspace = true } diff --git a/crates/sail-physical-plan/src/lib.rs b/crates/sail-physical-plan/src/lib.rs index 524382d39..c5c5cbaa1 100644 --- a/crates/sail-physical-plan/src/lib.rs +++ b/crates/sail-physical-plan/src/lib.rs @@ -6,6 +6,7 @@ pub mod map_partitions; pub mod merge_cardinality_check; pub mod monotonic_id; pub mod range; +pub mod remote_checkpoint; pub mod repartition; pub mod schema_pivot; pub mod show_string; diff --git a/crates/sail-physical-plan/src/remote_checkpoint.rs b/crates/sail-physical-plan/src/remote_checkpoint.rs new file mode 100644 index 000000000..e3820b8eb --- /dev/null +++ b/crates/sail-physical-plan/src/remote_checkpoint.rs @@ -0,0 +1,1050 @@ +use std::any::Any; +use std::collections::BTreeMap; +use std::fmt::Formatter; +use std::sync::Arc; + +use datafusion::arrow::array::{Array, ArrayRef, StringArray}; +use datafusion::arrow::datatypes::{DataType, Field, Schema, SchemaRef}; +use datafusion::arrow::record_batch::RecordBatch; +use datafusion::common::config::{ConfigOptions, TableParquetOptions}; +use datafusion::datasource::listing::ListingTableUrl; +use datafusion::datasource::physical_plan::{FileOutputMode, FileSinkConfig}; +use datafusion::datasource::sink::DataSink; +use datafusion::datasource::source::{DataSource, OpenArgs}; +use datafusion::execution::object_store::ObjectStoreUrl; +use datafusion::execution::{SendableRecordBatchStream, TaskContext}; +use datafusion::logical_expr::dml::InsertOp; +use datafusion::physical_expr::projection::ProjectionExprs; +use datafusion::physical_expr::{ + Distribution, EquivalenceProperties, LexOrdering, Partitioning, PhysicalExpr, PhysicalSortExpr, +}; +use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType, SchedulingType}; +use datafusion::physical_plan::filter_pushdown::FilterPushdownPropagation; +use datafusion::physical_plan::metrics::ExecutionPlanMetricsSet; +use datafusion::physical_plan::stream::RecordBatchStreamAdapter; +use datafusion::physical_plan::{ + DisplayAs, DisplayFormatType, ExecutionPlan, ExecutionPlanProperties, PlanProperties, + SortOrderPushdownResult, +}; +use datafusion_common::{DataFusionError, Result, Statistics, internal_datafusion_err}; +use datafusion_datasource_parquet::ParquetSink; +use futures::{StreamExt, stream}; +use object_store::path::Path; +use object_store::{ObjectStore, ObjectStoreExt}; +use sail_cache::remote_checkpoint::{ + RemoteCheckpointDescriptor, RemoteCheckpointFile, RemoteCheckpointPartition, + RemoteCheckpointRegistry, +}; +use sail_common_datafusion::array::record_batch::record_batch_with_schema; +use sail_common_datafusion::extension::SessionExtensionAccessor; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +pub const METADATA_COLUMN: &str = "metadata"; + +/// Exposes the partitioning recorded by a checkpoint instead of the layout-derived partitioning +/// reported by its underlying file or memory source. +/// +/// Source rewrites remain wrapped so optimizer pushdowns cannot discard this property. +#[derive(Debug)] +pub struct CheckpointDataSource { + source: Arc, + output_partitioning: Partitioning, +} + +impl CheckpointDataSource { + pub fn new(source: Arc, output_partitioning: Partitioning) -> Self { + Self { + source, + output_partitioning, + } + } + + pub fn source(&self) -> &Arc { + &self.source + } + + pub fn checkpoint_partitioning(&self) -> &Partitioning { + &self.output_partitioning + } + + fn map_source(&self, source: Arc) -> Arc { + Arc::new(Self::new(source, self.output_partitioning.clone())) + } +} + +impl DataSource for CheckpointDataSource { + fn open( + &self, + partition: usize, + context: Arc, + ) -> Result { + self.source.open(partition, context) + } + + fn fmt_as(&self, display_type: DisplayFormatType, f: &mut Formatter) -> std::fmt::Result { + self.source.fmt_as(display_type, f) + } + + fn output_partitioning(&self) -> Partitioning { + self.output_partitioning.clone() + } + + fn eq_properties(&self) -> EquivalenceProperties { + self.source.eq_properties() + } + + fn scheduling_type(&self) -> SchedulingType { + self.source.scheduling_type() + } + + fn partition_statistics(&self, partition: Option) -> Result> { + self.source.partition_statistics(partition) + } + + fn with_fetch(&self, limit: Option) -> Option> { + self.source + .with_fetch(limit) + .map(|source| self.map_source(source)) + } + + fn fetch(&self) -> Option { + self.source.fetch() + } + + fn metrics(&self) -> ExecutionPlanMetricsSet { + self.source.metrics() + } + + fn try_swapping_with_projection( + &self, + _projection: &ProjectionExprs, + ) -> Result>> { + Ok(None) + } + + fn try_pushdown_filters( + &self, + filters: Vec>, + config: &ConfigOptions, + ) -> Result>> { + let result = self.source.try_pushdown_filters(filters, config)?; + Ok(FilterPushdownPropagation { + filters: result.filters, + updated_node: result.updated_node.map(|source| self.map_source(source)), + }) + } + + fn try_pushdown_sort( + &self, + order: &[PhysicalSortExpr], + ) -> Result>> { + Ok(self + .source + .try_pushdown_sort(order)? + .map(|source| self.map_source(source))) + } + + fn with_preserve_order(&self, preserve_order: bool) -> Option> { + self.source + .with_preserve_order(preserve_order) + .map(|source| self.map_source(source)) + } + + fn with_new_state(&self, state: Arc) -> Option> { + self.source + .with_new_state(state) + .map(|source| self.map_source(source)) + } + + fn create_sibling_state(&self, config: &ConfigOptions) -> Option> { + self.source.create_sibling_state(config) + } + + fn open_with_args(&self, args: OpenArgs) -> Result { + self.source.open_with_args(args) + } +} + +pub fn checkpoint_storage_schema(logical_schema: &SchemaRef) -> SchemaRef { + let fields = logical_schema + .fields() + .iter() + .enumerate() + .map(|(index, field)| Arc::new(field.as_ref().clone().with_name(format!("_c{index}")))) + .collect::>(); + Arc::new(Schema::new_with_metadata( + fields, + logical_schema.metadata().clone(), + )) +} + +fn checkpoint_metadata_schema() -> SchemaRef { + Arc::new(Schema::new(vec![Field::new( + METADATA_COLUMN, + DataType::Utf8, + false, + )])) +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct CheckpointPartitionMetadata { + partition: u64, + location: Option, + size: u64, + row_count: u64, +} + +fn checkpoint_command_schema() -> SchemaRef { + Arc::new(Schema::empty()) +} + +#[derive(Debug)] +pub struct RemoteCheckpointCommitExec { + input: Arc, + relation_id: String, + object_store_url: ObjectStoreUrl, + prefix: Path, + logical_schema: SchemaRef, + storage_schema: SchemaRef, + output_partitioning: Partitioning, + output_ordering: Option, + properties: Arc, +} + +impl RemoteCheckpointCommitExec { + pub fn new( + input: Arc, + relation_id: String, + object_store_url: ObjectStoreUrl, + prefix: Path, + logical_schema: SchemaRef, + storage_schema: SchemaRef, + output_partitioning: Partitioning, + output_ordering: Option, + ) -> Self { + let properties = Arc::new(PlanProperties::new( + EquivalenceProperties::new(checkpoint_command_schema()), + Partitioning::UnknownPartitioning(1), + EmissionType::Final, + Boundedness::Bounded, + )); + Self { + input, + relation_id, + object_store_url, + prefix, + logical_schema, + storage_schema, + output_partitioning, + output_ordering, + properties, + } + } + + pub fn input(&self) -> &Arc { + &self.input + } + + pub fn relation_id(&self) -> &str { + &self.relation_id + } + + pub fn object_store_url(&self) -> &ObjectStoreUrl { + &self.object_store_url + } + + pub fn prefix(&self) -> &Path { + &self.prefix + } + + pub fn logical_schema(&self) -> &SchemaRef { + &self.logical_schema + } + + pub fn storage_schema(&self) -> &SchemaRef { + &self.storage_schema + } + + pub fn checkpoint_partitioning(&self) -> &Partitioning { + &self.output_partitioning + } + + pub fn checkpoint_ordering(&self) -> Option<&LexOrdering> { + self.output_ordering.as_ref() + } +} + +impl DisplayAs for RemoteCheckpointCommitExec { + fn fmt_as(&self, _display_type: DisplayFormatType, f: &mut Formatter) -> std::fmt::Result { + write!( + f, + "RemoteCheckpointCommitExec: relation_id={}", + self.relation_id + ) + } +} + +impl ExecutionPlan for RemoteCheckpointCommitExec { + fn name(&self) -> &str { + Self::static_name() + } + + fn properties(&self) -> &Arc { + &self.properties + } + + fn required_input_distribution(&self) -> Vec { + vec![Distribution::SinglePartition] + } + + fn benefits_from_input_partitioning(&self) -> Vec { + vec![false] + } + + fn children(&self) -> Vec<&Arc> { + vec![&self.input] + } + + fn with_new_children( + self: Arc, + children: Vec>, + ) -> Result> { + let [input] = children.as_slice() else { + return Err(internal_datafusion_err!( + "RemoteCheckpointCommitExec must have exactly one child" + )); + }; + Ok(Arc::new(Self::new( + Arc::clone(input), + self.relation_id.clone(), + self.object_store_url.clone(), + self.prefix.clone(), + Arc::clone(&self.logical_schema), + Arc::clone(&self.storage_schema), + self.output_partitioning.clone(), + self.output_ordering.clone(), + ))) + } + + fn execute( + &self, + partition: usize, + context: Arc, + ) -> Result { + if partition != 0 { + return Err(internal_datafusion_err!( + "RemoteCheckpointCommitExec can only execute partition 0" + )); + } + let input_partitions = self.input.output_partitioning().partition_count(); + if input_partitions != 1 { + return Err(internal_datafusion_err!( + "RemoteCheckpointCommitExec requires one input partition, got {input_partitions}" + )); + } + let input = self.input.execute(0, Arc::clone(&context))?; + let relation_id = self.relation_id.clone(); + let object_store_url = self.object_store_url.clone(); + let prefix = self.prefix.clone(); + let logical_schema = Arc::clone(&self.logical_schema); + let storage_schema = Arc::clone(&self.storage_schema); + let output_partitioning = self.output_partitioning.clone(); + let output_ordering = self.output_ordering.clone(); + let output_schema = self.schema(); + let stream_schema = Arc::clone(&output_schema); + let output = stream::once(async move { + commit_checkpoint( + input, + context, + relation_id, + object_store_url, + prefix, + logical_schema, + storage_schema, + output_partitioning, + output_ordering, + output_schema, + ) + .await + }); + Ok(Box::pin(RecordBatchStreamAdapter::new( + stream_schema, + output, + ))) + } +} + +#[derive(Debug)] +pub struct RemoteCheckpointWriteExec { + input: Arc, + object_store_url: ObjectStoreUrl, + prefix: Path, + storage_schema: SchemaRef, + properties: Arc, +} + +impl RemoteCheckpointWriteExec { + pub fn try_new( + input: Arc, + object_store_url: ObjectStoreUrl, + prefix: Path, + storage_schema: SchemaRef, + ) -> Result { + if matches!(input.boundedness(), Boundedness::Unbounded { .. }) { + return Err(DataFusionError::NotImplemented( + "checkpoint does not support unbounded input".to_string(), + )); + } + let input_schema = input.schema(); + let schema_matches = input_schema.fields().len() == storage_schema.fields().len() + && input_schema + .fields() + .iter() + .zip(storage_schema.fields()) + .all(|(input, storage)| input.data_type() == storage.data_type()); + if !schema_matches { + return Err(DataFusionError::Plan( + "checkpoint storage schema does not match the input schema by position".to_string(), + )); + } + let properties = Arc::new(PlanProperties::new( + EquivalenceProperties::new(checkpoint_metadata_schema()), + Partitioning::UnknownPartitioning(input.output_partitioning().partition_count()), + EmissionType::Final, + Boundedness::Bounded, + )); + Ok(Self { + input, + object_store_url, + prefix, + storage_schema, + properties, + }) + } + + pub fn input(&self) -> &Arc { + &self.input + } + + pub fn object_store_url(&self) -> &ObjectStoreUrl { + &self.object_store_url + } + + pub fn prefix(&self) -> &Path { + &self.prefix + } + + pub fn storage_schema(&self) -> &SchemaRef { + &self.storage_schema + } +} + +impl DisplayAs for RemoteCheckpointWriteExec { + fn fmt_as(&self, _display_type: DisplayFormatType, f: &mut Formatter) -> std::fmt::Result { + write!( + f, + "RemoteCheckpointWriteExec: object_store_url={}, prefix={}", + self.object_store_url, self.prefix + ) + } +} + +impl ExecutionPlan for RemoteCheckpointWriteExec { + fn name(&self) -> &str { + Self::static_name() + } + + fn properties(&self) -> &Arc { + &self.properties + } + + fn benefits_from_input_partitioning(&self) -> Vec { + vec![false] + } + + fn children(&self) -> Vec<&Arc> { + vec![&self.input] + } + + fn with_new_children( + self: Arc, + children: Vec>, + ) -> Result> { + let [input] = children.as_slice() else { + return Err(internal_datafusion_err!( + "RemoteCheckpointWriteExec must have exactly one child" + )); + }; + Ok(Arc::new(Self::try_new( + Arc::clone(input), + self.object_store_url.clone(), + self.prefix.clone(), + Arc::clone(&self.storage_schema), + )?)) + } + + fn execute( + &self, + partition: usize, + context: Arc, + ) -> Result { + let input = self.input.execute(partition, Arc::clone(&context))?; + let store = context.runtime_env().object_store(&self.object_store_url)?; + let location = self + .prefix + .clone() + .join(format!("part-{partition:05}-{}.parquet", Uuid::new_v4())); + let storage_schema = Arc::clone(&self.storage_schema); + let output_schema = self.schema(); + let stream_schema = Arc::clone(&output_schema); + let output = stream::once(write_checkpoint_partition( + partition, + input, + context, + self.object_store_url.clone(), + store, + location, + storage_schema, + output_schema, + )); + Ok(Box::pin(RecordBatchStreamAdapter::new( + stream_schema, + output, + ))) + } +} + +async fn write_checkpoint_partition( + partition: usize, + mut input: SendableRecordBatchStream, + context: Arc, + object_store_url: ObjectStoreUrl, + store: Arc, + location: Path, + storage_schema: SchemaRef, + output_schema: SchemaRef, +) -> Result { + let mut row_count = 0_u64; + let first_batch = loop { + let Some(batch) = input.next().await else { + break None; + }; + let batch = batch?; + row_count = row_count + .checked_add(u64::try_from(batch.num_rows()).map_err(|_| { + internal_datafusion_err!("checkpoint partition row count is too large") + })?) + .ok_or_else(|| internal_datafusion_err!("checkpoint partition row count overflow"))?; + if storage_schema.fields().is_empty() || batch.num_rows() == 0 { + continue; + } + break Some(batch); + }; + + // An empty partition stays in the descriptor but does not allocate an object. + let (location, size, row_count) = if let Some(first_batch) = first_batch { + let stream_schema = Arc::clone(&storage_schema); + let batch_schema = Arc::clone(&storage_schema); + let batches = stream::once(async move { Ok(first_batch) }) + .chain(input) + .map(move |batch| { + batch.and_then(|batch| record_batch_with_schema(batch, &batch_schema)) + }); + let batches: SendableRecordBatchStream = + Box::pin(RecordBatchStreamAdapter::new(stream_schema, batches)); + let location_url = format!("{object_store_url}{location}"); + let sink = ParquetSink::new( + FileSinkConfig { + original_url: location_url.clone(), + object_store_url, + file_group: Default::default(), + table_paths: vec![ListingTableUrl::parse(location_url)?], + output_schema: storage_schema, + table_partition_cols: vec![], + insert_op: InsertOp::Append, + keep_partition_by_columns: false, + file_extension: "parquet".to_string(), + file_output_mode: FileOutputMode::SingleFile, + }, + TableParquetOptions { + global: context.session_config().options().execution.parquet.clone(), + ..Default::default() + }, + ); + let row_count = sink.write_all(batches, &context).await?; + let size = store + .head(&location) + .await + .map_err(|error| DataFusionError::ObjectStore(Box::new(error)))? + .size; + (Some(location.to_string()), size, row_count) + } else { + (None, 0, row_count) + }; + let partition = u64::try_from(partition) + .map_err(|_| internal_datafusion_err!("checkpoint partition index is too large"))?; + let metadata = serde_json::to_string(&CheckpointPartitionMetadata { + partition, + location, + size, + row_count, + }) + .map_err(|error| internal_datafusion_err!("failed to encode checkpoint metadata: {error}"))?; + let columns: Vec = vec![Arc::new(StringArray::from(vec![metadata]))]; + Ok(RecordBatch::try_new(output_schema, columns)?) +} + +#[expect(clippy::too_many_arguments)] +async fn commit_checkpoint( + input: SendableRecordBatchStream, + context: Arc, + relation_id: String, + object_store_url: ObjectStoreUrl, + prefix: Path, + logical_schema: SchemaRef, + storage_schema: SchemaRef, + output_partitioning: Partitioning, + output_ordering: Option, + output_schema: SchemaRef, +) -> Result { + let registry = context.extension::()?; + let commit = async { + let partitions = collect_checkpoint_partitions(input, &prefix, &storage_schema).await?; + let output_partitioning = + checkpoint_partitioning_with_count(output_partitioning, partitions.len()); + registry.insert(RemoteCheckpointDescriptor { + relation_id, + object_store_url: object_store_url.clone(), + prefix: prefix.clone(), + logical_schema, + storage_schema, + output_partitioning, + output_ordering, + partitions, + })?; + Ok(RecordBatch::new_empty(output_schema)) + } + .await; + match commit { + Ok(batch) => Ok(batch), + Err(error) => { + match registry + .cleanup_relation(context.runtime_env().as_ref(), &object_store_url, &prefix) + .await + { + Ok(()) => Err(error), + Err(cleanup_error) => Err(internal_datafusion_err!( + "checkpoint failed: {error}; additionally failed to clean {prefix}: {cleanup_error}" + )), + } + } + } +} + +async fn collect_checkpoint_partitions( + mut stream: SendableRecordBatchStream, + relation_prefix: &Path, + storage_schema: &Schema, +) -> Result> { + let mut partitions = BTreeMap::new(); + while let Some(batch) = stream.next().await { + let batch = batch?; + if batch.num_columns() != 1 { + return Err(internal_datafusion_err!( + "checkpoint returned {} metadata columns instead of 1", + batch.num_columns() + )); + } + let metadata_values = batch + .column(0) + .as_any() + .downcast_ref::() + .ok_or_else(|| { + internal_datafusion_err!("invalid checkpoint {METADATA_COLUMN} column") + })?; + for row in 0..batch.num_rows() { + if metadata_values.is_null(row) { + return Err(internal_datafusion_err!( + "checkpoint returned null metadata" + )); + } + let CheckpointPartitionMetadata { + partition, + location, + size, + row_count, + } = serde_json::from_str(metadata_values.value(row)).map_err(|error| { + internal_datafusion_err!("invalid checkpoint metadata JSON: {error}") + })?; + let partition = usize::try_from(partition) + .map_err(|_| internal_datafusion_err!("checkpoint partition index is too large"))?; + let file = if let Some(location) = location { + if storage_schema.fields().is_empty() { + return Err(internal_datafusion_err!( + "zero-column checkpoint partition {partition} returned a file" + )); + } + if size == 0 || row_count == 0 { + return Err(internal_datafusion_err!( + "checkpoint partition {partition} has an empty file" + )); + } + let location = Path::parse(location).map_err(|error| { + internal_datafusion_err!("invalid checkpoint object location: {error}") + })?; + validate_partition_location(relation_prefix, partition, &location)?; + Some(RemoteCheckpointFile { location, size }) + } else { + if size != 0 { + return Err(internal_datafusion_err!( + "checkpoint partition {partition} has a size without a file" + )); + } + if row_count != 0 && !storage_schema.fields().is_empty() { + return Err(internal_datafusion_err!( + "non-empty checkpoint partition {partition} is missing a file" + )); + } + None + }; + if partitions + .insert( + partition, + RemoteCheckpointPartition { + partition, + row_count, + file, + }, + ) + .is_some() + { + return Err(internal_datafusion_err!( + "checkpoint returned duplicate partition {partition}" + )); + } + } + } + if partitions.is_empty() { + return Err(internal_datafusion_err!( + "checkpoint did not return partition 0" + )); + } + partitions + .into_iter() + .enumerate() + .map(|(expected, (partition, value))| { + if partition != expected { + return Err(internal_datafusion_err!( + "checkpoint did not return partition {expected}" + )); + } + Ok(value) + }) + .collect() +} + +fn checkpoint_partitioning_with_count( + partitioning: Partitioning, + partition_count: usize, +) -> Partitioning { + match partitioning { + Partitioning::RoundRobinBatch(_) => Partitioning::RoundRobinBatch(partition_count), + Partitioning::Hash(expressions, expected) if expected == partition_count => { + Partitioning::Hash(expressions, partition_count) + } + Partitioning::Hash(_, _) => Partitioning::UnknownPartitioning(partition_count), + Partitioning::UnknownPartitioning(_) => Partitioning::UnknownPartitioning(partition_count), + } +} + +fn validate_partition_location( + relation_prefix: &Path, + partition: usize, + location: &Path, +) -> Result<()> { + let Some(filename) = location.filename() else { + return Err(internal_datafusion_err!( + "checkpoint returned an invalid object for partition {partition}: {location}" + )); + }; + let prefix = format!("part-{partition:05}-"); + let file_id = filename + .strip_prefix(&prefix) + .and_then(|value| value.strip_suffix(".parquet")); + let valid_file_id = file_id.is_some_and(|value| Uuid::parse_str(value).is_ok()); + if !location.prefix_matches(relation_prefix) + || location.parts_count() != relation_prefix.parts_count() + 1 + || !valid_file_id + { + return Err(internal_datafusion_err!( + "checkpoint returned unexpected object {location} for partition {partition}" + )); + } + Ok(()) +} + +#[cfg(test)] +#[expect(clippy::expect_used)] +mod tests { + use datafusion::arrow::array::Int32Array; + use datafusion::arrow::record_batch::RecordBatchOptions; + use datafusion::datasource::memory::MemorySourceConfig; + use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec; + use datafusion::prelude::{SessionConfig, SessionContext}; + use object_store::memory::InMemory; + use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder; + + use super::*; + + fn partition_metadata(batch: &RecordBatch) -> CheckpointPartitionMetadata { + let values = batch + .column(0) + .as_any() + .downcast_ref::() + .expect("metadata column"); + serde_json::from_str(values.value(0)).expect("valid checkpoint metadata") + } + + #[test] + fn storage_schema_uses_positional_names_and_preserves_metadata() { + let logical = Arc::new(Schema::new_with_metadata( + vec![ + Field::new("value", DataType::Int64, false).with_metadata( + [("field-key".to_string(), "field-value".to_string())] + .into_iter() + .collect(), + ), + Field::new("value", DataType::Utf8, true), + ], + [("schema-key".to_string(), "schema-value".to_string())] + .into_iter() + .collect(), + )); + + let storage = checkpoint_storage_schema(&logical); + + assert_eq!(storage.field(0).name(), "_c0"); + assert_eq!(storage.field(1).name(), "_c1"); + assert_eq!(storage.field(0).metadata(), logical.field(0).metadata()); + assert_eq!(storage.metadata(), logical.metadata()); + + let empty = checkpoint_storage_schema(&Arc::new(Schema::empty())); + assert!(empty.fields().is_empty()); + } + + #[tokio::test] + async fn writes_a_partition_directly_to_object_storage() -> Result<()> { + let logical_schema = Arc::new(Schema::new(vec![Field::new( + "value", + DataType::Int32, + false, + )])); + let storage_schema = Arc::new(Schema::new(vec![Field::new("_c0", DataType::Int32, false)])); + let batch = RecordBatch::try_new( + Arc::clone(&logical_schema), + vec![Arc::new(Int32Array::from(vec![1, 2, 3]))], + )?; + let input = + MemorySourceConfig::try_new_exec(&[vec![batch]], Arc::clone(&logical_schema), None)?; + let object_store_url = ObjectStoreUrl::parse("memory://")?; + let checkpoint = RemoteCheckpointWriteExec::try_new( + input, + object_store_url.clone(), + Path::from("checkpoint"), + storage_schema, + )?; + let context = SessionContext::new(); + context + .runtime_env() + .register_object_store(object_store_url.as_ref(), Arc::new(InMemory::new())); + + let mut output = checkpoint.execute(0, context.task_ctx())?; + let commit = output.next().await.expect("commit batch")?; + let metadata = partition_metadata(&commit); + assert_eq!(metadata.partition, 0); + assert_eq!(metadata.row_count, 3); + let path = Path::parse(metadata.location.expect("checkpoint file")) + .map_err(|error| internal_datafusion_err!("invalid path: {error}"))?; + assert!( + path.filename() + .is_some_and(|name| name.starts_with("part-00000-")) + ); + let store = context.runtime_env().object_store(&object_store_url)?; + let bytes = store + .get(&path) + .await + .map_err(|error| DataFusionError::ObjectStore(Box::new(error)))? + .bytes() + .await + .map_err(|error| DataFusionError::ObjectStore(Box::new(error)))?; + let rows = ParquetRecordBatchReaderBuilder::try_new(bytes)? + .build()? + .map(|batch| batch.map(|batch| batch.num_rows())) + .collect::, _>>()?; + assert_eq!(rows, vec![3]); + Ok(()) + } + + #[tokio::test] + async fn empty_partition_does_not_create_an_object() -> Result<()> { + let schema = Arc::new(Schema::new(vec![Field::new( + "value", + DataType::Int32, + false, + )])); + let input = MemorySourceConfig::try_new_exec(&[vec![]], Arc::clone(&schema), None)?; + let object_store_url = ObjectStoreUrl::parse("memory://")?; + let checkpoint = RemoteCheckpointWriteExec::try_new( + input, + object_store_url.clone(), + Path::from("checkpoint"), + schema, + )?; + let context = SessionContext::new(); + context + .runtime_env() + .register_object_store(object_store_url.as_ref(), Arc::new(InMemory::new())); + + let mut output = checkpoint.execute(0, context.task_ctx())?; + let commit = output.next().await.expect("commit batch")?; + let metadata = partition_metadata(&commit); + assert!(metadata.location.is_none()); + assert_eq!(metadata.row_count, 0); + assert_eq!(metadata.size, 0); + let store = context.runtime_env().object_store(&object_store_url)?; + assert!( + store + .list(Some(&Path::from("checkpoint"))) + .next() + .await + .is_none() + ); + Ok(()) + } + + #[tokio::test] + async fn zero_column_partition_records_rows_without_an_object() -> Result<()> { + let schema = Arc::new(Schema::empty()); + let batch = RecordBatch::try_new_with_options( + Arc::clone(&schema), + vec![], + &RecordBatchOptions::new().with_row_count(Some(3)), + )?; + let input = MemorySourceConfig::try_new_exec(&[vec![batch]], Arc::clone(&schema), None)?; + let registry = Arc::new(RemoteCheckpointRegistry::new(Some( + "memory:///checkpoint".to_string(), + ))); + let config = SessionConfig::new().with_extension(Arc::clone(®istry)); + let context = SessionContext::new_with_config(config); + let object_store_url = ObjectStoreUrl::parse("memory://")?; + context + .runtime_env() + .register_object_store(object_store_url.as_ref(), Arc::new(InMemory::new())); + let (object_store_url, prefix) = + registry.resolve_relation(context.runtime_env().as_ref(), "zero-columns")?; + let writer: Arc = Arc::new(RemoteCheckpointWriteExec::try_new( + input, + object_store_url.clone(), + prefix.clone(), + Arc::clone(&schema), + )?); + let checkpoint = RemoteCheckpointCommitExec::new( + Arc::new(CoalescePartitionsExec::new(writer)), + "zero-columns".to_string(), + object_store_url.clone(), + prefix.clone(), + Arc::clone(&schema), + schema, + Partitioning::UnknownPartitioning(1), + None, + ); + + let mut output = checkpoint.execute(0, context.task_ctx())?; + output.next().await.expect("checkpoint commit batch")?; + let descriptor = registry + .get("zero-columns")? + .expect("checkpoint descriptor must be published"); + assert_eq!(descriptor.partitions[0].row_count, 3); + assert!(descriptor.partitions[0].file.is_none()); + let store = context.runtime_env().object_store(&object_store_url)?; + assert!(store.list(Some(&prefix)).next().await.is_none()); + Ok(()) + } + + #[tokio::test] + async fn commit_publishes_checkpoint_and_registry_cleans_it() -> Result<()> { + let logical_schema = Arc::new(Schema::new(vec![Field::new( + "value", + DataType::Int32, + false, + )])); + let storage_schema = checkpoint_storage_schema(&logical_schema); + let batch = RecordBatch::try_new( + Arc::clone(&logical_schema), + vec![Arc::new(Int32Array::from(vec![1, 2, 3]))], + )?; + let input = MemorySourceConfig::try_new_exec( + &[vec![batch], vec![]], + Arc::clone(&logical_schema), + None, + )?; + let registry = Arc::new(RemoteCheckpointRegistry::new(Some( + "memory:///checkpoint".to_string(), + ))); + let config = SessionConfig::new().with_extension(Arc::clone(®istry)); + let context = SessionContext::new_with_config(config); + let object_store_url = ObjectStoreUrl::parse("memory://")?; + context + .runtime_env() + .register_object_store(object_store_url.as_ref(), Arc::new(InMemory::new())); + let (object_store_url, prefix) = + registry.resolve_relation(context.runtime_env().as_ref(), "relation")?; + let writer: Arc = Arc::new(RemoteCheckpointWriteExec::try_new( + input, + object_store_url.clone(), + prefix.clone(), + Arc::clone(&storage_schema), + )?); + let commit = RemoteCheckpointCommitExec::new( + Arc::new(CoalescePartitionsExec::new(writer)), + "relation".to_string(), + object_store_url.clone(), + prefix.clone(), + Arc::clone(&logical_schema), + storage_schema, + Partitioning::UnknownPartitioning(1), + None, + ); + + let mut output = commit.execute(0, context.task_ctx())?; + output.next().await.expect("checkpoint commit batch")?; + let descriptor = registry + .get("relation")? + .expect("checkpoint descriptor must be published"); + assert!(matches!( + descriptor.output_partitioning, + Partitioning::UnknownPartitioning(2) + )); + assert_eq!(descriptor.partitions.len(), 2); + assert_eq!(descriptor.partitions[0].row_count, 3); + assert_eq!(descriptor.partitions[1].row_count, 0); + assert!(descriptor.partitions[0].file.is_some()); + assert!(descriptor.partitions[1].file.is_none()); + let store = context.runtime_env().object_store(&object_store_url)?; + assert!(store.list(Some(&prefix)).next().await.is_some()); + + registry + .cleanup_session(context.runtime_env().as_ref()) + .await?; + assert!(registry.get("relation")?.is_none()); + assert!(store.list(Some(&prefix)).next().await.is_none()); + assert!( + registry + .resolve_relation(context.runtime_env().as_ref(), "another") + .is_err() + ); + Ok(()) + } +} diff --git a/crates/sail-plan/Cargo.toml b/crates/sail-plan/Cargo.toml index 9d687b4a6..834e62b4d 100644 --- a/crates/sail-plan/Cargo.toml +++ b/crates/sail-plan/Cargo.toml @@ -17,6 +17,7 @@ sail-function = { path = "../sail-function" } sail-logical-plan = { path = "../sail-logical-plan" } sail-physical-plan = { path = "../sail-physical-plan" } sail-data-source = { path = "../sail-data-source" } +sail-cache = { path = "../sail-cache" } datafusion = { workspace = true } datafusion-common = { workspace = true } diff --git a/crates/sail-plan/src/resolver/command/checkpoint.rs b/crates/sail-plan/src/resolver/command/checkpoint.rs new file mode 100644 index 000000000..c0150bf9f --- /dev/null +++ b/crates/sail-plan/src/resolver/command/checkpoint.rs @@ -0,0 +1,36 @@ +use std::sync::Arc; + +use datafusion::arrow::datatypes::Schema; +use datafusion_expr::{Extension, LogicalPlan}; +use sail_common::spec; +use sail_common_datafusion::rename::schema::rename_schema; +use sail_logical_plan::remote_checkpoint::RemoteCheckpointCommandNode; + +use crate::error::PlanResult; +use crate::resolver::PlanResolver; +use crate::resolver::state::PlanResolverState; + +impl PlanResolver<'_> { + pub(super) async fn resolve_command_remote_checkpoint( + &self, + relation_id: String, + input: spec::QueryPlan, + state: &mut PlanResolverState, + ) -> PlanResult { + let input = self.resolve_query_plan(input, state).await?; + let names = Self::get_field_names(input.schema(), state)?; + let input_schema = input.schema().inner(); + let renamed = rename_schema(input_schema, &names)?; + let logical_schema = Arc::new(Schema::new_with_metadata( + renamed.fields().clone(), + input_schema.metadata().clone(), + )); + Ok(LogicalPlan::Extension(Extension { + node: Arc::new(RemoteCheckpointCommandNode::new( + relation_id, + Arc::new(input), + logical_schema, + )), + })) + } +} diff --git a/crates/sail-plan/src/resolver/command/mod.rs b/crates/sail-plan/src/resolver/command/mod.rs index ee27b9365..5111c21b8 100644 --- a/crates/sail-plan/src/resolver/command/mod.rs +++ b/crates/sail-plan/src/resolver/command/mod.rs @@ -13,6 +13,7 @@ use crate::resolver::PlanResolver; use crate::resolver::state::PlanResolverState; mod catalog; +mod checkpoint; mod delete; mod delta; mod explain; @@ -230,6 +231,10 @@ impl PlanResolver<'_> { self.resolve_catalog_create_temporary_view(view, is_global, definition, state) .await } + CommandNode::RemoteCheckpoint { relation_id, input } => { + self.resolve_command_remote_checkpoint(relation_id, *input, state) + .await + } CommandNode::Write(write) => self.resolve_command_write(write, state).await, CommandNode::WriteTo(write_to) => self.resolve_command_write_to(write_to, state).await, CommandNode::WriteStream(write_stream) => { diff --git a/crates/sail-plan/src/resolver/query/mod.rs b/crates/sail-plan/src/resolver/query/mod.rs index f43ddaad0..fb117e5bc 100644 --- a/crates/sail-plan/src/resolver/query/mod.rs +++ b/crates/sail-plan/src/resolver/query/mod.rs @@ -1,6 +1,12 @@ +use std::sync::Arc; + use async_recursion::async_recursion; -use datafusion_expr::{Expr, LogicalPlan, LogicalPlanBuilder}; +use datafusion::arrow::datatypes::Schema; +use datafusion_expr::{Expr, Extension, LogicalPlan, LogicalPlanBuilder}; +use sail_cache::remote_checkpoint::RemoteCheckpointRegistry; use sail_common::spec; +use sail_common_datafusion::extension::SessionExtensionAccessor; +use sail_logical_plan::remote_checkpoint::RemoteCheckpointRelationNode; use crate::error::{PlanError, PlanResult}; use crate::resolver::PlanResolver; @@ -227,8 +233,28 @@ impl PlanResolver<'_> { QueryNode::CachedLocalRelation { .. } => { return Err(PlanError::todo("cached local relation")); } - QueryNode::CachedRemoteRelation { .. } => { - return Err(PlanError::todo("cached remote relation")); + QueryNode::CachedRemoteRelation { relation_id } => { + let registry = self.ctx.extension::()?; + let descriptor = registry.get(&relation_id)?.ok_or_else(|| { + PlanError::analysis(format!( + "remote checkpoint relation does not exist: {relation_id}" + )) + })?; + let names = state.register_fields(descriptor.logical_schema.fields()); + let fields = descriptor + .logical_schema + .fields() + .iter() + .zip(names) + .map(|(field, name)| Arc::new(field.as_ref().clone().with_name(name))) + .collect::>(); + let schema = Arc::new(Schema::new_with_metadata( + fields, + descriptor.logical_schema.metadata().clone(), + )); + LogicalPlan::Extension(Extension { + node: Arc::new(RemoteCheckpointRelationNode::try_new(relation_id, schema)?), + }) } QueryNode::CommonInlineUserDefinedTableFunction(udtf) => { self.resolve_query_common_inline_udtf(udtf, state).await? diff --git a/crates/sail-session/Cargo.toml b/crates/sail-session/Cargo.toml index 5c8d49526..1ee398216 100644 --- a/crates/sail-session/Cargo.toml +++ b/crates/sail-session/Cargo.toml @@ -39,11 +39,14 @@ fastrace = { workspace = true } log = { workspace = true } datafusion = { workspace = true } datafusion-common = { workspace = true } +datafusion-datasource = { workspace = true } datafusion-expr = { workspace = true } datafusion-physical-expr = { workspace = true } +object_store = { workspace = true } secrecy = { workspace = true } tonic = { workspace = true } tokio = { workspace = true } chrono = { workspace = true } indexmap = { workspace = true } futures = { workspace = true } +uuid = { workspace = true } diff --git a/crates/sail-session/src/planner.rs b/crates/sail-session/src/planner.rs index 0f9a23608..0a6e259e6 100644 --- a/crates/sail-session/src/planner.rs +++ b/crates/sail-session/src/planner.rs @@ -1,17 +1,33 @@ use std::sync::Arc; use async_trait::async_trait; +use datafusion::arrow::record_batch::{RecordBatch, RecordBatchOptions}; +use datafusion::common::config::TableParquetOptions; +use datafusion::datasource::memory::MemorySourceConfig; +use datafusion::datasource::physical_plan::ParquetSource; use datafusion::execution::SessionState; use datafusion::execution::context::QueryPlanner; -use datafusion::physical_expr::{LexOrdering, OrderingRequirements}; +use datafusion::physical_expr::expressions::Column; +use datafusion::physical_expr::{ + LexOrdering, OrderingRequirements, PhysicalExpr, PhysicalSortExpr, +}; use datafusion::physical_optimizer::output_requirements::OutputRequirementExec; -use datafusion::physical_plan::ExecutionPlan; use datafusion::physical_plan::sorts::sort::SortExec; +use datafusion::physical_plan::{ExecutionPlan, ExecutionPlanProperties}; use datafusion::physical_planner::{DefaultPhysicalPlanner, ExtensionPlanner, PhysicalPlanner}; -use datafusion_common::{DFSchema, internal_err}; +use datafusion_common::stats::Precision; +use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode}; +use datafusion_common::{DFSchema, Statistics, internal_datafusion_err, internal_err}; +use datafusion_datasource::file_groups::FileGroup; +use datafusion_datasource::file_scan_config::FileScanConfigBuilder; +use datafusion_datasource::source::{DataSource, DataSourceExec}; +use datafusion_datasource::{PartitionedFile, TableSchema}; use datafusion_expr::{Expr, LogicalPlan, UserDefinedLogicalNode}; +use datafusion_physical_expr::projection::{ProjectionExpr, ProjectionExprs}; use datafusion_physical_expr::{Partitioning, create_physical_sort_exprs}; +use sail_cache::remote_checkpoint::RemoteCheckpointRegistry; use sail_catalog_system::planner::SystemTablePhysicalPlanner; +use sail_common_datafusion::extension::SessionExtensionAccessor; use sail_common_datafusion::logical_rewriter::LogicalRewriter; use sail_common_datafusion::rename::physical_plan::rename_projected_physical_plan; use sail_common_datafusion::streaming::event::schema::{ @@ -27,6 +43,9 @@ use sail_logical_plan::barrier::BarrierNode; use sail_logical_plan::map_partitions::MapPartitionsNode; use sail_logical_plan::monotonic_id::MonotonicIdNode; use sail_logical_plan::range::RangeNode; +use sail_logical_plan::remote_checkpoint::{ + RemoteCheckpointCommandNode, RemoteCheckpointRelationNode, +}; use sail_logical_plan::repartition::{ExplicitRepartitionKind, ExplicitRepartitionNode}; use sail_logical_plan::schema_pivot::SchemaPivotNode; use sail_logical_plan::show_string::ShowStringNode; @@ -42,6 +61,10 @@ use sail_physical_plan::catalog_command::CatalogCommandExec; use sail_physical_plan::map_partitions::MapPartitionsExec; use sail_physical_plan::monotonic_id::MonotonicIdExec; use sail_physical_plan::range::RangeExec; +use sail_physical_plan::remote_checkpoint::{ + CheckpointDataSource, RemoteCheckpointCommitExec, RemoteCheckpointWriteExec, + checkpoint_storage_schema, +}; use sail_physical_plan::repartition::ExplicitRepartitionExec; use sail_physical_plan::schema_pivot::SchemaPivotExec; use sail_physical_plan::show_string::ShowStringExec; @@ -99,8 +122,193 @@ impl ExtensionPlanner for ExtensionPhysicalPlanner { session_state: &SessionState, ) -> datafusion_common::Result>> { let plan: Arc = if let Some(node) = - node.as_any().downcast_ref::() + node.as_any().downcast_ref::() { + let [_logical_input] = logical_inputs else { + return internal_err!("RemoteCheckpointCommand requires exactly one logical input"); + }; + let [input] = physical_inputs else { + return internal_err!( + "RemoteCheckpointCommand requires exactly one physical input" + ); + }; + let registry = session_state.extension::()?; + let (object_store_url, prefix) = registry + .resolve_relation(session_state.runtime_env().as_ref(), node.relation_id())?; + let storage_schema = checkpoint_storage_schema(node.logical_schema()); + let output_partitioning = + checkpoint_schema_partitioning(input.output_partitioning(), &storage_schema)?; + let output_ordering = input + .output_ordering() + .map(|ordering| checkpoint_schema_ordering(ordering, &storage_schema)) + .transpose()?; + let writer: Arc = Arc::new(RemoteCheckpointWriteExec::try_new( + Arc::clone(input), + object_store_url.clone(), + prefix.clone(), + Arc::clone(&storage_schema), + )?); + Arc::new(RemoteCheckpointCommitExec::new( + writer, + node.relation_id().to_string(), + object_store_url, + prefix, + Arc::clone(node.logical_schema()), + storage_schema, + output_partitioning, + output_ordering, + )) + } else if let Some(node) = node.as_any().downcast_ref::() { + let registry = session_state.extension::()?; + let descriptor = registry.get(node.relation_id())?.ok_or_else(|| { + datafusion_common::DataFusionError::Plan(format!( + "checkpoint relation is not available: {}", + node.relation_id() + )) + })?; + let logical_schema = UserDefinedLogicalNode::schema(node).inner().clone(); + let schema_matches = descriptor.storage_schema.fields().len() + == logical_schema.fields().len() + && descriptor + .storage_schema + .fields() + .iter() + .zip(logical_schema.fields()) + .all(|(storage, logical)| storage.data_type() == logical.data_type()); + if !schema_matches { + return internal_err!( + "checkpoint storage schema does not match the relation schema by position" + ); + } + let output_partitioning = checkpoint_schema_partitioning( + &descriptor.output_partitioning, + logical_schema.as_ref(), + )?; + let storage_output_ordering = descriptor + .output_ordering + .as_ref() + .map(|ordering| { + checkpoint_schema_ordering(ordering, descriptor.storage_schema.as_ref()) + }) + .transpose()?; + let scan: Arc = if descriptor.storage_schema.fields().is_empty() { + let partitions = descriptor + .partitions + .iter() + .map(|partition| { + let row_count = usize::try_from(partition.row_count).map_err(|_| { + datafusion_common::DataFusionError::Plan( + "checkpoint row count is too large".to_string(), + ) + })?; + let batch = RecordBatch::try_new_with_options( + Arc::clone(&descriptor.storage_schema), + vec![], + &RecordBatchOptions::new().with_row_count(Some(row_count)), + )?; + Ok(vec![batch]) + }) + .collect::>>()?; + let source = MemorySourceConfig::try_new( + &partitions, + Arc::clone(&descriptor.storage_schema), + None, + )? + .try_with_sort_information(storage_output_ordering.clone().into_iter().collect())?; + Arc::new(DataSourceExec::new(Arc::new(CheckpointDataSource::new( + Arc::new(source), + output_partitioning, + )))) + } else { + let parquet_options = TableParquetOptions { + global: session_state.config().options().execution.parquet.clone(), + ..Default::default() + }; + let source = ParquetSource::new(TableSchema::new( + Arc::clone(&descriptor.storage_schema), + vec![], + )) + .with_table_parquet_options(parquet_options); + let file_groups = descriptor + .partitions + .iter() + .map(|partition| { + let files = partition + .file + .as_ref() + .map(|file| { + let mut statistics = + Statistics::new_unknown(descriptor.storage_schema.as_ref()); + statistics.num_rows = Precision::Exact( + usize::try_from(partition.row_count).map_err(|_| { + datafusion_common::DataFusionError::Plan( + "checkpoint row count is too large".to_string(), + ) + })?, + ); + Ok(PartitionedFile::new(file.location.to_string(), file.size) + .with_statistics(Arc::new(statistics))) + }) + .into_iter() + .collect::>>()?; + Ok(FileGroup::new(files)) + }) + .collect::>>()?; + let total_rows = + descriptor + .partitions + .iter() + .try_fold(0_u64, |total, partition| { + total.checked_add(partition.row_count).ok_or_else(|| { + datafusion_common::DataFusionError::Plan( + "checkpoint row count overflow".to_string(), + ) + }) + })?; + let mut statistics = Statistics::new_unknown(descriptor.storage_schema.as_ref()); + statistics.num_rows = + Precision::Exact(usize::try_from(total_rows).map_err(|_| { + datafusion_common::DataFusionError::Plan( + "checkpoint row count is too large".to_string(), + ) + })?); + let scan = FileScanConfigBuilder::new( + descriptor.object_store_url.clone(), + Arc::new(source), + ) + .with_file_groups(file_groups) + .with_statistics(statistics) + .with_output_ordering(storage_output_ordering.into_iter().collect()) + .with_preserve_order(true) + .with_partitioned_by_file_group(true) + .build(); + let projection = descriptor + .storage_schema + .fields() + .iter() + .zip(logical_schema.fields()) + .enumerate() + .map(|(index, (storage, logical))| { + ProjectionExpr::new( + Arc::new(Column::new(storage.name(), index)) as Arc, + logical.name(), + ) + }) + .collect::(); + let source = scan + .try_swapping_with_projection(&projection)? + .ok_or_else(|| { + internal_datafusion_err!( + "checkpoint Parquet source rejected its column projection" + ) + })?; + Arc::new(DataSourceExec::new(Arc::new(CheckpointDataSource::new( + source, + output_partitioning, + )))) + }; + scan + } else if let Some(node) = node.as_any().downcast_ref::() { let schema = UserDefinedLogicalNode::schema(node).inner().clone(); let projection = (0..schema.fields().len()).collect(); Arc::new(RangeExec::try_new( @@ -283,6 +491,66 @@ impl ExtensionPlanner for ExtensionPhysicalPlanner { } } +// Column ordinals survive logical/storage renames; names may be duplicated or synthetic. +fn checkpoint_schema_partitioning( + partitioning: &Partitioning, + schema: &datafusion::arrow::datatypes::Schema, +) -> datafusion_common::Result { + match partitioning { + Partitioning::RoundRobinBatch(partitions) => Ok(Partitioning::RoundRobinBatch(*partitions)), + Partitioning::Hash(expressions, partitions) => Ok(Partitioning::Hash( + expressions + .iter() + .map(|expression| checkpoint_schema_expression(Arc::clone(expression), schema)) + .collect::>>()?, + *partitions, + )), + Partitioning::UnknownPartitioning(partitions) => { + Ok(Partitioning::UnknownPartitioning(*partitions)) + } + } +} + +fn checkpoint_schema_ordering( + ordering: &LexOrdering, + schema: &datafusion::arrow::datatypes::Schema, +) -> datafusion_common::Result { + let expressions = ordering + .iter() + .map(|sort| { + Ok(PhysicalSortExpr::new( + checkpoint_schema_expression(Arc::clone(&sort.expr), schema)?, + sort.options, + )) + }) + .collect::>>()?; + LexOrdering::new(expressions) + .ok_or_else(|| internal_datafusion_err!("checkpoint output ordering cannot be empty")) +} + +fn checkpoint_schema_expression( + expression: Arc, + schema: &datafusion::arrow::datatypes::Schema, +) -> datafusion_common::Result> { + expression + .transform_down(|expression| { + let Some(column) = expression.downcast_ref::() else { + return Ok(Transformed::no(expression)); + }; + let field = schema.fields().get(column.index()).ok_or_else(|| { + internal_datafusion_err!( + "checkpoint property references column {} at invalid index {}", + column.name(), + column.index() + ) + })?; + Ok(Transformed::yes( + Arc::new(Column::new(field.name(), column.index())) as Arc, + )) + }) + .data() +} + /// Plans the explicit repartitioning emitted by the logical planner. /// Empty expressions keep round-robin explicit here, while hash repartitioning /// and `UnknownPartitioning(1)` are normalized later by `RewriteExplicitRepartition`. @@ -342,11 +610,20 @@ mod tests { use std::sync::Arc; use datafusion::arrow::datatypes::{DataType, Field, Schema}; + use datafusion::execution::SessionStateBuilder; + use datafusion::execution::runtime_env::RuntimeEnv; + use datafusion::physical_expr::expressions::Column; + use datafusion::physical_expr::{PhysicalExpr, PhysicalSortExpr}; + use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec; + use datafusion::physical_plan::collect; use datafusion::physical_plan::empty::EmptyExec; use datafusion::physical_plan::repartition::RepartitionExec; - use datafusion::prelude::SessionContext; + use datafusion::prelude::{SessionConfig, SessionContext}; use datafusion_common::ToDFSchema; - use datafusion_expr::col; + use datafusion_expr::{LogicalPlanBuilder, col}; + use object_store::memory::InMemory; + use sail_common::spec; + use sail_plan::config::PlanConfig; use super::*; @@ -354,6 +631,221 @@ mod tests { Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)])) } + #[test] + fn checkpoint_properties_map_columns_to_target_schema_by_position() { + let target_schema = Schema::new(vec![ + Field::new("_c0", DataType::Int64, false), + Field::new("_c1", DataType::Int64, false), + ]); + let first = Arc::new(Column::new("value", 0)) as Arc; + let second = Arc::new(Column::new("value", 1)) as Arc; + let ordering = LexOrdering::new([PhysicalSortExpr::new_default(first)]).unwrap(); + + let partitioning = + checkpoint_schema_partitioning(&Partitioning::Hash(vec![second], 2), &target_schema) + .unwrap(); + let ordering = checkpoint_schema_ordering(&ordering, &target_schema).unwrap(); + + let Partitioning::Hash(expressions, 2) = partitioning else { + panic!("expected hash partitioning"); + }; + let partition_column = expressions[0].downcast_ref::().unwrap(); + assert_eq!(partition_column.name(), "_c1"); + assert_eq!(partition_column.index(), 1); + let ordering_column = ordering.first().expr.downcast_ref::().unwrap(); + assert_eq!(ordering_column.name(), "_c0"); + assert_eq!(ordering_column.index(), 0); + } + + #[tokio::test] + async fn checkpoint_command_optimizer_coalesces_writer_before_commit() + -> datafusion_common::Result<()> { + let registry = Arc::new(RemoteCheckpointRegistry::new(Some( + "memory:///checkpoint".to_string(), + ))); + let config = SessionConfig::new().with_extension(registry); + let runtime_env = Arc::new(RuntimeEnv::default()); + let object_store_url = + datafusion::execution::object_store::ObjectStoreUrl::parse("memory://")?; + runtime_env.register_object_store(object_store_url.as_ref(), Arc::new(InMemory::new())); + let state = SessionStateBuilder::new() + .with_config(config) + .with_runtime_env(runtime_env) + .with_default_features() + .with_query_planner(new_query_planner()) + .build(); + let context = SessionContext::new_with_state(state); + let command = spec::Plan::Command(spec::CommandPlan::new( + spec::CommandNode::RemoteCheckpoint { + relation_id: "relation".to_string(), + input: Box::new(spec::QueryPlan::new(spec::QueryNode::Range(spec::Range { + start: Some(0), + end: 10, + step: 1, + num_partitions: Some(2), + }))), + }, + )); + + let (physical, _) = + sail_plan::resolve_and_execute_plan(&context, Arc::new(PlanConfig::default()), command) + .await + .map_err(|error| internal_datafusion_err!("{error}"))?; + + let commit = physical + .downcast_ref::() + .ok_or_else(|| internal_datafusion_err!("checkpoint command did not plan a commit"))?; + let coalesce = commit + .input() + .downcast_ref::() + .ok_or_else(|| internal_datafusion_err!("checkpoint commit input is not coalesced"))?; + assert!(coalesce.input().is::()); + Ok(()) + } + + #[tokio::test] + async fn zero_column_checkpoint_scan_preserves_partition_rows() -> datafusion_common::Result<()> + { + let registry = Arc::new(RemoteCheckpointRegistry::new(Some( + "memory:///checkpoint".to_string(), + ))); + registry.insert(sail_cache::remote_checkpoint::RemoteCheckpointDescriptor { + relation_id: "zero-columns".to_string(), + object_store_url: datafusion::execution::object_store::ObjectStoreUrl::parse( + "memory://", + )?, + prefix: object_store::path::Path::from("checkpoint/session/zero-columns"), + logical_schema: Arc::new(Schema::empty()), + storage_schema: Arc::new(Schema::empty()), + output_partitioning: Partitioning::RoundRobinBatch(2), + output_ordering: None, + partitions: vec![ + sail_cache::remote_checkpoint::RemoteCheckpointPartition { + partition: 0, + row_count: 2, + file: None, + }, + sail_cache::remote_checkpoint::RemoteCheckpointPartition { + partition: 1, + row_count: 3, + file: None, + }, + ], + })?; + let config = SessionConfig::new().with_extension(registry); + let state = SessionStateBuilder::new() + .with_config(config) + .with_default_features() + .with_query_planner(new_query_planner()) + .build(); + let context = SessionContext::new_with_state(state); + let logical = LogicalPlan::Extension(datafusion_expr::Extension { + node: Arc::new(RemoteCheckpointRelationNode::try_new( + "zero-columns".to_string(), + Arc::new(Schema::empty()), + )?), + }); + let state = context.state(); + let physical = state + .query_planner() + .create_physical_plan(&logical, &state) + .await?; + + assert!(physical.is::()); + assert!(matches!( + physical.output_partitioning(), + Partitioning::RoundRobinBatch(2) + )); + let batches = collect(physical, context.task_ctx()).await?; + assert_eq!(batches.iter().map(RecordBatch::num_rows).sum::(), 5); + assert!(batches.iter().all(|batch| batch.num_columns() == 0)); + Ok(()) + } + + #[tokio::test] + async fn checkpoint_scan_uses_data_source_properties() -> datafusion_common::Result<()> { + let logical_schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)])); + let storage_schema = checkpoint_storage_schema(&logical_schema); + let storage_key = Arc::new(Column::new("_c0", 0)) as Arc; + let registry = Arc::new(RemoteCheckpointRegistry::new(Some( + "memory:///checkpoint".to_string(), + ))); + registry.insert(sail_cache::remote_checkpoint::RemoteCheckpointDescriptor { + relation_id: "relation".to_string(), + object_store_url: datafusion::execution::object_store::ObjectStoreUrl::parse( + "memory://", + )?, + prefix: object_store::path::Path::from("checkpoint/session/relation"), + logical_schema: Arc::clone(&logical_schema), + storage_schema, + output_partitioning: Partitioning::Hash(vec![Arc::clone(&storage_key)], 2), + output_ordering: LexOrdering::new([PhysicalSortExpr::new_default(storage_key)]), + partitions: (0..2) + .map( + |partition| sail_cache::remote_checkpoint::RemoteCheckpointPartition { + partition, + row_count: 0, + file: None, + }, + ) + .collect(), + })?; + let config = SessionConfig::new().with_extension(registry); + let state = SessionStateBuilder::new() + .with_config(config) + .with_default_features() + .with_query_planner(new_query_planner()) + .build(); + let context = SessionContext::new_with_state(state); + let logical = LogicalPlan::Extension(datafusion_expr::Extension { + node: Arc::new(RemoteCheckpointRelationNode::try_new( + "relation".to_string(), + logical_schema, + )?), + }); + let state = context.state(); + let physical = state + .query_planner() + .create_physical_plan(&logical, &state) + .await?; + + assert!(physical.is::()); + let Partitioning::Hash(expressions, 2) = physical.output_partitioning() else { + return internal_err!("checkpoint scan lost its hash partitioning"); + }; + let partition_column = expressions[0] + .downcast_ref::() + .ok_or_else(|| internal_datafusion_err!("checkpoint hash key is not a column"))?; + assert_eq!(partition_column.name(), "id"); + assert_eq!(partition_column.index(), 0); + let ordering = physical + .output_ordering() + .ok_or_else(|| internal_datafusion_err!("checkpoint scan lost its ordering"))?; + let ordering_column = ordering[0] + .expr + .downcast_ref::() + .ok_or_else(|| internal_datafusion_err!("checkpoint ordering key is not a column"))?; + assert_eq!(ordering_column.name(), "id"); + assert_eq!(ordering_column.index(), 0); + + let projected_logical = LogicalPlanBuilder::from(logical) + .project(vec![col("id").alias("key")])? + .build()?; + let projected = state + .query_planner() + .create_physical_plan(&projected_logical, &state) + .await?; + let Partitioning::Hash(expressions, 2) = projected.output_partitioning() else { + return internal_err!("checkpoint projection lost its hash partitioning"); + }; + let partition_column = expressions[0] + .downcast_ref::() + .ok_or_else(|| internal_datafusion_err!("projected hash key is not a column"))?; + assert_eq!(partition_column.name(), "key"); + assert_eq!(partition_column.index(), 0); + Ok(()) + } + fn plan_partitioning( input: &dyn ExecutionPlan, schema: &Arc, diff --git a/crates/sail-session/src/session_factory/server.rs b/crates/sail-session/src/session_factory/server.rs index 3cc0d453a..eb6698057 100644 --- a/crates/sail-session/src/session_factory/server.rs +++ b/crates/sail-session/src/session_factory/server.rs @@ -8,6 +8,7 @@ use datafusion::execution::{SessionState, SessionStateBuilder}; use datafusion::functions_aggregate::first_last::first_value_udaf; use datafusion::prelude::{SessionConfig, SessionContext}; use datafusion_expr::registry::FunctionRegistry; +use sail_cache::remote_checkpoint::RemoteCheckpointRegistry; use sail_catalog::provider::CatalogCacheManager; use sail_catalog_system::service::SystemTableService; use sail_common::config::AppConfig; @@ -116,6 +117,9 @@ impl ServerSessionFactory { )?)) .with_extension(Arc::new(ActivityTracker::new())) .with_extension(Arc::new(JobService::new(job_runner))) + .with_extension(Arc::new(RemoteCheckpointRegistry::new( + self.config.execution.checkpoint.path.clone(), + ))) .with_extension(Arc::new(RepartitionBufferConfig::new( self.config.cluster.task_stream_buffer, ))) diff --git a/crates/sail-session/src/session_manager/actor/handler.rs b/crates/sail-session/src/session_manager/actor/handler.rs index d5b18e55d..0ace99e33 100644 --- a/crates/sail-session/src/session_manager/actor/handler.rs +++ b/crates/sail-session/src/session_manager/actor/handler.rs @@ -5,6 +5,7 @@ use datafusion::prelude::SessionContext; use fastrace::Span; use fastrace::collector::SpanContext; use log::{info, warn}; +use sail_cache::remote_checkpoint::RemoteCheckpointRegistry; use sail_common_datafusion::extension::SessionExtensionAccessor; use sail_common_datafusion::session::activity::ActivityTracker; use sail_common_datafusion::session::job::JobService; @@ -399,11 +400,22 @@ impl SessionManagerActor { warn!("job service not found for session {session_id}"); return; }; + let checkpoint_registry = context.extension::().ok(); + let runtime_env = context.runtime_env(); let handle = ctx.handle().clone(); let (tx, rx) = oneshot::channel(); ctx.spawn(async move { + // Stop tasks before deleting the namespace so late attempts cannot recreate objects. service.runner().stop(tx).await; - let message = match rx.await { + let history = rx.await; + if let Some(checkpoint_registry) = checkpoint_registry + && let Err(error) = checkpoint_registry + .cleanup_session(runtime_env.as_ref()) + .await + { + warn!("failed to clean checkpoints for session {session_id}: {error}"); + } + let message = match history { Ok(x) => SessionManagerEvent::SetSessionHistory { session_id, history: SessionHistory { job_runner: x }, diff --git a/crates/sail-spark-connect/src/server.rs b/crates/sail-spark-connect/src/server.rs index 6d2317793..3032f8d1b 100644 --- a/crates/sail-spark-connect/src/server.rs +++ b/crates/sail-spark-connect/src/server.rs @@ -100,8 +100,9 @@ async fn handle_command( CommandType::CheckpointCommand(checkpoint) => { service::handle_execute_checkpoint_command(ctx, checkpoint, metadata).await } - CommandType::RemoveCachedRemoteRelationCommand(_) => { - Err(SparkError::todo("remove cached remote relation command")) + CommandType::RemoveCachedRemoteRelationCommand(command) => { + service::handle_execute_remove_cached_remote_relation_command(ctx, command, metadata) + .await } CommandType::MergeIntoTableCommand(command) => { service::handle_execute_merge_into_table_command(ctx, command, metadata).await diff --git a/crates/sail-spark-connect/src/service/plan_executor.rs b/crates/sail-spark-connect/src/service/plan_executor.rs index d372544a6..8c7e67ab8 100644 --- a/crates/sail-spark-connect/src/service/plan_executor.rs +++ b/crates/sail-spark-connect/src/service/plan_executor.rs @@ -26,10 +26,11 @@ use crate::spark::connect::execute_plan_response::{ ResponseType, ResultComplete, SqlCommandResult, }; use crate::spark::connect::{ - CheckpointCommand, CheckpointCommandResult, CommonInlineUserDefinedDataSource, - CommonInlineUserDefinedFunction, CommonInlineUserDefinedTableFunction, - CreateDataFrameViewCommand, ExecutePlanResponse, GetResourcesCommand, LocalRelation, - MergeIntoTableCommand, Relation, SqlCommand, StreamingQueryCommand, + CachedRemoteRelation, CheckpointCommand, CheckpointCommandResult, + CommonInlineUserDefinedDataSource, CommonInlineUserDefinedFunction, + CommonInlineUserDefinedTableFunction, CreateDataFrameViewCommand, ExecutePlanResponse, + GetResourcesCommand, LocalRelation, MergeIntoTableCommand, Relation, + RemoveCachedRemoteRelationCommand, SqlCommand, StreamingQueryCommand, StreamingQueryCommandResult, StreamingQueryListenerBusCommand, StreamingQueryManagerCommand, StreamingQueryManagerCommandResult, WriteOperation, WriteOperationV2, WriteStreamOperationStart, WriteStreamOperationStartResult, relation, @@ -515,16 +516,43 @@ pub(crate) async fn handle_execute_streaming_query_listener_bus_command( pub(crate) async fn handle_execute_checkpoint_command( ctx: &SessionContext, - _checkpoint: CheckpointCommand, + checkpoint: CheckpointCommand, metadata: ExecutorMetadata, ) -> SparkResult { - // TODO: Implement - warn!("Checkpoint operation is not yet supported and is a no-op"); let spark = ctx.extension::()?; - let result = CheckpointCommandResult { relation: None }; - let mut output = vec![ExecutorOutput::new(ExecutorBatch::CheckpointCommandResult( - Box::new(result), - ))]; + let CheckpointCommand { + relation, + local: _, + eager, + storage_level, + } = checkpoint; + if !eager { + return Err(SparkError::unsupported("lazy DataFrame checkpoint")); + } + if storage_level.is_some() { + return Err(SparkError::unsupported( + "checkpoint StorageLevel; Sail checkpoints are object-store backed", + )); + } + let relation = relation.required("checkpoint relation")?; + let query: spec::QueryPlan = relation.try_into()?; + let relation_id = uuid::Uuid::new_v4().to_string(); + let plan = spec::Plan::Command(spec::CommandPlan::new( + spec::CommandNode::RemoteCheckpoint { + relation_id: relation_id.clone(), + input: Box::new(query), + }, + )); + let (plan, _) = resolve_and_execute_plan(ctx, spark.plan_config()?, plan).await?; + let service = ctx.extension::()?; + let stream = service.runner().execute(ctx, plan).await?; + let _ = read_stream(stream).await?; + let result = ExecutorOutput::new(ExecutorBatch::CheckpointCommandResult(Box::new( + CheckpointCommandResult { + relation: Some(CachedRemoteRelation { relation_id }), + }, + ))); + let mut output = vec![result]; if metadata.reattachable { output.push(ExecutorOutput::complete()); } @@ -535,6 +563,26 @@ pub(crate) async fn handle_execute_checkpoint_command( )) } +pub(crate) async fn handle_execute_remove_cached_remote_relation_command( + ctx: &SessionContext, + _command: RemoveCachedRemoteRelationCommand, + metadata: ExecutorMetadata, +) -> SparkResult { + let spark = ctx.extension::()?; + // TODO: Remove checkpoint data on a best-effort basis when the client releases the relation. + // Checkpoints have session lifetime so plans can safely share their immutable relation ID. + // The registry and object-store namespace are cleared together after the job runner stops. + let output = metadata + .reattachable + .then(ExecutorOutput::complete) + .into_iter(); + Ok(ExecutePlanResponseStream::new( + spark.session_id().to_string(), + metadata.operation_id, + Box::pin(stream::iter(output)), + )) +} + pub(crate) async fn handle_interrupt_all(ctx: &SessionContext) -> SparkResult> { let spark = ctx.extension::()?; let mut results = vec![]; diff --git a/python/pysail/testing/spark/steps/plan.py b/python/pysail/testing/spark/steps/plan.py index 059e48f00..f37e5005f 100644 --- a/python/pysail/testing/spark/steps/plan.py +++ b/python/pysail/testing/spark/steps/plan.py @@ -74,6 +74,20 @@ def normalize_path(path: str) -> str: text, ) text = pytest_tmp_prefix.sub(lambda m: f"{m.group(1)}/", text) + # Preserve checkpoint partition layout while removing generated identifiers. + checkpoint_uuid = r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}" + text = re.sub( + rf"(__checkpoint_testing__/){checkpoint_uuid}/{checkpoint_uuid}", + r"\1/", + text, + flags=re.IGNORECASE, + ) + text = re.sub( + rf"(__checkpoint_testing__///part-\d{{5}}-){checkpoint_uuid}(\.parquet)", + r"\1\2", + text, + flags=re.IGNORECASE, + ) # Normalize Delta Lake parquet files: part---c.snappy.parquet text = re.sub( r"part-\d+-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}-c\d+\.snappy\.parquet", diff --git a/python/pysail/tests/spark/dataframe/__snapshots__/test_checkpoint.plan.yaml b/python/pysail/tests/spark/dataframe/__snapshots__/test_checkpoint.plan.yaml new file mode 100644 index 000000000..6bc24c73c --- /dev/null +++ b/python/pysail/tests/spark/dataframe/__snapshots__/test_checkpoint.plan.yaml @@ -0,0 +1,59 @@ +# serializer version: 1 +--- +name: "test_checkpoint_preserves_partitioning_and_ordering[checkpoint]" +data: + |- + == Physical Plan == + ProjectionExec: expr=[#2@0 as key, #3@1 as count] + ProjectionExec: expr=[#1@0 as #2, count(Int32(1))@1 as #3] + AggregateExec: mode=SinglePartitioned, gby=[#1@0 as #1], aggr=[count(Int32(1))], ordering_mode=Sorted + ProjectionExec: expr=[#1@1 as #1] + DataSourceExec: file_groups={4 groups: [[/__checkpoint_testing__///part-00000-.parquet], [/__checkpoint_testing__///part-00001-.parquet], [/__checkpoint_testing__///part-00003-.parquet], []]}, projection=[_c0@0 as #0, _c1@1 as #1], file_type=parquet +--- +name: "test_checkpoint_preserves_partitioning_and_ordering[checkpoint].1" +data: + |- + == Physical Plan == + ProjectionExec: expr=[#0@0 as id, #1@1 as key] + DataSourceExec: file_groups={4 groups: [[/__checkpoint_testing__///part-00000-.parquet], [/__checkpoint_testing__///part-00001-.parquet], [/__checkpoint_testing__///part-00003-.parquet], []]}, projection=[_c0@0 as #0, _c1@1 as #1], file_type=parquet +--- +name: "test_checkpoint_preserves_partitioning_and_ordering[local-checkpoint]" +data: + |- + == Physical Plan == + ProjectionExec: expr=[#2@0 as key, #3@1 as count] + ProjectionExec: expr=[#1@0 as #2, count(Int32(1))@1 as #3] + AggregateExec: mode=SinglePartitioned, gby=[#1@0 as #1], aggr=[count(Int32(1))], ordering_mode=Sorted + ProjectionExec: expr=[#1@1 as #1] + DataSourceExec: file_groups={4 groups: [[/__checkpoint_testing__///part-00000-.parquet], [/__checkpoint_testing__///part-00001-.parquet], [/__checkpoint_testing__///part-00003-.parquet], []]}, projection=[_c0@0 as #0, _c1@1 as #1], file_type=parquet +--- +name: "test_checkpoint_preserves_partitioning_and_ordering[local-checkpoint].1" +data: + |- + == Physical Plan == + ProjectionExec: expr=[#0@0 as id, #1@1 as key] + DataSourceExec: file_groups={4 groups: [[/__checkpoint_testing__///part-00000-.parquet], [/__checkpoint_testing__///part-00001-.parquet], [/__checkpoint_testing__///part-00003-.parquet], []]}, projection=[_c0@0 as #0, _c1@1 as #1], file_type=parquet +--- +name: "test_checkpoint_supports_iterative_weakly_connected_components" +data: + |- + == Physical Plan == + ProjectionExec: expr=[#24@0 as node, #25@1 as label] + ProjectionExec: expr=[#0@0 as #24, least(#1@1, CASE WHEN #20@2 IS NOT NULL THEN #20@2 ELSE #1@1 END) as #25] + HashJoinExec: mode=CollectLeft, join_type=Left, on=[(#0@0, #19@0)], projection=[#0@0, #1@1, #20@3] + CoalescePartitionsExec + DataSourceExec: file_groups={4 groups: [[/__checkpoint_testing__///part-00000-.parquet], [/__checkpoint_testing__///part-00001-.parquet], [/__checkpoint_testing__///part-00003-.parquet], []]}, projection=[_c0@0 as #0, _c1@1 as #1], file_type=parquet + ProjectionExec: expr=[#4@0 as #19, min(#13)@1 as #20] + AggregateExec: mode=FinalPartitioned, gby=[#4@0 as #4], aggr=[min(#13)] + RepartitionExec: partitioning=Hash([#4@0], 4), input_partitions=4 + AggregateExec: mode=Partial, gby=[#4@0 as #4], aggr=[min(#13)] + RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=2 + HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(#12@0, #5@1)], projection=[#4@2, #13@1] + CoalescePartitionsExec + ProjectionExec: expr=[#10@0 as #12, #11@1 as #13] + DataSourceExec: file_groups={4 groups: [[/__checkpoint_testing__///part-00000-.parquet], [/__checkpoint_testing__///part-00001-.parquet], [/__checkpoint_testing__///part-00003-.parquet], []]}, projection=[_c0@0 as #10, _c1@1 as #11], file_type=parquet + UnionExec + ProjectionExec: expr=[a@0 as #4, b@1 as #5] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[b@1 as #4, a@0 as #5] + DataSourceExec: partitions=1, partition_sizes=[1] diff --git a/python/pysail/tests/spark/dataframe/test_checkpoint.py b/python/pysail/tests/spark/dataframe/test_checkpoint.py new file mode 100644 index 000000000..5852f8b54 --- /dev/null +++ b/python/pysail/tests/spark/dataframe/test_checkpoint.py @@ -0,0 +1,169 @@ +import shutil + +import pytest +from pyspark import StorageLevel +from pyspark.sql import functions as sf + +from pysail.testing.spark.session import spark_connect_server +from pysail.testing.spark.steps.plan import normalize_plan_text +from pysail.testing.spark.utils.common import pyspark_version + +pytestmark = pytest.mark.skipif( + pyspark_version() < (4,), + reason="checkpoint and localCheckpoint require PySpark Connect 4+", +) + + +@pytest.fixture(scope="module") +def checkpoint_root(tmp_path_factory): + root = tmp_path_factory.mktemp("object-store-checkpoints") / "__checkpoint_testing__" + root.mkdir() + return root + + +@pytest.fixture(scope="module") +def remote(checkpoint_root): + with spark_connect_server(envs={"SAIL_EXECUTION__CHECKPOINT__PATH": checkpoint_root.as_uri()}) as server: + yield server.remote + + +@pytest.mark.parametrize("local", [False, True], ids=["checkpoint", "local-checkpoint"]) +def test_eager_checkpoint_is_an_object_store_snapshot(spark, checkpoint_root, tmp_path, local): + source_path = tmp_path / "source" + spark.createDataFrame([(1, "a"), (2, "b"), (3, "c")], "id INT, value STRING").write.parquet(str(source_path)) + source = spark.read.parquet(str(source_path)).where("id <= 2") + files_before = {path for path in checkpoint_root.rglob("*") if path.is_file()} + + checkpointed = source.localCheckpoint() if local else source.checkpoint() + + checkpoint_files = {path for path in checkpoint_root.rglob("*") if path.is_file()} - files_before + assert checkpoint_files + assert all(path.suffix == ".parquet" for path in checkpoint_files) + + shutil.rmtree(source_path) + assert checkpointed.orderBy("id").collect() == [(1, "a"), (2, "b")] + assert checkpointed.where("id = 2").select("value").collect() == [("b",)] + + +def test_checkpoint_preserves_duplicate_column_names(spark): + checkpointed = ( + spark.range(3, numPartitions=2) + .selectExpr( + "id AS value", + "id + 10 AS value", + ) + .checkpoint() + ) + + assert [field.name for field in checkpointed.schema.fields] == ["value", "value"] + assert sorted(tuple(row) for row in checkpointed.collect()) == [ + (0, 10), + (1, 11), + (2, 12), + ] + + +def test_checkpoint_preserves_rows_with_no_columns(spark): + checkpointed = spark.range(3, numPartitions=2).select().checkpoint() + + assert checkpointed.schema.simpleString() == "struct<>" + assert checkpointed.collect() == [(), (), ()] + + +def test_checkpoint_preserves_field_metadata(spark): + checkpointed = ( + spark.range(1).select(sf.col("id").alias("value", metadata={"source": "checkpoint-test"})).checkpoint() + ) + + assert checkpointed.schema["value"].metadata == {"source": "checkpoint-test"} + + +def _weakly_connected_components(nodes, pairs): + forward = pairs.select(sf.col("a").alias("source"), sf.col("b").alias("target")) + reverse = pairs.select(sf.col("b").alias("source"), sf.col("a").alias("target")) + edges = forward.unionByName(reverse) + labels = nodes.select(sf.col("id").alias("node")).withColumn("label", sf.col("node")) + + for _ in range(100): + neighbor_labels = labels.select( + sf.col("node").alias("target"), + sf.col("label").alias("target_label"), + ) + neighbor_minimum = ( + edges.join(neighbor_labels, on="target") + .groupBy("source") + .agg(sf.min("target_label").alias("neighbor_minimum")) + .select(sf.col("source").alias("node"), "neighbor_minimum") + ) + next_labels = labels.join(neighbor_minimum, on="node", how="left").select( + "node", + sf.least( + "label", + sf.coalesce("neighbor_minimum", "label"), + ).alias("label"), + ) + previous_labels = labels.select("node", sf.col("label").alias("previous_label")) + changed = ( + next_labels.join(previous_labels, on="node") + .where(sf.col("label") != sf.col("previous_label")) + .limit(1) + .count() + ) + labels = next_labels + if changed == 0: + break + labels = labels.checkpoint() + else: + pytest.fail("weakly connected components did not converge") + + return labels + + +def _component_partition(labels): + components = {} + for row in labels.collect(): + components.setdefault(row.label, set()).add(row.node) + return {frozenset(members) for members in components.values()} + + +@pytest.mark.yamlsnapshot(group="plan") +def test_checkpoint_supports_iterative_weakly_connected_components(spark, snapshot): + nodes = spark.range(7) + pairs = spark.createDataFrame( + [(0, 1), (1, 2), (2, 3), (3, 4), (4, 5)], + "a LONG, b LONG", + ) + + expected = {frozenset(range(6)), frozenset({6})} + checkpointed = _weakly_connected_components(nodes, pairs) + checkpointed_plan = normalize_plan_text(checkpointed._explain_string()) # noqa: SLF001 + + assert _component_partition(checkpointed) == expected + assert checkpointed_plan == snapshot + + +@pytest.mark.parametrize("local", [False, True], ids=["checkpoint", "local-checkpoint"]) +@pytest.mark.yamlsnapshot(group="plan") +def test_checkpoint_preserves_partitioning_and_ordering(spark, local, snapshot): + source = ( + spark.range(100, numPartitions=4) + .withColumn("key", sf.col("id") % 4) + .repartition(4, "key") + .sortWithinPartitions("key", "id") + ) + + checkpointed = source.localCheckpoint() if local else source.checkpoint() + aggregate_plan = normalize_plan_text(checkpointed.groupBy("key").count()._explain_string()) # noqa: SLF001 + ordered_plan = normalize_plan_text(checkpointed.sortWithinPartitions("key", "id")._explain_string()) # noqa: SLF001 + + assert aggregate_plan == snapshot + assert ordered_plan == snapshot + + +def test_checkpoint_rejects_unimplemented_fallback_semantics(spark): + source = spark.range(3) + + with pytest.raises(Exception, match="lazy DataFrame checkpoint"): + source.checkpoint(eager=False) + with pytest.raises(Exception, match="StorageLevel"): + source.localCheckpoint(storageLevel=StorageLevel.DISK_ONLY) From 33ecb399f597b0f4cf422c6045c50342e567fbe5 Mon Sep 17 00:00:00 2001 From: Heran Lin Date: Wed, 29 Jul 2026 15:15:25 +0800 Subject: [PATCH 093/131] feat: blocking shuffle via object storage (#2232) --- Cargo.lock | 1 + crates/sail-common/src/config/application.rs | 79 +++- .../sail-common/src/config/application.yaml | 34 ++ crates/sail-execution/Cargo.toml | 1 + .../proto/sail/task/common.proto | 3 - .../sail-execution/src/driver/actor/core.rs | 8 +- .../src/driver/actor/handler.rs | 19 +- crates/sail-execution/src/driver/event.rs | 10 +- .../src/driver/job_scheduler/core.rs | 36 +- .../src/driver/job_scheduler/mod.rs | 4 + .../src/driver/job_scheduler/options.rs | 5 + .../src/driver/job_scheduler/state.rs | 13 +- crates/sail-execution/src/driver/options.rs | 3 + .../src/driver/worker_pool/core.rs | 1 + .../src/driver/worker_pool/options.rs | 3 + crates/sail-execution/src/job_graph/mod.rs | 7 +- .../sail-execution/src/job_graph/planner.rs | 132 ++++++- crates/sail-execution/src/job_runner.rs | 24 +- crates/sail-execution/src/lib.rs | 1 + crates/sail-execution/src/shuffle.rs | 39 ++ crates/sail-execution/src/stream/reader.rs | 5 +- crates/sail-execution/src/stream/writer.rs | 6 +- .../src/stream_accessor/core.rs | 15 +- .../sail-execution/src/stream_accessor/mod.rs | 32 +- .../sail-execution/src/stream_manager/core.rs | 62 ++- .../sail-execution/src/stream_manager/mod.rs | 3 + .../src/stream_manager/options.rs | 4 + .../src/stream_manager/remote.rs | 357 ++++++++++++++++++ crates/sail-execution/src/task/definition.rs | 22 +- crates/sail-execution/src/task_runner/core.rs | 51 +-- .../sail-execution/src/worker/actor/core.rs | 8 +- .../src/worker/actor/handler.rs | 13 +- crates/sail-execution/src/worker/event.rs | 12 +- crates/sail-execution/src/worker/options.rs | 4 + .../src/worker_manager/kubernetes.rs | 44 +++ .../src/worker_manager/options.rs | 2 + crates/sail-telemetry/src/common.rs | 1 - .../test_shuffle_storage.plan.yaml | 98 +++++ .../spark/execution/test_shuffle_storage.py | 90 +++++ 39 files changed, 1088 insertions(+), 164 deletions(-) create mode 100644 crates/sail-execution/src/shuffle.rs create mode 100644 crates/sail-execution/src/stream_manager/remote.rs create mode 100644 python/pysail/tests/spark/execution/__snapshots__/test_shuffle_storage.plan.yaml create mode 100644 python/pysail/tests/spark/execution/test_shuffle_storage.py diff --git a/Cargo.lock b/Cargo.lock index 959fef93e..9d93b3680 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7717,6 +7717,7 @@ dependencies = [ "sail-function", "sail-iceberg", "sail-logical-plan", + "sail-object-store", "sail-physical-plan", "sail-python-udf", "sail-server", diff --git a/crates/sail-common/src/config/application.rs b/crates/sail-common/src/config/application.rs index 8c811cd67..564773efd 100644 --- a/crates/sail-common/src/config/application.rs +++ b/crates/sail-common/src/config/application.rs @@ -210,6 +210,7 @@ pub struct ClusterConfig { pub task_stream_creation_timeout_secs: u64, pub task_max_attempts: usize, pub rpc_retry_strategy: RetryStrategy, + pub shuffle_backend: ShuffleBackend, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -294,6 +295,78 @@ mod retry_strategy { } } +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde( + into = "shuffle_backend::ShuffleBackend", + from = "shuffle_backend::ShuffleBackend" +)] +pub enum ShuffleBackend { + Streaming, + Storage(StorageShuffleBackend), +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct StorageShuffleBackend { + pub path: String, + pub max_file_size: usize, + pub compression: ShuffleCompression, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ShuffleCompression { + None, + Lz4, + Zstd, +} + +mod shuffle_backend { + use serde::{Deserialize, Serialize}; + + #[derive(Debug, Clone, Serialize, Deserialize)] + #[serde(rename_all = "snake_case")] + pub enum Type { + Streaming, + Storage, + } + + #[derive(Debug, Clone, Serialize, Deserialize)] + #[serde(deny_unknown_fields)] + pub struct ShuffleBackend { + pub r#type: Type, + pub storage: super::StorageShuffleBackend, + } + + impl From for super::ShuffleBackend { + fn from(value: ShuffleBackend) -> Self { + match value.r#type { + Type::Streaming => super::ShuffleBackend::Streaming, + Type::Storage => super::ShuffleBackend::Storage(value.storage), + } + } + } + + impl From for ShuffleBackend { + fn from(value: super::ShuffleBackend) -> Self { + match value { + super::ShuffleBackend::Streaming => ShuffleBackend { + r#type: Type::Streaming, + storage: super::StorageShuffleBackend { + path: String::new(), + max_file_size: 0, + compression: super::ShuffleCompression::None, + }, + }, + super::ShuffleBackend::Storage(storage) => ShuffleBackend { + r#type: Type::Storage, + storage, + }, + } + } + } +} + #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct ExecutionConfig { @@ -522,7 +595,7 @@ pub struct OptimizerConfig { pub expand_views_at_output: bool, } -#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Clone, Default, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum OneLakeApi { Delta, @@ -688,5 +761,9 @@ impl ClusterConfigEnv { TASK_STREAM_BUFFER, TASK_STREAM_CREATION_TIMEOUT_SECS, RPC_RETRY_STRATEGY, + SHUFFLE_BACKEND__TYPE, + SHUFFLE_BACKEND__STORAGE__PATH, + SHUFFLE_BACKEND__STORAGE__MAX_FILE_SIZE, + SHUFFLE_BACKEND__STORAGE__COMPRESSION, } } diff --git a/crates/sail-common/src/config/application.yaml b/crates/sail-common/src/config/application.yaml index a5e72742a..9660e6a5e 100644 --- a/crates/sail-common/src/config/application.yaml +++ b/crates/sail-common/src/config/application.yaml @@ -250,6 +250,40 @@ when using the `exponential_backoff` retry strategy. experimental: true +- key: cluster.shuffle_backend.type + type: string + default: "streaming" + description: | + The shuffle backend type. Valid values are `streaming` and `storage`. + The `streaming` shuffle backend streams shuffle data directly among tasks. + The `storage` shuffle backend materializes merged shuffle partitions in object storage + before running dependent stages. + experimental: true + +- key: cluster.shuffle_backend.storage.path + type: string + default: "file:///tmp/sail/shuffle" + description: | + The object storage base URL for storage-backed shuffle data. + This is only used when `cluster.shuffle_backend.type` is `storage`. + experimental: true + +- key: cluster.shuffle_backend.storage.max_file_size + type: number + default: "134217728" + description: | + The maximum size in bytes of each storage-backed shuffle file. Larger partitions are + split into multiple Arrow IPC files. + experimental: true + +- key: cluster.shuffle_backend.storage.compression + type: string + default: "zstd" + description: | + Arrow IPC compression for storage-backed shuffle files. Valid values are `none`, + `lz4`, and `zstd`. + experimental: true + - key: execution.batch_size type: number default: "8192" diff --git a/crates/sail-execution/Cargo.toml b/crates/sail-execution/Cargo.toml index ea3a17c4e..e46324999 100644 --- a/crates/sail-execution/Cargo.toml +++ b/crates/sail-execution/Cargo.toml @@ -20,6 +20,7 @@ sail-logical-plan = { path = "../sail-logical-plan" } sail-physical-plan = { path = "../sail-physical-plan" } sail-telemetry = { path = "../sail-telemetry" } sail-catalog-system = { path = "../sail-catalog-system" } +sail-object-store = { path = "../sail-object-store" } thiserror = { workspace = true } tokio = { workspace = true } diff --git a/crates/sail-execution/proto/sail/task/common.proto b/crates/sail-execution/proto/sail/task/common.proto index 6d15940cf..2adeaa946 100644 --- a/crates/sail-execution/proto/sail/task/common.proto +++ b/crates/sail-execution/proto/sail/task/common.proto @@ -52,7 +52,6 @@ message TaskInputWorkerKeyList { } message TaskInputRemoteLocator { - string uri = 1; uint64 stage = 2; repeated TaskInputRemoteKeyList keys = 3; } @@ -105,6 +104,4 @@ message TaskOutputLocalLocator { } message TaskOutputRemoteLocator { - string uri = 1; } - diff --git a/crates/sail-execution/src/driver/actor/core.rs b/crates/sail-execution/src/driver/actor/core.rs index b4b4aca7d..1c0caf24b 100644 --- a/crates/sail-execution/src/driver/actor/core.rs +++ b/crates/sail-execution/src/driver/actor/core.rs @@ -88,11 +88,11 @@ impl Actor for DriverActor { result, } => self.handle_create_local_stream(ctx, key, storage, schema, result), DriverEvent::CreateRemoteStream { - uri, key, schema, + context, result, - } => self.handle_create_remote_stream(ctx, uri, key, schema, result), + } => self.handle_create_remote_stream(ctx, key, schema, context, result), DriverEvent::FetchDriverStream { key, result } => { self.handle_fetch_driver_stream(ctx, key, result) } @@ -103,11 +103,11 @@ impl Actor for DriverActor { result, } => self.handle_fetch_worker_stream(ctx, worker_id, key, schema, result), DriverEvent::FetchRemoteStream { - uri, key, schema, + context, result, - } => self.handle_fetch_remote_stream(ctx, uri, key, schema, result), + } => self.handle_fetch_remote_stream(ctx, key, schema, context, result), DriverEvent::ObserveState { observer } => self.handle_observe_state(ctx, observer), DriverEvent::Shutdown { history } => self.handle_shutdown(ctx, history), } diff --git a/crates/sail-execution/src/driver/actor/handler.rs b/crates/sail-execution/src/driver/actor/handler.rs index 62a5c8839..5ec0164ca 100644 --- a/crates/sail-execution/src/driver/actor/handler.rs +++ b/crates/sail-execution/src/driver/actor/handler.rs @@ -303,12 +303,15 @@ impl DriverActor { pub(super) fn handle_create_remote_stream( &mut self, _ctx: &mut ActorContext, - uri: String, key: TaskStreamKey, schema: SchemaRef, + context: Arc, result: oneshot::Sender>>, ) -> ActorAction { - let _ = result.send(self.stream_manager.create_remote_stream(uri, key, schema)); + let _ = result.send( + self.stream_manager + .create_remote_stream(key, schema, &context), + ); ActorAction::Continue } @@ -340,14 +343,14 @@ impl DriverActor { pub(super) fn handle_fetch_remote_stream( &mut self, ctx: &mut ActorContext, - uri: String, key: TaskStreamKey, schema: SchemaRef, + context: Arc, result: oneshot::Sender>, ) -> ActorAction { let _ = result.send( self.stream_manager - .fetch_remote_stream(ctx, uri, &key, schema), + .fetch_remote_stream(ctx, &key, schema, &context), ); ActorAction::Continue } @@ -497,10 +500,14 @@ impl DriverActor { handle.send(JobOutputItem::Error { cause }).await; }); } - JobAction::CleanUpJob { job_id, stage } => { + JobAction::CleanUpJob { + job_id, + stage, + context, + } => { if self.task_assigner.untrack_remote_streams(job_id, stage) { self.stream_manager - .remove_remote_streams(ctx, job_id, stage); + .remove_remote_streams(ctx, job_id, stage, context); } for x in self.task_assigner.untrack_local_streams(job_id, stage) { match x { diff --git a/crates/sail-execution/src/driver/event.rs b/crates/sail-execution/src/driver/event.rs index 1857ac253..c61a16ce6 100644 --- a/crates/sail-execution/src/driver/event.rs +++ b/crates/sail-execution/src/driver/event.rs @@ -75,9 +75,9 @@ pub enum DriverEvent { result: oneshot::Sender>>, }, CreateRemoteStream { - uri: String, key: TaskStreamKey, schema: SchemaRef, + context: Arc, result: oneshot::Sender>>, }, FetchDriverStream { @@ -91,9 +91,9 @@ pub enum DriverEvent { result: oneshot::Sender>, }, FetchRemoteStream { - uri: String, key: TaskStreamKey, schema: SchemaRef, + context: Arc, result: oneshot::Sender>, }, ObserveState { @@ -292,7 +292,6 @@ impl SpanAssociation for DriverEvent { )); } DriverEvent::CreateRemoteStream { - uri, key: TaskStreamKey { job_id, @@ -302,6 +301,7 @@ impl SpanAssociation for DriverEvent { channel, }, schema: _, + context: _, result: _, } => { p.push((SpanAttribute::EXECUTION_JOB_ID, job_id.to_string())); @@ -309,7 +309,6 @@ impl SpanAssociation for DriverEvent { p.push((SpanAttribute::EXECUTION_PARTITION, partition.to_string())); p.push((SpanAttribute::EXECUTION_ATTEMPT, attempt.to_string())); p.push((SpanAttribute::EXECUTION_CHANNEL, channel.to_string())); - p.push((SpanAttribute::EXECUTION_STREAM_REMOTE_URI, uri.clone())); } DriverEvent::FetchDriverStream { key: @@ -349,7 +348,6 @@ impl SpanAssociation for DriverEvent { p.push((SpanAttribute::EXECUTION_CHANNEL, channel.to_string())); } DriverEvent::FetchRemoteStream { - uri, key: TaskStreamKey { job_id, @@ -359,6 +357,7 @@ impl SpanAssociation for DriverEvent { channel, }, schema: _, + context: _, result: _, } => { p.push((SpanAttribute::EXECUTION_JOB_ID, job_id.to_string())); @@ -366,7 +365,6 @@ impl SpanAssociation for DriverEvent { p.push((SpanAttribute::EXECUTION_PARTITION, partition.to_string())); p.push((SpanAttribute::EXECUTION_ATTEMPT, attempt.to_string())); p.push((SpanAttribute::EXECUTION_CHANNEL, channel.to_string())); - p.push((SpanAttribute::EXECUTION_STREAM_REMOTE_URI, uri.clone())); } DriverEvent::ObserveState { observer: _ } => {} DriverEvent::Shutdown { .. } => {} diff --git a/crates/sail-execution/src/driver/job_scheduler/core.rs b/crates/sail-execution/src/driver/job_scheduler/core.rs index 989f1411e..66f02b8ba 100644 --- a/crates/sail-execution/src/driver/job_scheduler/core.rs +++ b/crates/sail-execution/src/driver/job_scheduler/core.rs @@ -49,11 +49,16 @@ impl JobScheduler { "job {job_id} execution plan\n{}", DisplayableExecutionPlan::new(plan.as_ref()).indent(true) ); - let graph = JobGraph::try_new(plan)?; + let graph = JobGraph::try_new( + plan, + crate::job_graph::JobGraphOptions { + use_blocking_shuffle: self.options.use_blocking_shuffle, + }, + )?; debug!("job {job_id} job graph \n{graph}"); let (output, stream) = build_job_output(ctx, job_id, graph.schema().clone()); - let descriptor = JobDescriptor::try_new(graph, JobState::Running { output, context })?; + let descriptor = JobDescriptor::try_new(graph, JobState::Running { output }, context)?; self.jobs.insert(job_id, descriptor); Ok((job_id, stream)) @@ -261,6 +266,7 @@ impl JobScheduler { actions.push(JobAction::CleanUpJob { job_id, stage: Some(s), + context: job.context.clone(), }); } } @@ -489,6 +495,7 @@ impl JobScheduler { actions.push(JobAction::CleanUpJob { job_id, stage: None, + context: job.context.clone(), }); if matches!(job.state, JobState::Draining) { job.state = JobState::Succeeded; @@ -511,12 +518,6 @@ impl JobScheduler { key.job_id ))); }; - let JobState::Running { context, .. } = &job.state else { - return Err(ExecutionError::InvalidArgument(format!( - "job {} is not running", - key.job_id - ))); - }; let Some(stage) = job.graph.stages().get(key.stage) else { return Err(ExecutionError::InvalidArgument(format!( "stage {} not found in job {}", @@ -536,7 +537,7 @@ impl JobScheduler { inputs, output, }; - Ok((definition, context.clone())) + Ok((definition, job.context.clone())) } pub fn stop(&mut self) { @@ -652,14 +653,10 @@ impl JobScheduler { } } }, - OutputMode::Blocking => { - let uri = Err(ExecutionError::InternalError("not implemented".to_string()))?; - TaskInputLocator::Remote { - uri, - stage: input.stage, - keys, - } - } + OutputMode::Blocking => TaskInputLocator::Remote { + stage: input.stage, + keys, + }, }; Ok(TaskInput { locator }) } @@ -696,10 +693,7 @@ impl JobScheduler { }; let locator = match stage.mode { OutputMode::Pipelined => TaskOutputLocator::Local { replicas }, - OutputMode::Blocking => { - let uri = Err(ExecutionError::InternalError("not implemented".to_string()))?; - TaskOutputLocator::Remote { uri } - } + OutputMode::Blocking => TaskOutputLocator::Remote, }; Ok(TaskOutput { distribution, diff --git a/crates/sail-execution/src/driver/job_scheduler/mod.rs b/crates/sail-execution/src/driver/job_scheduler/mod.rs index 76df8075d..82d0a8812 100644 --- a/crates/sail-execution/src/driver/job_scheduler/mod.rs +++ b/crates/sail-execution/src/driver/job_scheduler/mod.rs @@ -4,7 +4,10 @@ mod options; mod state; mod topology; +use std::sync::Arc; + use datafusion::arrow::datatypes::SchemaRef; +use datafusion::execution::TaskContext; use datafusion_proto::physical_plan::PhysicalExtensionCodec; use indexmap::IndexMap; pub use options::JobSchedulerOptions; @@ -55,5 +58,6 @@ pub enum JobAction { CleanUpJob { job_id: JobId, stage: Option, + context: Arc, }, } diff --git a/crates/sail-execution/src/driver/job_scheduler/options.rs b/crates/sail-execution/src/driver/job_scheduler/options.rs index 47edebff9..1612081bf 100644 --- a/crates/sail-execution/src/driver/job_scheduler/options.rs +++ b/crates/sail-execution/src/driver/job_scheduler/options.rs @@ -8,6 +8,7 @@ use crate::driver::DriverOptions; pub struct JobSchedulerOptions { pub task_launch_timeout: Duration, pub task_max_attempts: usize, + pub use_blocking_shuffle: bool, } impl From<&DriverOptions> for JobSchedulerOptions { @@ -15,6 +16,10 @@ impl From<&DriverOptions> for JobSchedulerOptions { Self { task_launch_timeout: options.task_launch_timeout, task_max_attempts: options.task_max_attempts, + use_blocking_shuffle: matches!( + &options.shuffle_backend, + crate::shuffle::ShuffleBackendKind::Storage { .. } + ), } } } diff --git a/crates/sail-execution/src/driver/job_scheduler/state.rs b/crates/sail-execution/src/driver/job_scheduler/state.rs index f744a3ad1..122b99eb0 100644 --- a/crates/sail-execution/src/driver/job_scheduler/state.rs +++ b/crates/sail-execution/src/driver/job_scheduler/state.rs @@ -14,6 +14,7 @@ use crate::job_graph::JobGraph; /// Tracks graph/topology and runtime state for a single job. pub struct JobDescriptor { + pub context: Arc, pub graph: JobGraph, pub topology: JobTopology, pub stages: Vec, @@ -24,10 +25,7 @@ pub struct JobDescriptor { } pub enum JobState { - Running { - output: JobOutputManager, - context: Arc, - }, + Running { output: JobOutputManager }, Draining, Succeeded, Failed, @@ -47,7 +45,11 @@ impl JobState { } impl JobDescriptor { - pub fn try_new(graph: JobGraph, state: JobState) -> ExecutionResult { + pub fn try_new( + graph: JobGraph, + state: JobState, + context: Arc, + ) -> ExecutionResult { let mut stages = vec![]; for stage in graph.stages().iter() { let mut descriptor = StageDescriptor { @@ -68,6 +70,7 @@ impl JobDescriptor { }) .collect(); Ok(Self { + context, graph, topology, stages, diff --git a/crates/sail-execution/src/driver/options.rs b/crates/sail-execution/src/driver/options.rs index 9862e1342..52beb4ccc 100644 --- a/crates/sail-execution/src/driver/options.rs +++ b/crates/sail-execution/src/driver/options.rs @@ -6,6 +6,7 @@ use sail_common::runtime::RuntimeHandle; use sail_server::RetryStrategy; use crate::id::DriverId; +use crate::shuffle::ShuffleBackendKind; use crate::worker_manager::WorkerManager; #[readonly::make] @@ -26,6 +27,7 @@ pub struct DriverOptions { pub task_stream_buffer: usize, pub task_stream_creation_timeout: Duration, pub task_max_attempts: usize, + pub shuffle_backend: ShuffleBackendKind, pub rpc_retry_strategy: RetryStrategy, pub runtime: RuntimeHandle, pub worker_manager: Arc, @@ -63,6 +65,7 @@ impl DriverOptions { config.cluster.task_stream_creation_timeout_secs, ), task_max_attempts: config.cluster.task_max_attempts, + shuffle_backend: (&config.cluster.shuffle_backend).into(), runtime, worker_manager, } diff --git a/crates/sail-execution/src/driver/worker_pool/core.rs b/crates/sail-execution/src/driver/worker_pool/core.rs index 01ef626c5..a824bed64 100644 --- a/crates/sail-execution/src/driver/worker_pool/core.rs +++ b/crates/sail-execution/src/driver/worker_pool/core.rs @@ -79,6 +79,7 @@ impl WorkerPool { task_stream_buffer: self.options.task_stream_buffer, task_stream_creation_timeout: self.options.task_stream_creation_timeout, rpc_retry_strategy: self.options.rpc_retry_strategy.clone(), + shuffle_backend: self.options.shuffle_backend.clone(), }; let worker_manager = Arc::clone(&self.worker_manager); ctx.spawn(async move { diff --git a/crates/sail-execution/src/driver/worker_pool/options.rs b/crates/sail-execution/src/driver/worker_pool/options.rs index e8d925fbc..1b846c227 100644 --- a/crates/sail-execution/src/driver/worker_pool/options.rs +++ b/crates/sail-execution/src/driver/worker_pool/options.rs @@ -5,6 +5,7 @@ use sail_server::RetryStrategy; use crate::driver::DriverOptions; use crate::id::DriverId; +use crate::shuffle::ShuffleBackendKind; #[readonly::make] pub struct WorkerPoolOptions { @@ -20,6 +21,7 @@ pub struct WorkerPoolOptions { pub task_stream_buffer: usize, pub task_stream_creation_timeout: Duration, pub rpc_retry_strategy: RetryStrategy, + pub shuffle_backend: ShuffleBackendKind, } impl From<&DriverOptions> for WorkerPoolOptions { @@ -37,6 +39,7 @@ impl From<&DriverOptions> for WorkerPoolOptions { task_stream_buffer: options.task_stream_buffer, task_stream_creation_timeout: options.task_stream_creation_timeout, rpc_retry_strategy: options.rpc_retry_strategy.clone(), + shuffle_backend: options.shuffle_backend.clone(), } } } diff --git a/crates/sail-execution/src/job_graph/mod.rs b/crates/sail-execution/src/job_graph/mod.rs index cdefa9b11..a944cdd35 100644 --- a/crates/sail-execution/src/job_graph/mod.rs +++ b/crates/sail-execution/src/job_graph/mod.rs @@ -8,6 +8,11 @@ use datafusion::physical_expr::PhysicalExpr; use datafusion::physical_plan::display::DisplayableExecutionPlan; use datafusion::physical_plan::{ExecutionPlan, ExecutionPlanProperties}; +#[derive(Debug, Clone)] +pub struct JobGraphOptions { + pub use_blocking_shuffle: bool, +} + /// A job graph represents a distributed execution plan for a job. /// A job consists of multiple *stages*, where each stage has one or more /// *partitions*. There are *tasks* which each corresponds to the execution of a single partition @@ -21,6 +26,7 @@ pub struct JobGraph { stages: Vec, /// The output schema of the job. schema: SchemaRef, + options: JobGraphOptions, } impl JobGraph { @@ -165,7 +171,6 @@ impl fmt::Display for InputMode { #[derive(Debug, Clone, Copy)] pub enum OutputMode { Pipelined, - #[expect(unused)] Blocking, } diff --git a/crates/sail-execution/src/job_graph/planner.rs b/crates/sail-execution/src/job_graph/planner.rs index 8cd58c790..053fad1e5 100644 --- a/crates/sail-execution/src/job_graph/planner.rs +++ b/crates/sail-execution/src/job_graph/planner.rs @@ -35,17 +35,22 @@ use sail_physical_plan::repartition::ExplicitRepartitionExec; use crate::error::{ExecutionError, ExecutionResult}; use crate::job_graph::{ - InputMode, JobGraph, OutputDistribution, OutputMode, Stage, StageInput, TaskPlacement, + InputMode, JobGraph, JobGraphOptions, OutputDistribution, OutputMode, Stage, StageInput, + TaskPlacement, }; use crate::plan::{ShuffleConsumption, StageInputExec}; impl JobGraph { - pub fn try_new(plan: Arc) -> ExecutionResult { + pub fn try_new( + plan: Arc, + options: JobGraphOptions, + ) -> ExecutionResult { let plan = ensure_single_input_partition_for_global_limit(plan)?; let plan = ensure_partitioned_hash_join_if_build_side_emits_unmatched_rows(plan)?; let mut graph = Self { stages: vec![], schema: plan.schema(), + options, }; let last = build_job_graph(plan, PartitionUsage::Once, &mut graph)?.plan; let (last, inputs) = rewrite_inputs(last)?; @@ -752,6 +757,7 @@ fn create_merge_input( graph, OutputDistribution::RoundRobin { channels: 1 }, TaskPlacement::Worker, + OutputMode::Pipelined, )?; Ok(stage_input_exec(stage, InputMode::Merge, properties)) } @@ -765,6 +771,7 @@ fn create_scalar_subquery_input( graph, OutputDistribution::RoundRobin { channels: 1 }, TaskPlacement::Worker, + OutputMode::Pipelined, )?; // ScalarSubqueryExec reads the link as a scalar value on every output // partition, so the materialized stage is exposed as one broadcast input. @@ -784,6 +791,7 @@ fn create_rescale_input( graph, OutputDistribution::RoundRobin { channels: 1 }, TaskPlacement::Worker, + OutputMode::Pipelined, )?; let properties = stage_properties_with_unknown_partitioning(graph, stage, output_partitions); Ok(stage_input_exec(stage, InputMode::Rescale, properties)) @@ -805,12 +813,18 @@ fn create_shuffle( Partitioning::Hash(keys, channels) => OutputDistribution::Hash { keys, channels }, }; let plan = wrap_pending_scalar_subqueries(plan, scalar_context); - let stage = push_stage(plan, graph, distribution, TaskPlacement::Worker)?; + let stage = push_stage( + plan, + graph, + distribution, + TaskPlacement::Worker, + OutputMode::Pipelined, + )?; let mode = match consumption { ShuffleConsumption::Single => InputMode::Shuffle, ShuffleConsumption::Multiple => InputMode::Broadcast, }; - Ok(stage_input_exec(stage, mode, properties)) + create_shuffle_input(stage, mode, properties, graph) } /// Creates a shuffle stage with row-level round-robin distribution. @@ -826,12 +840,42 @@ fn create_row_shuffle( ) -> ExecutionResult> { let distribution = OutputDistribution::RoundRobinRow { channels }; let plan = wrap_pending_scalar_subqueries(plan, scalar_context); - let stage = push_stage(plan, graph, distribution, TaskPlacement::Worker)?; + let stage = push_stage( + plan, + graph, + distribution, + TaskPlacement::Worker, + OutputMode::Pipelined, + )?; let mode = match consumption { ShuffleConsumption::Single => InputMode::Shuffle, ShuffleConsumption::Multiple => InputMode::Broadcast, }; - Ok(stage_input_exec(stage, mode, properties)) + create_shuffle_input(stage, mode, properties, graph) +} + +/// In storage shuffle mode, collect each logical shuffle partition in a separate +/// blocking stage. This turns the many task/channel streams produced by the upstream +/// stage into one (or a few size-bounded) object-storage files per output partition. +fn create_shuffle_input( + stage: usize, + mode: InputMode, + properties: Arc, + graph: &mut JobGraph, +) -> ExecutionResult> { + if !graph.options.use_blocking_shuffle { + return Ok(stage_input_exec(stage, mode, properties)); + } + + let merge = stage_input_exec(stage, mode, properties.clone()); + let stage = push_stage( + merge, + graph, + OutputDistribution::RoundRobin { channels: 1 }, + TaskPlacement::Worker, + OutputMode::Blocking, + )?; + Ok(stage_input_exec(stage, InputMode::Forward, properties)) } fn rewrite_inputs( @@ -856,13 +900,14 @@ fn push_stage( graph: &mut JobGraph, distribution: OutputDistribution, placement: TaskPlacement, + mode: OutputMode, ) -> ExecutionResult { let (plan, inputs) = rewrite_inputs(plan)?; let stage = Stage { inputs, plan, group: String::new(), - mode: OutputMode::Pipelined, + mode, distribution, placement, }; @@ -904,6 +949,7 @@ fn create_driver_stage( graph, OutputDistribution::RoundRobin { channels: 1 }, TaskPlacement::Driver, + OutputMode::Pipelined, )?; let properties = graph.stages[stage].plan.properties().clone(); Ok(stage_input_exec(stage, InputMode::Forward, properties)) @@ -938,8 +984,9 @@ mod tests { use sail_physical_plan::remote_checkpoint::RemoteCheckpointCommitExec; use sail_physical_plan::repartition::ExplicitRepartitionExec; - use super::JobGraph; - use crate::job_graph::{InputMode, OutputDistribution, StageInput, TaskPlacement}; + use super::{JobGraph, JobGraphOptions}; + use crate::error::ExecutionResult; + use crate::job_graph::{InputMode, OutputDistribution, OutputMode, StageInput, TaskPlacement}; use crate::plan::StageInputExec; fn schema() -> SchemaRef { @@ -950,13 +997,28 @@ mod tests { Arc::new(EmptyExec::new(schema())) } + fn job_graph(plan: Arc) -> ExecutionResult { + JobGraph::try_new( + plan, + JobGraphOptions { + use_blocking_shuffle: false, + }, + ) + } + + fn blocking_shuffle_options() -> JobGraphOptions { + JobGraphOptions { + use_blocking_shuffle: true, + } + } + #[test] fn test_job_graph_distinguishes_batch_and_explicit_round_robin_shuffle() { - let generic_graph = JobGraph::try_new(Arc::new( + let generic_graph = job_graph(Arc::new( RepartitionExec::try_new(empty_plan(), Partitioning::RoundRobinBatch(4)).unwrap(), )) .unwrap(); - let explicit_graph = JobGraph::try_new(Arc::new(ExplicitRepartitionExec::new( + let explicit_graph = job_graph(Arc::new(ExplicitRepartitionExec::new( empty_plan(), Partitioning::RoundRobinBatch(4), ))) @@ -989,12 +1051,41 @@ mod tests { )); } + #[test] + fn test_storage_shuffle_inserts_a_blocking_merge_stage() { + let graph = JobGraph::try_new( + Arc::new( + RepartitionExec::try_new(empty_plan(), Partitioning::RoundRobinBatch(4)).unwrap(), + ), + blocking_shuffle_options(), + ) + .unwrap(); + + assert_eq!(graph.stages().len(), 3); + assert!(matches!(graph.stages()[0].mode, OutputMode::Pipelined)); + assert!(matches!( + graph.stages()[1].inputs.as_slice(), + [StageInput { + stage: 0, + mode: InputMode::Shuffle, + }] + )); + assert!(matches!(graph.stages()[1].mode, OutputMode::Blocking)); + assert!(matches!( + graph.stages()[2].inputs.as_slice(), + [StageInput { + stage: 1, + mode: InputMode::Forward, + }] + )); + } + #[test] fn test_job_graph_uses_rescale_input_for_coalesce_exec() { let input = UnionExec::try_new(vec![empty_plan(), empty_plan(), empty_plan(), empty_plan()]) .unwrap(); - let graph = JobGraph::try_new(Arc::new(CoalesceExec::new(input, 2))).unwrap(); + let graph = job_graph(Arc::new(CoalesceExec::new(input, 2))).unwrap(); assert_eq!(graph.stages().len(), 2); assert_eq!( @@ -1027,7 +1118,13 @@ mod tests { None, )); - let graph = JobGraph::try_new(commit).unwrap(); + let graph = JobGraph::try_new( + commit, + JobGraphOptions { + use_blocking_shuffle: false, + }, + ) + .unwrap(); assert_eq!(graph.stages().len(), 2); assert_eq!(graph.stages()[0].placement, TaskPlacement::Driver); @@ -1051,8 +1148,7 @@ mod tests { schema(), )); let command = Arc::new(CooperativeExec::new(command)); - let graph = - JobGraph::try_new(Arc::new(BarrierExec::new(vec![precondition], command))).unwrap(); + let graph = job_graph(Arc::new(BarrierExec::new(vec![precondition], command))).unwrap(); assert_eq!(graph.stages().len(), 3); assert_eq!(graph.stages()[1].placement, TaskPlacement::Driver); @@ -1104,7 +1200,7 @@ mod tests { results, )); - let graph = JobGraph::try_new(plan).unwrap(); + let graph = job_graph(plan).unwrap(); assert!(graph.stages().len() >= 3); let stage = graph @@ -1182,7 +1278,7 @@ mod tests { results, )); - let graph = JobGraph::try_new(plan).unwrap(); + let graph = job_graph(plan).unwrap(); let stage = graph .stages() @@ -1217,7 +1313,7 @@ mod tests { results, )); - let graph = JobGraph::try_new(plan).unwrap(); + let graph = job_graph(plan).unwrap(); let stage = graph .stages() .iter() diff --git a/crates/sail-execution/src/job_runner.rs b/crates/sail-execution/src/job_runner.rs index 5fb684264..7f6fa36ad 100644 --- a/crates/sail-execution/src/job_runner.rs +++ b/crates/sail-execution/src/job_runner.rs @@ -14,7 +14,8 @@ use tokio::sync::mpsc::error::SendError; use tokio::sync::oneshot; use crate::driver::{DriverActor, DriverEvent, DriverHandle, DriverOptions}; -use crate::job_graph::JobGraph; +use crate::job_graph::{JobGraph, JobGraphOptions}; +use crate::shuffle::ShuffleBackendKind; pub struct LocalJobRunner { next_job_id: AtomicU64, @@ -78,12 +79,17 @@ impl JobRunner for LocalJobRunner { pub struct ClusterJobRunner { driver: DriverHandle, + shuffle_backend: ShuffleBackendKind, } impl ClusterJobRunner { pub fn new(system: &mut ActorSystem, options: DriverOptions) -> Self { + let shuffle_backend = options.shuffle_backend.clone(); let driver = DriverHandle::new(system.spawn::(options)); - Self { driver } + Self { + driver, + shuffle_backend, + } } pub fn driver(&self) -> DriverHandle { @@ -109,9 +115,17 @@ impl StateObservable for ClusterJobRunner { #[tonic::async_trait] impl JobRunner for ClusterJobRunner { fn explain(&self, plan: Arc) -> Result> { - JobGraph::try_new(plan) - .map(|graph| Some(graph.to_string())) - .map_err(|e| DataFusionError::External(Box::new(e))) + JobGraph::try_new( + plan, + JobGraphOptions { + use_blocking_shuffle: matches!( + &self.shuffle_backend, + ShuffleBackendKind::Storage { .. } + ), + }, + ) + .map(|graph| Some(graph.to_string())) + .map_err(|e| DataFusionError::External(Box::new(e))) } /// Executes a plan on the cluster. This is where the cool stuff happens. diff --git a/crates/sail-execution/src/lib.rs b/crates/sail-execution/src/lib.rs index 8f84bc252..561ea6554 100644 --- a/crates/sail-execution/src/lib.rs +++ b/crates/sail-execution/src/lib.rs @@ -6,6 +6,7 @@ pub mod job_runner; mod plan; mod proto; mod rpc; +mod shuffle; mod stream; mod stream_accessor; mod stream_manager; diff --git a/crates/sail-execution/src/shuffle.rs b/crates/sail-execution/src/shuffle.rs new file mode 100644 index 000000000..a2306bb5d --- /dev/null +++ b/crates/sail-execution/src/shuffle.rs @@ -0,0 +1,39 @@ +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ShuffleBackendKind { + Streaming, + Storage { + path: String, + max_file_size: usize, + compression: ShuffleCompression, + }, +} + +impl From<&sail_common::config::ShuffleBackend> for ShuffleBackendKind { + fn from(value: &sail_common::config::ShuffleBackend) -> Self { + match value { + sail_common::config::ShuffleBackend::Streaming => Self::Streaming, + sail_common::config::ShuffleBackend::Storage(storage) => Self::Storage { + path: storage.path.clone(), + max_file_size: storage.max_file_size, + compression: storage.compression.clone().into(), + }, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ShuffleCompression { + None, + Lz4, + Zstd, +} + +impl From for ShuffleCompression { + fn from(value: sail_common::config::ShuffleCompression) -> Self { + match value { + sail_common::config::ShuffleCompression::None => Self::None, + sail_common::config::ShuffleCompression::Lz4 => Self::Lz4, + sail_common::config::ShuffleCompression::Zstd => Self::Zstd, + } + } +} diff --git a/crates/sail-execution/src/stream/reader.rs b/crates/sail-execution/src/stream/reader.rs index e762c0c63..f77ed3858 100644 --- a/crates/sail-execution/src/stream/reader.rs +++ b/crates/sail-execution/src/stream/reader.rs @@ -18,7 +18,6 @@ pub enum TaskReadLocation { key: TaskStreamKey, }, Remote { - uri: String, key: TaskStreamKey, }, } @@ -32,8 +31,8 @@ impl fmt::Display for TaskReadLocation { TaskReadLocation::Worker { worker_id, key } => { write!(f, "Worker({worker_id}, {})", TaskStreamKeyDenseDisplay(key)) } - TaskReadLocation::Remote { uri, key } => { - write!(f, "Remote({uri}, {})", TaskStreamKeyDenseDisplay(key)) + TaskReadLocation::Remote { key } => { + write!(f, "Remote({})", TaskStreamKeyDenseDisplay(key)) } } } diff --git a/crates/sail-execution/src/stream/writer.rs b/crates/sail-execution/src/stream/writer.rs index a751d29cd..57217e1af 100644 --- a/crates/sail-execution/src/stream/writer.rs +++ b/crates/sail-execution/src/stream/writer.rs @@ -14,7 +14,6 @@ pub enum TaskWriteLocation { key: TaskStreamKey, }, Remote { - uri: String, key: TaskStreamKey, }, } @@ -34,8 +33,8 @@ impl fmt::Display for TaskWriteLocation { TaskWriteLocation::Local { key, storage } => { write!(f, "Local({storage}, {})", TaskStreamKeyDenseDisplay(key)) } - TaskWriteLocation::Remote { uri, key } => { - write!(f, "Remote({uri}, {})", TaskStreamKeyDenseDisplay(key)) + TaskWriteLocation::Remote { key } => { + write!(f, "Remote({})", TaskStreamKeyDenseDisplay(key)) } } } @@ -78,7 +77,6 @@ pub enum TaskStreamSinkState { /// The sink is ready to accept more writes. Ok, /// The sink has encountered an error and no further writes should be attempted. - #[expect(unused)] Error(DataFusionError), /// The sink has been closed and no further writes should be attempted. Closed, diff --git a/crates/sail-execution/src/stream_accessor/core.rs b/crates/sail-execution/src/stream_accessor/core.rs index f9077cf8d..d21a55a5b 100644 --- a/crates/sail-execution/src/stream_accessor/core.rs +++ b/crates/sail-execution/src/stream_accessor/core.rs @@ -1,5 +1,8 @@ +use std::sync::Arc; + use datafusion::arrow::datatypes::SchemaRef; use datafusion::common::{DataFusionError, Result}; +use datafusion::execution::TaskContext; use sail_server::actor::{Actor, ActorHandle}; use tokio::sync::oneshot; @@ -8,8 +11,8 @@ use crate::stream::writer::{TaskStreamSink, TaskStreamWriter, TaskWriteLocation} use crate::stream_accessor::{StreamAccessor, StreamAccessorMessage}; impl StreamAccessor { - pub fn new(handle: ActorHandle) -> Self { - Self { handle } + pub fn new(handle: ActorHandle, context: Arc) -> Self { + Self { handle, context } } } @@ -31,8 +34,8 @@ where TaskReadLocation::Worker { worker_id, key } => { T::Message::fetch_worker_stream(*worker_id, key.clone(), schema, tx) } - TaskReadLocation::Remote { uri, key } => { - T::Message::fetch_remote_stream(uri.clone(), key.clone(), schema, tx) + TaskReadLocation::Remote { key } => { + T::Message::fetch_remote_stream(key.clone(), schema, self.context.clone(), tx) } }; self.handle.send(event).await.map_err(|_| { @@ -59,8 +62,8 @@ where TaskWriteLocation::Local { key, storage } => { T::Message::create_local_stream(key.clone(), *storage, schema, tx) } - TaskWriteLocation::Remote { uri, key } => { - T::Message::create_remote_stream(uri.clone(), key.clone(), schema, tx) + TaskWriteLocation::Remote { key } => { + T::Message::create_remote_stream(key.clone(), schema, self.context.clone(), tx) } }; self.handle.send(event).await.map_err(|_| { diff --git a/crates/sail-execution/src/stream_accessor/mod.rs b/crates/sail-execution/src/stream_accessor/mod.rs index 4336d44a7..026caf901 100644 --- a/crates/sail-execution/src/stream_accessor/mod.rs +++ b/crates/sail-execution/src/stream_accessor/mod.rs @@ -1,8 +1,10 @@ mod core; use std::fmt; +use std::sync::Arc; use datafusion::arrow::datatypes::SchemaRef; +use datafusion::execution::TaskContext; use sail_server::actor::{Actor, ActorHandle}; use tokio::sync::oneshot; @@ -15,6 +17,16 @@ use crate::worker::{WorkerEvent, WorkerStreamOwner}; pub struct StreamAccessor { handle: ActorHandle, + context: Arc, +} + +impl Clone for StreamAccessor { + fn clone(&self) -> Self { + Self { + handle: self.handle.clone(), + context: Arc::clone(&self.context), + } + } } impl fmt::Debug for StreamAccessor { @@ -32,9 +44,9 @@ pub trait StreamAccessorMessage { ) -> Self; fn create_remote_stream( - uri: String, key: TaskStreamKey, schema: SchemaRef, + context: Arc, result: oneshot::Sender>>, ) -> Self; @@ -52,9 +64,9 @@ pub trait StreamAccessorMessage { ) -> Self; fn fetch_remote_stream( - uri: String, key: TaskStreamKey, schema: SchemaRef, + context: Arc, result: oneshot::Sender>, ) -> Self; } @@ -75,15 +87,15 @@ impl StreamAccessorMessage for DriverEvent { } fn create_remote_stream( - uri: String, key: TaskStreamKey, schema: SchemaRef, + context: Arc, result: oneshot::Sender>>, ) -> Self { DriverEvent::CreateRemoteStream { - uri, key, schema, + context, result, } } @@ -111,15 +123,15 @@ impl StreamAccessorMessage for DriverEvent { } fn fetch_remote_stream( - uri: String, key: TaskStreamKey, schema: SchemaRef, + context: Arc, result: oneshot::Sender>, ) -> Self { DriverEvent::FetchRemoteStream { - uri, key, schema, + context, result, } } @@ -141,15 +153,15 @@ impl StreamAccessorMessage for WorkerEvent { } fn create_remote_stream( - uri: String, key: TaskStreamKey, schema: SchemaRef, + context: Arc, result: oneshot::Sender>>, ) -> Self { WorkerEvent::CreateRemoteStream { - uri, key, schema, + context, result, } } @@ -180,15 +192,15 @@ impl StreamAccessorMessage for WorkerEvent { } fn fetch_remote_stream( - uri: String, key: TaskStreamKey, schema: SchemaRef, + context: Arc, result: oneshot::Sender>, ) -> Self { WorkerEvent::FetchRemoteStream { - uri, key, schema, + context, result, } } diff --git a/crates/sail-execution/src/stream_manager/core.rs b/crates/sail-execution/src/stream_manager/core.rs index 1e6e640f6..c5abb7008 100644 --- a/crates/sail-execution/src/stream_manager/core.rs +++ b/crates/sail-execution/src/stream_manager/core.rs @@ -1,8 +1,10 @@ use std::collections::HashMap; use std::collections::hash_map::Entry; +use std::sync::Arc; use datafusion::arrow::array::RecordBatch; use datafusion::arrow::datatypes::SchemaRef; +use datafusion::execution::TaskContext; use log::warn; use sail_common_datafusion::error::CommonErrorCause; use sail_python_udf::error::PyErrExtractor; @@ -17,12 +19,26 @@ use crate::stream::reader::TaskStreamSource; use crate::stream::writer::{LocalStreamStorage, TaskStreamSink}; use crate::stream_manager::local::{LocalStream, MemoryStream}; use crate::stream_manager::options::StreamManagerOptions; +use crate::stream_manager::remote::RemoteStreamManager; use crate::stream_manager::{LocalStreamState, StreamManager, StreamManagerMessage}; impl StreamManager { pub fn new(options: StreamManagerOptions) -> Self { + let remote_streams = match &options.shuffle_backend { + crate::shuffle::ShuffleBackendKind::Streaming => None, + crate::shuffle::ShuffleBackendKind::Storage { + path, + max_file_size, + compression, + } => Some(Arc::new(RemoteStreamManager::new( + path.clone(), + *max_file_size, + *compression, + ))), + }; Self { options, + remote_streams, local_streams: HashMap::new(), } } @@ -87,13 +103,16 @@ impl StreamManager { pub fn create_remote_stream( &mut self, - _uri: String, - _key: TaskStreamKey, - _schema: SchemaRef, + key: TaskStreamKey, + schema: SchemaRef, + context: &TaskContext, ) -> ExecutionResult> { - Err(ExecutionError::InternalError( - "not implemented: remote stream".to_string(), - )) + let Some(remote_streams) = &self.remote_streams else { + return Err(ExecutionError::InternalError( + "remote stream requested without a storage shuffle backend".to_string(), + )); + }; + remote_streams.create_stream(key, schema, context) } pub fn fetch_local_stream( @@ -135,17 +154,20 @@ impl StreamManager { pub fn fetch_remote_stream( &mut self, _ctx: &mut ActorContext, - _uri: String, - _key: &TaskStreamKey, - _schema: SchemaRef, + key: &TaskStreamKey, + schema: SchemaRef, + context: &TaskContext, ) -> ExecutionResult where T: Actor, T::Message: StreamManagerMessage, { - Err(ExecutionError::InternalError( - "not implemented: fetch remote stream".to_string(), - )) + let Some(remote_streams) = &self.remote_streams else { + return Err(ExecutionError::InternalError( + "remote stream requested without a storage shuffle backend".to_string(), + )); + }; + remote_streams.fetch_stream(key.clone(), schema, context) } pub fn remove_local_streams(&mut self, job_id: JobId, stage: Option) { @@ -159,13 +181,21 @@ impl StreamManager { pub fn remove_remote_streams( &mut self, - _ctx: &mut ActorContext, - _job_id: JobId, - _stage: Option, + ctx: &mut ActorContext, + job_id: JobId, + stage: Option, + context: Arc, ) where T: Actor, { - warn!("removing remote streams is not implemented"); + let Some(remote_streams) = self.remote_streams.clone() else { + return; + }; + ctx.spawn(async move { + if let Err(e) = remote_streams.remove_streams(job_id, stage, &context).await { + warn!("failed to remove remote shuffle data for job {job_id}: {e}"); + } + }); } pub fn fail_local_stream_if_pending(&mut self, key: &TaskStreamKey) { diff --git a/crates/sail-execution/src/stream_manager/mod.rs b/crates/sail-execution/src/stream_manager/mod.rs index a06b961cf..f03c652a2 100644 --- a/crates/sail-execution/src/stream_manager/mod.rs +++ b/crates/sail-execution/src/stream_manager/mod.rs @@ -1,8 +1,10 @@ mod core; mod local; mod options; +mod remote; use std::collections::HashMap; +use std::sync::Arc; use datafusion::arrow::array::RecordBatch; pub use options::StreamManagerOptions; @@ -16,6 +18,7 @@ use crate::worker::WorkerEvent; pub struct StreamManager { options: StreamManagerOptions, + remote_streams: Option>, local_streams: HashMap, } diff --git a/crates/sail-execution/src/stream_manager/options.rs b/crates/sail-execution/src/stream_manager/options.rs index e50d6bde1..f520dfc45 100644 --- a/crates/sail-execution/src/stream_manager/options.rs +++ b/crates/sail-execution/src/stream_manager/options.rs @@ -1,12 +1,14 @@ use std::time::Duration; use crate::driver::DriverOptions; +use crate::shuffle::ShuffleBackendKind; use crate::worker::WorkerOptions; #[readonly::make] pub struct StreamManagerOptions { pub task_stream_buffer: usize, pub task_stream_creation_timeout: Duration, + pub shuffle_backend: ShuffleBackendKind, } impl From<&DriverOptions> for StreamManagerOptions { @@ -14,6 +16,7 @@ impl From<&DriverOptions> for StreamManagerOptions { Self { task_stream_buffer: options.task_stream_buffer, task_stream_creation_timeout: options.task_stream_creation_timeout, + shuffle_backend: options.shuffle_backend.clone(), } } } @@ -23,6 +26,7 @@ impl From<&WorkerOptions> for StreamManagerOptions { Self { task_stream_buffer: options.task_stream_buffer, task_stream_creation_timeout: options.task_stream_creation_timeout, + shuffle_backend: options.shuffle_backend.clone(), } } } diff --git a/crates/sail-execution/src/stream_manager/remote.rs b/crates/sail-execution/src/stream_manager/remote.rs new file mode 100644 index 000000000..658023fcc --- /dev/null +++ b/crates/sail-execution/src/stream_manager/remote.rs @@ -0,0 +1,357 @@ +use std::io::Cursor; +use std::sync::Arc; + +use datafusion::arrow::array::RecordBatch; +use datafusion::arrow::datatypes::SchemaRef; +use datafusion::arrow::error::ArrowError; +use datafusion::arrow::ipc::CompressionType; +use datafusion::arrow::ipc::reader::StreamReader; +use datafusion::arrow::ipc::writer::{IpcWriteOptions, StreamWriter}; +use datafusion::common::{DataFusionError, Result}; +use datafusion::execution::TaskContext; +use futures::{StreamExt, TryStreamExt}; +use object_store::path::Path; +use object_store::{ObjectStore, ObjectStoreExt, PutPayload}; +use url::Url; + +use crate::error::{ExecutionError, ExecutionResult}; +use crate::id::{JobId, TaskStreamKey}; +use crate::shuffle::ShuffleCompression; +use crate::stream::error::{TaskStreamError, TaskStreamResult}; +use crate::stream::reader::TaskStreamSource; +use crate::stream::writer::{TaskStreamSink, TaskStreamSinkState}; + +/// Object-storage implementation for a blocking shuffle stream. Every task attempt owns a +/// directory, allowing retries to be selected by the task scheduler without overwriting an +/// earlier attempt. +pub(super) struct RemoteStreamManager { + storage_path: String, + max_file_size: usize, + compression: ShuffleCompression, +} + +impl RemoteStreamManager { + pub(super) fn new( + storage_path: String, + max_file_size: usize, + compression: ShuffleCompression, + ) -> Self { + Self { + storage_path, + max_file_size, + compression, + } + } + + pub(super) fn create_stream( + &self, + key: TaskStreamKey, + schema: SchemaRef, + context: &TaskContext, + ) -> ExecutionResult> { + if self.max_file_size == 0 { + return Err(ExecutionError::InvalidArgument( + "cluster.shuffle_backend.storage.max_file_size must be greater than zero" + .to_string(), + )); + } + let (store, prefix) = self.store_and_prefix(&key, context)?; + let options = IpcWriteOptions::default() + .try_with_compression(match self.compression { + ShuffleCompression::None => None, + ShuffleCompression::Lz4 => Some(CompressionType::LZ4_FRAME), + ShuffleCompression::Zstd => Some(CompressionType::ZSTD), + }) + .map_err(|e| DataFusionError::External(Box::new(e)))?; + Ok(Box::new(RemoteStreamSink::new( + store, + prefix, + schema, + options, + self.max_file_size, + )?)) + } + + pub(super) fn fetch_stream( + &self, + key: TaskStreamKey, + _schema: SchemaRef, + context: &TaskContext, + ) -> ExecutionResult { + let (store, prefix) = self.store_and_prefix(&key, context)?; + let list_store = Arc::clone(&store); + let output = futures::stream::once(async move { + let mut locations = list_store + .list(Some(&prefix)) + .map_err(|e| TaskStreamError::External(Arc::new(e))) + .try_collect::>() + .await? + .into_iter() + .map(|meta| meta.location) + .collect::>(); + locations.sort(); + Ok::<_, TaskStreamError>(locations) + }) + .flat_map(move |result| -> TaskStreamSource { + match result { + Ok(locations) => Box::pin(futures::stream::try_unfold( + RemoteReadState { + store: Arc::clone(&store), + locations: locations.into_iter(), + reader: None, + }, + |mut state| async move { + loop { + if let Some(reader) = state.reader.as_mut() { + match reader.next() { + Some(Ok(batch)) => return Ok(Some((batch, state))), + Some(Err(error)) => { + return Err(TaskStreamError::External(Arc::new(error))); + } + None => state.reader = None, + } + } + let Some(location) = state.locations.next() else { + return Ok(None); + }; + let bytes = state + .store + .get(&location) + .await + .map_err(|e| TaskStreamError::External(Arc::new(e)))? + .bytes() + .await + .map_err(|e| TaskStreamError::External(Arc::new(e)))?; + let reader = StreamReader::try_new(Cursor::new(bytes), None) + .map_err(|e| TaskStreamError::External(Arc::new(e)))?; + state.reader = Some(Box::new(reader)); + } + }, + )), + Err(error) => Box::pin(futures::stream::iter(vec![Err(error)])), + } + }); + Ok(Box::pin(output)) + } + + pub(super) async fn remove_streams( + &self, + job_id: JobId, + stage: Option, + context: &TaskContext, + ) -> Result<()> { + // All shuffle data for a job is beneath this prefix, so terminal job cleanup removes + // every attempt. Stage cleanup can safely reclaim data once all of its consumers finish. + let Some((store, prefix)) = self + .store_and_cleanup_prefix(job_id, stage, context) + .map_err(|e| DataFusionError::External(Box::new(e)))? + else { + return Ok(()); + }; + let locations = store + .list(Some(&prefix)) + .map_err(|e| DataFusionError::External(Box::new(e))) + .try_collect::>() + .await + .map_err(|e| DataFusionError::External(Box::new(e)))? + .into_iter() + .map(|meta| Ok(meta.location)); + store + .delete_stream(Box::pin(futures::stream::iter(locations))) + .try_collect::>() + .await + .map_err(|e| DataFusionError::External(Box::new(e)))?; + Ok(()) + } + + fn store_and_prefix( + &self, + key: &TaskStreamKey, + context: &TaskContext, + ) -> ExecutionResult<(Arc, Path)> { + let url = Url::parse(&self.storage_path) + .map_err(|e| ExecutionError::InvalidArgument(e.to_string()))?; + let store = context + .runtime_env() + .object_store_registry + .get_store(&url)?; + Ok((store, stream_prefix(&url, key)?)) + } + + fn store_and_cleanup_prefix( + &self, + job_id: JobId, + stage: Option, + context: &TaskContext, + ) -> ExecutionResult, Path)>> { + if self.storage_path.is_empty() { + return Ok(None); + } + let url = Url::parse(&self.storage_path) + .map_err(|e| ExecutionError::InvalidArgument(e.to_string()))?; + let store = context + .runtime_env() + .object_store_registry + .get_store(&url)?; + Ok(Some((store, shuffle_prefix(&url, job_id, stage)?))) + } +} + +struct RemoteReadState { + store: Arc, + locations: std::vec::IntoIter, + reader: Option, +} + +type RemoteBatchReader = + Box> + Send>; + +struct RemoteStreamSink { + store: Arc, + prefix: Path, + schema: SchemaRef, + options: IpcWriteOptions, + max_file_size: usize, + writer: Option>>, + file_index: usize, + has_batches: bool, +} + +impl RemoteStreamSink { + fn new( + store: Arc, + prefix: Path, + schema: SchemaRef, + options: IpcWriteOptions, + max_file_size: usize, + ) -> Result { + let writer = + StreamWriter::try_new_with_options(Vec::new(), schema.as_ref(), options.clone()) + .map_err(|e| DataFusionError::External(Box::new(e)))?; + Ok(Self { + store, + prefix, + schema, + options, + max_file_size, + writer: Some(writer), + file_index: 0, + has_batches: false, + }) + } + + fn new_writer(&self) -> Result>> { + StreamWriter::try_new_with_options(Vec::new(), self.schema.as_ref(), self.options.clone()) + .map_err(|e| DataFusionError::External(Box::new(e))) + } + + async fn flush_file(&mut self) -> Result<()> { + if !self.has_batches { + return Ok(()); + } + let mut writer = self.writer.take().ok_or_else(|| { + DataFusionError::Internal("remote shuffle writer is not present".to_string()) + })?; + writer + .finish() + .map_err(|e| DataFusionError::External(Box::new(e)))?; + let bytes = writer + .into_inner() + .map_err(|e| DataFusionError::External(Box::new(e)))?; + let path = self + .prefix + .clone() + .join(format!("part-{:020}.arrow", self.file_index)); + self.store + .put(&path, PutPayload::from(bytes)) + .await + .map_err(|e| DataFusionError::External(Box::new(e)))?; + self.file_index += 1; + self.has_batches = false; + self.writer = Some(self.new_writer()?); + Ok(()) + } +} + +#[tonic::async_trait] +impl TaskStreamSink for RemoteStreamSink { + async fn write(&mut self, batch: TaskStreamResult) -> TaskStreamSinkState { + let batch = match batch { + Ok(batch) => batch, + Err(error) => { + return TaskStreamSinkState::Error(DataFusionError::External(Box::new(error))); + } + }; + let result = (|| -> Result { + let writer = self.writer.as_mut().ok_or_else(|| { + DataFusionError::Internal("remote shuffle writer is not present".to_string()) + })?; + writer + .write(&batch) + .map_err(|e| DataFusionError::External(Box::new(e)))?; + self.has_batches = true; + Ok(writer.get_ref().len() >= self.max_file_size) + })(); + match result { + Ok(true) => match self.flush_file().await { + Ok(()) => TaskStreamSinkState::Ok, + Err(e) => TaskStreamSinkState::Error(e), + }, + Ok(false) => TaskStreamSinkState::Ok, + Err(e) => TaskStreamSinkState::Error(e), + } + } + + async fn close(mut self: Box) -> Result<()> { + self.flush_file().await + } +} + +fn stream_prefix(url: &Url, key: &TaskStreamKey) -> ExecutionResult { + Ok(shuffle_prefix(url, key.job_id, Some(key.stage))? + .join(format!("partition-{}", key.partition)) + .join(format!("attempt-{}", key.attempt)) + .join(format!("channel-{}", key.channel))) +} + +fn shuffle_prefix(url: &Url, job_id: JobId, stage: Option) -> ExecutionResult { + let mut prefix = Path::from_url_path(url.path()) + .map_err(|e| ExecutionError::InvalidArgument(e.to_string()))? + .join(format!("job-{job_id}")); + if let Some(stage) = stage { + prefix = prefix.join(format!("stage-{stage}")); + } + Ok(prefix) +} + +#[expect(clippy::unwrap_used)] +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn shuffle_prefix_decodes_url_path() { + let url = Url::parse("file:///tmp/sail%20shuffle").unwrap(); + assert_eq!( + shuffle_prefix(&url, JobId::from(1), Some(2)).unwrap(), + Path::from("tmp/sail shuffle/job-1/stage-2") + ); + } + + #[test] + fn stream_prefix_includes_channel() { + let url = Url::parse("memory:///shuffle").unwrap(); + let key = TaskStreamKey { + job_id: JobId::from(1), + stage: 2, + partition: 3, + attempt: 4, + channel: 5, + }; + let prefix = stream_prefix(&url, &key).unwrap(); + assert_eq!( + prefix, + Path::from("shuffle/job-1/stage-2/partition-3/attempt-4/channel-5") + ); + } +} diff --git a/crates/sail-execution/src/task/definition.rs b/crates/sail-execution/src/task/definition.rs index 898bd5a4a..eb0fa62d7 100644 --- a/crates/sail-execution/src/task/definition.rs +++ b/crates/sail-execution/src/task/definition.rs @@ -35,7 +35,6 @@ pub enum TaskInputLocator { keys: Vec>, }, Remote { - uri: String, stage: usize, keys: Vec>, }, @@ -71,7 +70,7 @@ pub enum TaskOutputDistribution { #[derive(Debug, Clone)] pub enum TaskOutputLocator { Local { replicas: usize }, - Remote { uri: String }, + Remote, } impl From for r#gen::TaskDefinition { @@ -154,9 +153,8 @@ impl From for r#gen::TaskInputLocator { keys: keys.into_iter().map(|x| x.into()).collect(), }) } - TaskInputLocator::Remote { uri, stage, keys } => { + TaskInputLocator::Remote { stage, keys } => { r#gen::task_input_locator::Kind::Remote(r#gen::TaskInputRemoteLocator { - uri, stage: stage as u64, keys: keys.into_iter().map(|x| x.into()).collect(), }) @@ -198,7 +196,6 @@ impl TryFrom for TaskInputLocator { }) } Some(r#gen::task_input_locator::Kind::Remote(r#gen::TaskInputRemoteLocator { - uri, stage, keys, })) => { @@ -207,7 +204,6 @@ impl TryFrom for TaskInputLocator { .map(|x| x.try_into()) .collect::>>()?; Ok(TaskInputLocator::Remote { - uri, stage: stage as usize, keys, }) @@ -471,8 +467,8 @@ impl From for r#gen::TaskOutputLocator { replicas: replicas as u64, }) } - TaskOutputLocator::Remote { uri } => { - r#gen::task_output_locator::Kind::Remote(r#gen::TaskOutputRemoteLocator { uri }) + TaskOutputLocator::Remote => { + r#gen::task_output_locator::Kind::Remote(r#gen::TaskOutputRemoteLocator {}) } }; r#gen::TaskOutputLocator { kind: Some(kind) } @@ -489,9 +485,7 @@ impl TryFrom for TaskOutputLocator { })) => Ok(TaskOutputLocator::Local { replicas: replicas as usize, }), - Some(r#gen::task_output_locator::Kind::Remote(r#gen::TaskOutputRemoteLocator { - uri, - })) => Ok(TaskOutputLocator::Remote { uri }), + Some(r#gen::task_output_locator::Kind::Remote(_)) => Ok(TaskOutputLocator::Remote), None => Err(ExecutionError::InvalidArgument( "cannot decode empty task output locator".to_string(), )), @@ -535,12 +529,11 @@ impl TaskInput { .collect() }) .collect(), - TaskInputLocator::Remote { uri, stage, keys } => keys + TaskInputLocator::Remote { stage, keys } => keys .iter() .map(|keys| { keys.iter() .map(|key| TaskReadLocation::Remote { - uri: uri.clone(), key: TaskStreamKey { job_id, stage: *stage, @@ -582,9 +575,8 @@ impl TaskOutput { }, }) .collect(), - TaskOutputLocator::Remote { uri } => (0..channels) + TaskOutputLocator::Remote => (0..channels) .map(|channel| TaskWriteLocation::Remote { - uri: uri.clone(), key: TaskStreamKey { job_id: key.job_id, stage: key.stage, diff --git a/crates/sail-execution/src/task_runner/core.rs b/crates/sail-execution/src/task_runner/core.rs index af125dfcc..ed5f38216 100644 --- a/crates/sail-execution/src/task_runner/core.rs +++ b/crates/sail-execution/src/task_runner/core.rs @@ -91,7 +91,7 @@ impl TaskRunner { &definition.inputs, &definition.output, plan, - &context, + context.clone(), )?; debug!( "{} execution plan\n{}", @@ -146,36 +146,37 @@ impl TaskRunner { inputs: &[TaskInput], output: &TaskOutput, plan: Arc, - context: &TaskContext, + context: Arc, ) -> ExecutionResult> where T::Message: TaskRunnerMessage + StreamAccessorMessage, { - let handle = ctx.handle(); - let result = plan.transform(move |node| { - if let Some(placeholder) = node.downcast_ref::>() { - let Some(input) = inputs.get(*placeholder.input()) else { - return internal_err!( - "stage input index {} out of bounds for {}", - placeholder.input(), - TaskKeyDisplay(key) + let accessor = StreamAccessor::new(ctx.handle().clone(), context.clone()); + let result = { + let accessor = accessor.clone(); + plan.transform(move |node| { + if let Some(placeholder) = node.downcast_ref::>() { + let Some(input) = inputs.get(*placeholder.input()) else { + return internal_err!( + "stage input index {} out of bounds for {}", + placeholder.input(), + TaskKeyDisplay(key) + ); + }; + let locations = input.locations(key.job_id); + let shuffle = ShuffleReadExec::new( + locations, + Arc::new(accessor.clone()), + placeholder.properties().clone(), ); - }; - let locations = input.locations(key.job_id); - let accessor = StreamAccessor::new(handle.clone()); - let shuffle = ShuffleReadExec::new( - locations, - Arc::new(accessor), - placeholder.properties().clone(), - ); - Ok(Transformed::yes(Arc::new(shuffle))) - } else { - Ok(Transformed::no(node)) - } - }); + Ok(Transformed::yes(Arc::new(shuffle))) + } else { + Ok(Transformed::no(node)) + } + }) + }; let plan = result.data()?; let schema = plan.schema(); - let accessor = StreamAccessor::new(handle.clone()); let mut locations = vec![vec![]; plan.output_partitioning().partition_count()]; match locations.get_mut(key.partition) { Some(x) => x.extend(output.locations(key)), @@ -186,7 +187,7 @@ impl TaskRunner { ))); } }; - let partitioning = output.partitioning(context, &schema, self.codec.as_ref())?; + let partitioning = output.partitioning(&context, &schema, self.codec.as_ref())?; let row_based = output.row_based(); let shuffle = ShuffleWriteExec::new(plan, locations, Arc::new(accessor), partitioning, row_based); diff --git a/crates/sail-execution/src/worker/actor/core.rs b/crates/sail-execution/src/worker/actor/core.rs index 1afe737bd..ccec51890 100644 --- a/crates/sail-execution/src/worker/actor/core.rs +++ b/crates/sail-execution/src/worker/actor/core.rs @@ -88,11 +88,11 @@ impl Actor for WorkerActor { result, } => self.handle_create_local_stream(ctx, key, storage, schema, result), WorkerEvent::CreateRemoteStream { - uri, key, schema, + context, result, - } => self.handle_create_remote_stream(ctx, uri, key, schema, result), + } => self.handle_create_remote_stream(ctx, key, schema, context, result), WorkerEvent::FetchDriverStream { key, schema, @@ -102,11 +102,11 @@ impl Actor for WorkerActor { self.handle_fetch_worker_stream(ctx, owner, key, result) } WorkerEvent::FetchRemoteStream { - uri, key, schema, + context, result, - } => self.handle_fetch_remote_stream(ctx, uri, key, schema, result), + } => self.handle_fetch_remote_stream(ctx, key, schema, context, result), WorkerEvent::CleanUpJob { job_id, stage } => { self.handle_clean_up_job(ctx, job_id, stage) } diff --git a/crates/sail-execution/src/worker/actor/handler.rs b/crates/sail-execution/src/worker/actor/handler.rs index 1a9397837..40a32dd57 100644 --- a/crates/sail-execution/src/worker/actor/handler.rs +++ b/crates/sail-execution/src/worker/actor/handler.rs @@ -1,6 +1,8 @@ use std::mem; +use std::sync::Arc; use datafusion::arrow::datatypes::SchemaRef; +use datafusion::execution::TaskContext; use log::{error, info, warn}; use sail_common_datafusion::error::CommonErrorCause; use sail_server::actor::{ActorAction, ActorContext}; @@ -191,12 +193,15 @@ impl WorkerActor { pub(super) fn handle_create_remote_stream( &mut self, _ctx: &mut ActorContext, - uri: String, key: TaskStreamKey, schema: SchemaRef, + context: Arc, result: oneshot::Sender>>, ) -> ActorAction { - let _ = result.send(self.stream_manager.create_remote_stream(uri, key, schema)); + let _ = result.send( + self.stream_manager + .create_remote_stream(key, schema, &context), + ); ActorAction::Continue } @@ -252,14 +257,14 @@ impl WorkerActor { pub(super) fn handle_fetch_remote_stream( &mut self, ctx: &mut ActorContext, - uri: String, key: TaskStreamKey, schema: SchemaRef, + context: Arc, result: oneshot::Sender>, ) -> ActorAction { let _ = result.send( self.stream_manager - .fetch_remote_stream(ctx, uri, &key, schema), + .fetch_remote_stream(ctx, &key, schema, &context), ); ActorAction::Continue } diff --git a/crates/sail-execution/src/worker/event.rs b/crates/sail-execution/src/worker/event.rs index 6a1154603..30f4c60a6 100644 --- a/crates/sail-execution/src/worker/event.rs +++ b/crates/sail-execution/src/worker/event.rs @@ -1,6 +1,8 @@ use std::borrow::Cow; +use std::sync::Arc; use datafusion::arrow::datatypes::SchemaRef; +use datafusion::execution::TaskContext; use sail_common_datafusion::error::CommonErrorCause; use sail_telemetry::common::{SpanAssociation, SpanAttribute}; use tokio::sync::oneshot; @@ -48,9 +50,9 @@ pub enum WorkerEvent { result: oneshot::Sender>>, }, CreateRemoteStream { - uri: String, key: TaskStreamKey, schema: SchemaRef, + context: Arc, result: oneshot::Sender>>, }, FetchDriverStream { @@ -64,9 +66,9 @@ pub enum WorkerEvent { result: oneshot::Sender>, }, FetchRemoteStream { - uri: String, key: TaskStreamKey, schema: SchemaRef, + context: Arc, result: oneshot::Sender>, }, CleanUpJob { @@ -210,7 +212,6 @@ impl SpanAssociation for WorkerEvent { )); } WorkerEvent::CreateRemoteStream { - uri, key: TaskStreamKey { job_id, @@ -220,6 +221,7 @@ impl SpanAssociation for WorkerEvent { channel, }, schema: _, + context: _, result: _, } => { p.push((SpanAttribute::EXECUTION_JOB_ID, job_id.to_string())); @@ -227,7 +229,6 @@ impl SpanAssociation for WorkerEvent { p.push((SpanAttribute::EXECUTION_PARTITION, partition.to_string())); p.push((SpanAttribute::EXECUTION_ATTEMPT, attempt.to_string())); p.push((SpanAttribute::EXECUTION_CHANNEL, channel.to_string())); - p.push((SpanAttribute::EXECUTION_STREAM_REMOTE_URI, uri.clone())); } WorkerEvent::FetchDriverStream { key: @@ -269,7 +270,6 @@ impl SpanAssociation for WorkerEvent { p.push((SpanAttribute::EXECUTION_CHANNEL, channel.to_string())); } WorkerEvent::FetchRemoteStream { - uri, key: TaskStreamKey { job_id, @@ -279,6 +279,7 @@ impl SpanAssociation for WorkerEvent { channel, }, schema: _, + context: _, result: _, } => { p.push((SpanAttribute::EXECUTION_JOB_ID, job_id.to_string())); @@ -286,7 +287,6 @@ impl SpanAssociation for WorkerEvent { p.push((SpanAttribute::EXECUTION_PARTITION, partition.to_string())); p.push((SpanAttribute::EXECUTION_ATTEMPT, attempt.to_string())); p.push((SpanAttribute::EXECUTION_CHANNEL, channel.to_string())); - p.push((SpanAttribute::EXECUTION_STREAM_REMOTE_URI, uri.clone())); } WorkerEvent::CleanUpJob { job_id, stage } => { p.push((SpanAttribute::EXECUTION_JOB_ID, job_id.to_string())); diff --git a/crates/sail-execution/src/worker/options.rs b/crates/sail-execution/src/worker/options.rs index f4e3dcc7e..b6de2f372 100644 --- a/crates/sail-execution/src/worker/options.rs +++ b/crates/sail-execution/src/worker/options.rs @@ -6,6 +6,7 @@ use sail_common::runtime::RuntimeHandle; use sail_server::RetryStrategy; use crate::id::{DriverId, WorkerId}; +use crate::shuffle::ShuffleBackendKind; use crate::worker_manager::WorkerLaunchOptions; #[readonly::make] @@ -23,6 +24,7 @@ pub struct WorkerOptions { pub task_stream_buffer: usize, pub task_stream_creation_timeout: Duration, pub rpc_retry_strategy: RetryStrategy, + pub shuffle_backend: ShuffleBackendKind, pub runtime: RuntimeHandle, pub session: SessionContext, } @@ -51,6 +53,7 @@ impl WorkerOptions { config.cluster.task_stream_creation_timeout_secs, ), rpc_retry_strategy: (&config.cluster.rpc_retry_strategy).into(), + shuffle_backend: (&config.cluster.shuffle_backend).into(), runtime, session, } @@ -76,6 +79,7 @@ impl WorkerOptions { task_stream_buffer: options.task_stream_buffer, task_stream_creation_timeout: options.task_stream_creation_timeout, rpc_retry_strategy: options.rpc_retry_strategy, + shuffle_backend: options.shuffle_backend, runtime: runtime.clone(), session, } diff --git a/crates/sail-execution/src/worker_manager/kubernetes.rs b/crates/sail-execution/src/worker_manager/kubernetes.rs index 17606c402..71144e5a7 100644 --- a/crates/sail-execution/src/worker_manager/kubernetes.rs +++ b/crates/sail-execution/src/worker_manager/kubernetes.rs @@ -18,6 +18,7 @@ use tokio::sync::OnceCell; use crate::error::{ExecutionError, ExecutionResult}; use crate::id::WorkerId; +use crate::shuffle::{ShuffleBackendKind, ShuffleCompression}; use crate::worker_manager::{WorkerLaunchOptions, WorkerManager}; #[derive(Debug, Clone)] @@ -120,6 +121,7 @@ impl KubernetesWorkerManager { task_stream_buffer, task_stream_creation_timeout, rpc_retry_strategy, + shuffle_backend, } = options; let w3c_traceparent = SpanContext::current_local_parent().map(|x| x.encode_w3c_traceparent()); @@ -216,7 +218,49 @@ impl KubernetesWorkerManager { value: Some(rpc_retry_strategy), value_from: None, }, + EnvVar { + name: ClusterConfigEnv::SHUFFLE_BACKEND__TYPE.to_string(), + value: Some( + match &shuffle_backend { + ShuffleBackendKind::Streaming => "streaming", + ShuffleBackendKind::Storage { .. } => "storage", + } + .to_string(), + ), + value_from: None, + }, ]; + if let ShuffleBackendKind::Storage { + path, + max_file_size, + compression, + } = shuffle_backend + { + env.extend([ + EnvVar { + name: ClusterConfigEnv::SHUFFLE_BACKEND__STORAGE__PATH.to_string(), + value: Some(path), + value_from: None, + }, + EnvVar { + name: ClusterConfigEnv::SHUFFLE_BACKEND__STORAGE__MAX_FILE_SIZE.to_string(), + value: Some(max_file_size.to_string()), + value_from: None, + }, + EnvVar { + name: ClusterConfigEnv::SHUFFLE_BACKEND__STORAGE__COMPRESSION.to_string(), + value: Some( + match compression { + ShuffleCompression::None => "none", + ShuffleCompression::Lz4 => "lz4", + ShuffleCompression::Zstd => "zstd", + } + .to_string(), + ), + value_from: None, + }, + ]); + } if let Some(traceparent) = w3c_traceparent { env.push(EnvVar { name: ContextPropagationEnv::TRACEPARENT.to_string(), diff --git a/crates/sail-execution/src/worker_manager/options.rs b/crates/sail-execution/src/worker_manager/options.rs index 328258a70..434ec3013 100644 --- a/crates/sail-execution/src/worker_manager/options.rs +++ b/crates/sail-execution/src/worker_manager/options.rs @@ -3,6 +3,7 @@ use std::time::Duration; use sail_server::RetryStrategy; use crate::id::DriverId; +use crate::shuffle::ShuffleBackendKind; #[derive(Debug, Clone)] pub struct WorkerLaunchOptions { @@ -14,4 +15,5 @@ pub struct WorkerLaunchOptions { pub task_stream_buffer: usize, pub task_stream_creation_timeout: Duration, pub rpc_retry_strategy: RetryStrategy, + pub shuffle_backend: ShuffleBackendKind, } diff --git a/crates/sail-telemetry/src/common.rs b/crates/sail-telemetry/src/common.rs index 57f57dc0a..0a4547f79 100644 --- a/crates/sail-telemetry/src/common.rs +++ b/crates/sail-telemetry/src/common.rs @@ -55,7 +55,6 @@ impl SpanAttribute { pub const EXECUTION_TASK_MESSAGE: &'static str = "execution.task.message"; pub const EXECUTION_TASK_ERROR_CAUSE: &'static str = "execution.task.error_cause"; pub const EXECUTION_STREAM_LOCAL_STORAGE: &'static str = "execution.stream.local.storage"; - pub const EXECUTION_STREAM_REMOTE_URI: &'static str = "execution.stream.remote.uri"; pub const SESSION_ID: &'static str = "session.id"; } diff --git a/python/pysail/tests/spark/execution/__snapshots__/test_shuffle_storage.plan.yaml b/python/pysail/tests/spark/execution/__snapshots__/test_shuffle_storage.plan.yaml new file mode 100644 index 000000000..697670fe5 --- /dev/null +++ b/python/pysail/tests/spark/execution/__snapshots__/test_shuffle_storage.plan.yaml @@ -0,0 +1,98 @@ +# serializer version: 1 +--- +name: "test_query_execution_with_storage_shuffle" +data: + |- + == Physical Plan == + ProjectionExec: expr=[#13@0 as g, #14@1 as count, #15@2 as s1, #16@3 as s2] + SortPreservingMergeExec: [#13@0 ASC] + SortExec: expr=[#13@0 ASC], preserve_partitioning=[true] + ProjectionExec: expr=[#12@0 as #13, count(Int64(1))@1 as #14, sum(#10)@2 as #15, sum(#11)@3 as #16] + AggregateExec: mode=FinalPartitioned, gby=[#12@0 as #12], aggr=[count(Int64(1)), sum(#10), sum(#11)] + RepartitionExec: partitioning=Hash([#12@0], 4), input_partitions=8 + AggregateExec: mode=Partial, gby=[#12@2 as #12], aggr=[count(Int64(1)), sum(#10), sum(#11)] + ProjectionExec: expr=[#2@0 as #10, #5@1 as #11, #1@2 % 4 as #12] + HashJoinExec: mode=Partitioned, join_type=Inner, on=[(#1@0, #4@0)], projection=[#2@1, #5@3, #1@0] + RepartitionExec: partitioning=Hash([#1@0], 8), input_partitions=4 + ProjectionExec: expr=[#0@0 as #1, #0@0 * 2 as #2] + RangeExec + RepartitionExec: partitioning=Hash([#4@0], 8), input_partitions=4 + ProjectionExec: expr=[#3@0 as #4, #3@0 + 1 as #5] + RangeExec + + + == Distributed Plan == + === stage 0 === + inputs=[] + mode=Pipelined + partitions=4 + distribution=Hash(keys=[#1@0], channels=8) + placement=Worker + ProjectionExec: expr=[#0@0 as #1, #0@0 * 2 as #2] + RangeExec + + === stage 1 === + inputs=[StageInput(stage=0, mode=Shuffle)] + mode=Blocking + partitions=8 + distribution=RoundRobin(channels=1) + placement=Worker + StageInputExec: input=0, partitioning=Hash([#1@0], 8) + + === stage 2 === + inputs=[] + mode=Pipelined + partitions=4 + distribution=Hash(keys=[#4@0], channels=8) + placement=Worker + ProjectionExec: expr=[#3@0 as #4, #3@0 + 1 as #5] + RangeExec + + === stage 3 === + inputs=[StageInput(stage=2, mode=Shuffle)] + mode=Blocking + partitions=8 + distribution=RoundRobin(channels=1) + placement=Worker + StageInputExec: input=0, partitioning=Hash([#4@0], 8) + + === stage 4 === + inputs=[StageInput(stage=1, mode=Forward), StageInput(stage=3, mode=Forward)] + mode=Pipelined + partitions=8 + distribution=Hash(keys=[#12@0], channels=4) + placement=Worker + AggregateExec: mode=Partial, gby=[#12@2 as #12], aggr=[count(Int64(1)), sum(#10), sum(#11)] + ProjectionExec: expr=[#2@0 as #10, #5@1 as #11, #1@2 % 4 as #12] + HashJoinExec: mode=Partitioned, join_type=Inner, on=[(#1@0, #4@0)], projection=[#2@1, #5@3, #1@0] + StageInputExec: input=0, partitioning=Hash([#1@0], 8) + StageInputExec: input=1, partitioning=Hash([#4@0], 8) + + === stage 5 === + inputs=[StageInput(stage=4, mode=Shuffle)] + mode=Blocking + partitions=4 + distribution=RoundRobin(channels=1) + placement=Worker + StageInputExec: input=0, partitioning=Hash([#12@0], 4) + + === stage 6 === + inputs=[StageInput(stage=5, mode=Forward)] + mode=Pipelined + partitions=4 + distribution=RoundRobin(channels=1) + placement=Worker + SortExec: expr=[#13@0 ASC], preserve_partitioning=[true] + ProjectionExec: expr=[#12@0 as #13, count(Int64(1))@1 as #14, sum(#10)@2 as #15, sum(#11)@3 as #16] + AggregateExec: mode=FinalPartitioned, gby=[#12@0 as #12], aggr=[count(Int64(1)), sum(#10), sum(#11)] + StageInputExec: input=0, partitioning=Hash([#12@0], 4) + + === stage 7 === + inputs=[StageInput(stage=6, mode=Merge)] + mode=Pipelined + partitions=1 + distribution=RoundRobin(channels=1) + placement=Worker + ProjectionExec: expr=[#13@0 as g, #14@1 as count, #15@2 as s1, #16@3 as s2] + SortPreservingMergeExec: [#13@0 ASC] + StageInputExec: input=0, partitioning=Hash([#13@0], 4) diff --git a/python/pysail/tests/spark/execution/test_shuffle_storage.py b/python/pysail/tests/spark/execution/test_shuffle_storage.py new file mode 100644 index 000000000..165568788 --- /dev/null +++ b/python/pysail/tests/spark/execution/test_shuffle_storage.py @@ -0,0 +1,90 @@ +from __future__ import annotations + +import pandas as pd +import pyspark.sql.functions as F # noqa: N812 +import pytest +from pandas.testing import assert_frame_equal +from pyspark.sql.types import Row + +from pysail.testing.spark.session import spark_connect_server +from pysail.testing.spark.steps.plan import normalize_plan_text +from pysail.testing.spark.utils.common import is_jvm_spark + +pytestmark = pytest.mark.skipif(is_jvm_spark(), reason="Sail local-cluster mode only") + + +@pytest.fixture(scope="module") +def remote(tmp_path_factory): + shuffle_path = tmp_path_factory.mktemp("shuffle_storage") + envs = { + "SAIL_MODE": "local-cluster", + "SAIL_CLUSTER__SHUFFLE_BACKEND__TYPE": "storage", + "SAIL_CLUSTER__SHUFFLE_BACKEND__STORAGE__PATH": shuffle_path.as_uri(), + "SAIL_CLUSTER__SHUFFLE_BACKEND__STORAGE__MAX_FILE_SIZE": "1024", + "SAIL_CLUSTER__SHUFFLE_BACKEND__STORAGE__COMPRESSION": "lz4", + } + with spark_connect_server(envs=envs) as server: + yield server.remote + + +@pytest.mark.yamlsnapshot(group="plan") +def test_query_execution_with_storage_shuffle(spark, snapshot): + left = spark.range(0, 64, 1, 4).select( + F.col("id").alias("k"), + (F.col("id") * 2).alias("v1"), + ) + right = spark.range(0, 64, 1, 4).select( + F.col("id").alias("k"), + (F.col("id") + 1).alias("v2"), + ) + + df = ( + left.repartition(8, "k") + .join(right.repartition(8, "k"), "k") + .withColumn("g", F.col("k") % 4) + .groupBy("g") + .agg( + F.count("*").alias("count"), + F.sum("v1").alias("s1"), + F.sum("v2").alias("s2"), + ) + .orderBy("g") + ) + + plan = normalize_plan_text(df._explain_string()) # noqa: SLF001 + assert plan == snapshot + + actual = df.toPandas() + expected = pd.DataFrame( + { + "g": [0, 1, 2, 3], + "count": [16, 16, 16, 16], + "s1": [960, 992, 1024, 1056], + "s2": [496, 512, 528, 544], + } + ).astype( + { + "g": "int64", + "count": "int64", + "s1": "int64", + "s2": "int64", + } + ) + assert_frame_equal(actual, expected) + + +def test_repartition_collect_with_storage_shuffle(spark): + rows = ( + spark.createDataFrame([Row(id=i, group=i % 3) for i in range(30)]) + .repartition(6, "group") + .groupBy("group") + .count() + .orderBy("group") + .collect() + ) + + assert rows == [ + Row(group=0, count=10), + Row(group=1, count=10), + Row(group=2, count=10), + ] From 0e1cbf8448b8e7b28eee6f8063a7b94e3fb99dea Mon Sep 17 00:00:00 2001 From: Xiaolong Liang Date: Wed, 29 Jul 2026 17:29:22 +0800 Subject: [PATCH 094/131] fix: honor absolute Iceberg write data paths (#2274) --- .../write/file_writer/location_generator.rs | 16 +-- .../src/operations/write/table_writer.rs | 14 +- .../src/physical_plan/writer_exec.rs | 135 +++++++++--------- crates/sail-iceberg/src/table_format.rs | 24 +--- crates/sail-iceberg/src/utils/mod.rs | 22 +++ .../spark/catalog/iceberg_rest/test_commit.py | 59 ++++++++ .../spark/catalog/iceberg_rest/test_nessie.py | 49 ++++++- .../iceberg/features/alter_table.feature | 40 ++++++ .../tests/spark/iceberg/test_iceberg_write.py | 31 ++++ 9 files changed, 279 insertions(+), 111 deletions(-) diff --git a/crates/sail-iceberg/src/operations/write/file_writer/location_generator.rs b/crates/sail-iceberg/src/operations/write/file_writer/location_generator.rs index 06800df38..fc5dab415 100644 --- a/crates/sail-iceberg/src/operations/write/file_writer/location_generator.rs +++ b/crates/sail-iceberg/src/operations/write/file_writer/location_generator.rs @@ -22,7 +22,6 @@ pub trait LocationGenerator { pub struct DefaultLocationGenerator { base: ObjectPath, - data_dir: String, counter: AtomicU64, } @@ -30,15 +29,6 @@ impl DefaultLocationGenerator { pub fn new(base: ObjectPath) -> Self { Self { base, - data_dir: "data".to_string(), - counter: AtomicU64::new(0), - } - } - - pub fn new_with_data_dir(base: ObjectPath, data_dir: String) -> Self { - Self { - base, - data_dir: data_dir.trim_matches('/').to_string(), counter: AtomicU64::new(0), } } @@ -53,10 +43,8 @@ impl LocationGenerator for DefaultLocationGenerator { let id = self.counter.fetch_add(1, Ordering::Relaxed); let file = format!("part-{}-{:020}.parquet", Uuid::new_v4(), id); let rel_unencoded = match partition_dir { - Some(dir) if !dir.is_empty() => { - format!("{}/{}/{}", self.data_dir, dir.trim_matches('/'), file) - } - _ => format!("{}/{}", self.data_dir, file), + Some(dir) if !dir.is_empty() => format!("{}/{}", dir.trim_matches('/'), file), + _ => file, }; // Join each component to avoid encoding '/' into '%2F' let mut full = self.base.clone(); diff --git a/crates/sail-iceberg/src/operations/write/table_writer.rs b/crates/sail-iceberg/src/operations/write/table_writer.rs index 25c6b2552..81d14968b 100644 --- a/crates/sail-iceberg/src/operations/write/table_writer.rs +++ b/crates/sail-iceberg/src/operations/write/table_writer.rs @@ -55,7 +55,7 @@ pub struct IcebergTableWriter { pub store: Arc, pub config: WriterConfig, pub generator: DefaultLocationGenerator, - pub table_url: Url, + pub data_url: Url, // partition_dir -> writer writers: HashMap, // partition_dir -> partition values aligned with spec @@ -70,14 +70,13 @@ impl IcebergTableWriter { root: ObjectPath, config: WriterConfig, partition_spec_id: i32, - data_dir: String, - table_url: Url, + data_url: Url, ) -> Self { Self { - generator: DefaultLocationGenerator::new_with_data_dir(root, data_dir), + generator: DefaultLocationGenerator::new(root), store, config, - table_url, + data_url, writers: HashMap::new(), partition_values_map: HashMap::new(), written: Vec::new(), @@ -285,10 +284,11 @@ impl IcebergTableWriter { rel, full ); - let file_path = match self.table_url.join(&rel) { + // Prevent a leading partition segment containing ':' from being parsed as a URI scheme. + let file_path = match self.data_url.join(&format!("./{rel}")) { Ok(u) => u.to_string(), Err(_) => { - format!("{}{}", self.table_url.as_str(), rel) + format!("{}{}", self.data_url.as_str(), rel) } }; let df = DataFileWriter::new(self.partition_spec_id, file_path, partition_values) diff --git a/crates/sail-iceberg/src/physical_plan/writer_exec.rs b/crates/sail-iceberg/src/physical_plan/writer_exec.rs index e2e0abae4..af740984b 100644 --- a/crates/sail-iceberg/src/physical_plan/writer_exec.rs +++ b/crates/sail-iceberg/src/physical_plan/writer_exec.rs @@ -220,73 +220,71 @@ impl IcebergWriterExec { } } - fn resolve_data_dir(table_meta: &TableMetadata, table_url: &Url) -> String { - Self::resolve_data_dir_from_properties(&table_meta.properties, table_url) + fn resolve_data_location(table_meta: &TableMetadata, table_url: &Url) -> Result { + Self::resolve_data_location_from_properties(&table_meta.properties, table_url) } - fn resolve_data_dir_from_property_value( + fn resolve_data_location_from_property_value( value: Option<&str>, table_url: &Url, - ) -> Option { - let raw = value?.trim(); + ) -> Result> { + let Some(raw) = value.map(str::trim) else { + return Ok(None); + }; if raw.is_empty() { - return None; + return Ok(None); } - let base_path = crate::utils::url_to_object_path(table_url).ok(); - if let Some(prop_url) = crate::utils::parse_absolute_url(raw) { - if prop_url.scheme() == table_url.scheme() - && prop_url.host_str() == table_url.host_str() - && let (Ok(prop_path), Some(base_path)) = ( - crate::utils::url_to_object_path(&prop_url), - base_path.as_ref(), - ) - { - let prop_str = prop_path.as_ref(); - let base_str = base_path.as_ref(); - if let Some(stripped) = prop_str.strip_prefix(base_str) { - let rel = stripped.trim_start_matches('/').trim_matches('/'); - if !rel.is_empty() { - return Some(rel.to_string()); - } - } - } - } else { - let prop_path = raw.replace('\\', "/"); - if prop_path.starts_with('/') { - if let Some(base_path) = base_path.as_ref() { - let base_str = base_path.as_ref(); - let prop_no_leading = prop_path.trim_start_matches('/'); - if let Some(stripped) = prop_no_leading.strip_prefix(base_str) { - let rel = stripped.trim_start_matches('/').trim_matches('/'); - if !rel.is_empty() { - return Some(rel.to_string()); - } - } - } - } else { - let rel = prop_path.trim_matches('/'); - if !rel.is_empty() { - return Some(rel.to_string()); - } - } + let normalized_path = raw.replace('\\', "/"); + let mut data_url = match ( + crate::utils::parse_absolute_url(raw), + crate::utils::file_url_from_absolute_path(&normalized_path), + ) { + (Some(prop_url), _) => prop_url, + (None, Some(file_url)) => file_url, + (None, None) => table_url + .join(&normalized_path) + .map_err(|e| DataFusionError::Plan(format!("Invalid Iceberg data path: {e}")))?, + }; + + // TODO: Remove this restriction once readers resolve each file URI through its object store. + let schemes_are_compatible = data_url.scheme() == table_url.scheme() + || matches!( + (table_url.scheme(), data_url.scheme()), + ("s3", "s3a") | ("s3a", "s3") + ); + if !schemes_are_compatible || data_url.authority() != table_url.authority() { + return Err(DataFusionError::Plan(format!( + "Iceberg data path {data_url} must use the same object store as table location \ + {table_url}" + ))); } - None + if !data_url.path().ends_with('/') { + data_url.set_path(&format!("{}/", data_url.path())); + } + Ok(Some(data_url)) } - fn resolve_data_dir_from_properties( + fn resolve_data_location_from_properties( properties: &std::collections::HashMap, table_url: &Url, - ) -> String { - Self::resolve_data_dir_from_property_value( + ) -> Result { + Self::resolve_data_location_from_property_value( properties .get("write.data.path") .or_else(|| properties.get("write.folder-storage.path")) .map(String::as_str), table_url, + )? + .map_or_else( + || { + table_url.join("data/").map_err(|e| { + DataFusionError::Plan(format!("Invalid default Iceberg data path: {e}")) + }) + }, + Ok, ) - .unwrap_or_else(|| "data".to_string()) } } @@ -376,14 +374,14 @@ impl ExecutionPlan for IcebergWriterExec { } } - let object_store = get_object_store_from_context(&context, &table_url)?; + let table_object_store = get_object_store_from_context(&context, &table_url)?; let input_schema = input_schema.clone(); let ( iceberg_schema, table_schema, default_spec, - data_dir, + data_location, spec_id_val, commit_schema, commit_requirements, @@ -394,21 +392,25 @@ impl ExecutionPlan for IcebergWriterExec { match metadata_location_from_properties(&options.table_properties) { Some(location) => metadata_location_to_object_path_string(&location)?, None => { - crate::table::find_latest_metadata_file(&object_store, &table_url) - .await? + crate::table::find_latest_metadata_file( + &table_object_store, + &table_url, + ) + .await? } } } else { - crate::table::find_latest_metadata_file(&object_store, &table_url).await? + crate::table::find_latest_metadata_file(&table_object_store, &table_url) + .await? }; let bytes = crate::table::metadata_loader::load_metadata_file_bytes( - &object_store, + &table_object_store, &latest_meta, ) .await?; let table_meta = TableMetadata::from_json(&bytes) .map_err(|e| DataFusionError::External(Box::new(e)))?; - let data_dir = Self::resolve_data_dir(&table_meta, &table_url); + let data_location = Self::resolve_data_location(&table_meta, &table_url)?; let variant_shredding = options.variant_shredding_config(&table_meta.properties)?; // FIXME: Concurrency Issue with Schema Evolution. // This requires a mechanism to reserve Field IDs or restart the Writer task upon conflict. @@ -478,7 +480,7 @@ impl ExecutionPlan for IcebergWriterExec { schema_outcome.iceberg_schema, schema_outcome.arrow_schema, default_spec, - data_dir, + data_location, spec_id_val, commit_schema, requirements, @@ -522,16 +524,19 @@ impl ExecutionPlan for IcebergWriterExec { iceberg_schema.clone(), Arc::new(iceberg_schema_to_arrow(&iceberg_schema)?), Some(spec), - Self::resolve_data_dir_from_property_value( + match Self::resolve_data_location_from_property_value( options .write_data_path .as_deref() .or(options.write_folder_storage_path.as_deref()), &table_url, - ) - .unwrap_or_else(|| { - Self::resolve_data_dir_from_properties(&metadata_properties, &table_url) - }), + )? { + Some(data_location) => data_location, + None => Self::resolve_data_location_from_properties( + &metadata_properties, + &table_url, + )?, + }, sid, Some(iceberg_schema), Vec::new(), @@ -568,15 +573,15 @@ impl ExecutionPlan for IcebergWriterExec { variant_shredding, }; - let writer_root = crate::utils::url_to_object_path(&table_url) + let data_object_store = get_object_store_from_context(&context, &data_location)?; + let writer_root = crate::utils::url_to_object_path(&data_location) .map_err(|e| DataFusionError::Plan(e.to_string()))?; let mut writer = IcebergTableWriter::new( - object_store.clone(), + data_object_store, writer_root, writer_config, spec_id_val, - data_dir, - table_url.clone(), + data_location, ); let mut total_rows = 0u64; diff --git a/crates/sail-iceberg/src/table_format.rs b/crates/sail-iceberg/src/table_format.rs index 341bb1e0c..21f141449 100644 --- a/crates/sail-iceberg/src/table_format.rs +++ b/crates/sail-iceberg/src/table_format.rs @@ -10,7 +10,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::path::Path; use std::sync::Arc; use async_trait::async_trait; @@ -756,7 +755,7 @@ impl IcebergTableFormat { let path = &paths[0]; let mut table_url = match crate::utils::parse_absolute_url(path) { Some(url) => url, - _ => file_url_from_absolute_path(path).ok_or_else(|| { + _ => crate::utils::file_url_from_absolute_path(path).ok_or_else(|| { DataFusionError::Plan(format!( "Iceberg table location must be an absolute path or URL: {path}" )) @@ -897,27 +896,6 @@ fn next_partition_spec_id(metadata: &TableMetadata) -> i32 { + 1 } -fn file_url_from_absolute_path(path: &str) -> Option { - if Path::new(path).is_absolute() { - return Url::from_file_path(path).ok(); - } - windows_drive_path_to_file_url(path) -} - -fn windows_drive_path_to_file_url(path: &str) -> Option { - let bytes = path.as_bytes(); - if bytes.len() < 3 - || !bytes[0].is_ascii_alphabetic() - || bytes[1] != b':' - || !matches!(bytes[2], b'/' | b'\\') - { - return None; - } - - let path = path.replace('\\', "/"); - Url::parse(&format!("file:///{path}")).ok() -} - pub(crate) fn table_metadata_location(table_url: &Url, metadata_file: &str) -> Result { if crate::utils::parse_absolute_url(metadata_file).is_some() { return Ok(metadata_file.to_string()); diff --git a/crates/sail-iceberg/src/utils/mod.rs b/crates/sail-iceberg/src/utils/mod.rs index b57b9d438..973251ced 100644 --- a/crates/sail-iceberg/src/utils/mod.rs +++ b/crates/sail-iceberg/src/utils/mod.rs @@ -17,6 +17,7 @@ pub mod snapshot_id; pub mod timestamp; pub mod transform; +use std::path::Path; use std::sync::Arc; use datafusion::catalog::Session; @@ -54,6 +55,27 @@ pub fn parse_absolute_url(raw: &str) -> Option { Url::parse(raw).ok().filter(|url| url.scheme().len() > 1) } +pub fn file_url_from_absolute_path(path: &str) -> Option { + if Path::new(path).is_absolute() { + return Url::from_file_path(path).ok(); + } + windows_drive_path_to_file_url(path) +} + +fn windows_drive_path_to_file_url(path: &str) -> Option { + let bytes = path.as_bytes(); + if bytes.len() < 3 + || !bytes[0].is_ascii_alphabetic() + || bytes[1] != b':' + || !matches!(bytes[2], b'/' | b'\\') + { + return None; + } + + let path = path.replace('\\', "/"); + Url::parse(&format!("file:///{path}")).ok() +} + pub fn url_to_object_path(url: &Url) -> Result { let is_file = url.scheme() == "file"; let p = if is_file { diff --git a/python/pysail/tests/spark/catalog/iceberg_rest/test_commit.py b/python/pysail/tests/spark/catalog/iceberg_rest/test_commit.py index 2cf1d7b40..11c778f76 100644 --- a/python/pysail/tests/spark/catalog/iceberg_rest/test_commit.py +++ b/python/pysail/tests/spark/catalog/iceberg_rest/test_commit.py @@ -7,7 +7,9 @@ from pathlib import PurePosixPath from typing import TYPE_CHECKING +import boto3 import pytest +from botocore.config import Config if TYPE_CHECKING: from collections.abc import Generator @@ -36,6 +38,22 @@ def _load_table(iceberg_rest_endpoint: str, table_name: str) -> dict: return json.load(response) +def _s3_object_keys(endpoint: str, location: str) -> set[str]: + parsed = urllib.parse.urlparse(location) + assert parsed.scheme == "s3" + prefix = parsed.path.lstrip("/").rstrip("/") + "/" + client = boto3.client( + "s3", + endpoint_url=endpoint, + aws_access_key_id="admin", + aws_secret_access_key="password", # noqa: S106 + region_name="us-east-1", + config=Config(signature_version="s3v4", s3={"addressing_style": "path"}), + ) + pages = client.get_paginator("list_objects_v2").paginate(Bucket=parsed.netloc, Prefix=prefix) + return {item["Key"] for page in pages for item in page.get("Contents", [])} + + def _assert_uuid_metadata_location(metadata_location: str, expected_version: int | None = None) -> None: filename = PurePosixPath(metadata_location).name assert UUID_METADATA_FILE_PATTERN.match(filename), filename @@ -121,6 +139,47 @@ def test_insert_advances_rest_catalog_metadata_location( assert [(row["id"], row["name"]) for row in rows] == [(1, "a"), (2, "b"), (3, "c")] +def test_rest_catalog_write_honors_absolute_data_path( + spark: SparkSession, + iceberg_rest_endpoint: str, + seaweedfs_host_endpoint: str, +) -> None: + table_name = "absolute_data_path_t" + table_fqn = f"{NAMESPACE}.{table_name}" + data_location = f"s3://icebergdata/managed-data/{NAMESPACE}/{table_name}" + spark.sql(f"DROP TABLE IF EXISTS {table_fqn}") + spark.sql( + f""" + CREATE TABLE {table_fqn} ( + id INT, + name STRING + ) + USING iceberg + TBLPROPERTIES ('write.data.path' = '{data_location}') + """ + ) + + created = _load_table(iceberg_rest_endpoint, table_name) + assert created["metadata"]["properties"]["write.data.path"] == data_location + + spark.sql(f"INSERT INTO {table_fqn} VALUES (1, 'one')") # noqa: S608 + + committed = _load_table(iceberg_rest_endpoint, table_name) + assert committed["metadata-location"] != created["metadata-location"] + assert committed["metadata"]["current-snapshot-id"] not in (None, -1) + table_location = committed["metadata"]["location"] + assert table_location != data_location + assert committed["metadata"]["properties"]["write.data.path"] == data_location + + data_keys = _s3_object_keys(seaweedfs_host_endpoint, data_location) + assert any(key.endswith(".parquet") for key in data_keys) + table_keys = _s3_object_keys(seaweedfs_host_endpoint, table_location) + assert not any(key.endswith(".parquet") for key in table_keys) + + rows = spark.sql(f"SELECT id, name FROM {table_fqn}").collect() # noqa: S608 + assert [(row.id, row.name) for row in rows] == [(1, "one")] + + def test_merge_schema_append_advances_rest_catalog_metadata_location( spark: SparkSession, iceberg_rest_endpoint: str, diff --git a/python/pysail/tests/spark/catalog/iceberg_rest/test_nessie.py b/python/pysail/tests/spark/catalog/iceberg_rest/test_nessie.py index 8e3650a5b..c38040c60 100644 --- a/python/pysail/tests/spark/catalog/iceberg_rest/test_nessie.py +++ b/python/pysail/tests/spark/catalog/iceberg_rest/test_nessie.py @@ -7,6 +7,7 @@ import pytest from pysail.testing.spark.session import spark_connect_server +from pysail.tests.spark.catalog.iceberg_rest.test_commit import _s3_object_keys if TYPE_CHECKING: from collections.abc import Generator @@ -15,10 +16,23 @@ @pytest.fixture(scope="module") -def remote(nessie_iceberg_rest_endpoint: str) -> Generator[str, None, None]: +def remote( + nessie_iceberg_rest_endpoint: str, + seaweedfs_host_endpoint: str, +) -> Generator[str, None, None]: """Start Sail server with Nessie as the Iceberg REST catalog.""" catalog_config = f'[{{name="sail", type="iceberg-rest", uri="{nessie_iceberg_rest_endpoint}"}}]' - with spark_connect_server(envs={"SAIL_CATALOG__LIST": catalog_config}) as server: + with spark_connect_server( + envs={ + "SAIL_CATALOG__LIST": catalog_config, + "AWS_ACCESS_KEY_ID": "admin", + "AWS_SECRET_ACCESS_KEY": "password", + "AWS_REGION": "us-east-1", + "AWS_ENDPOINT": seaweedfs_host_endpoint, + "AWS_VIRTUAL_HOSTED_STYLE_REQUEST": "false", + "AWS_ALLOW_HTTP": "true", + }, + ) as server: yield server.remote @@ -39,3 +53,34 @@ def test_create_table_with_nessie_catalog(spark: SparkSession) -> None: assert rows[0].isTemporary is False finally: spark.sql(f"DROP DATABASE IF EXISTS {namespace} CASCADE").collect() + + +def test_nessie_write_honors_absolute_data_path( + spark: SparkSession, + seaweedfs_host_endpoint: str, +) -> None: + namespace = "nessie_absolute_data_path" + table = f"{namespace}.t1" + data_location = f"s3://icebergdata/managed-data/{namespace}/t1" + warehouse_namespace = f"s3://icebergdata/nessie/{namespace}" + + try: + spark.sql(f"CREATE DATABASE IF NOT EXISTS {namespace}").collect() + spark.sql( + f""" + CREATE TABLE {table} (id INT, name STRING) + USING iceberg + TBLPROPERTIES ('write.data.path' = '{data_location}') + """ + ).collect() + spark.sql(f"INSERT INTO {table} VALUES (1, 'one')") # noqa: S608 + + data_keys = _s3_object_keys(seaweedfs_host_endpoint, data_location) + assert any(key.endswith(".parquet") for key in data_keys) + warehouse_keys = _s3_object_keys(seaweedfs_host_endpoint, warehouse_namespace) + assert not any(key.endswith(".parquet") for key in warehouse_keys) + + rows = spark.sql(f"SELECT id, name FROM {table}").collect() # noqa: S608 + assert [(row.id, row.name) for row in rows] == [(1, "one")] + finally: + spark.sql(f"DROP DATABASE IF EXISTS {namespace} CASCADE").collect() diff --git a/python/pysail/tests/spark/iceberg/features/alter_table.feature b/python/pysail/tests/spark/iceberg/features/alter_table.feature index 99961ceac..f1ea0f7de 100644 --- a/python/pysail/tests/spark/iceberg/features/alter_table.feature +++ b/python/pysail/tests/spark/iceberg/features/alter_table.feature @@ -36,6 +36,46 @@ Feature: Iceberg ALTER TABLE SET/UNSET TBLPROPERTIES 📄 snap-*.avro """ + Scenario: Absolute write data path stores files outside the table location + Given variable location for temporary directory iceberg_absolute_write_data_path_table + Given variable data_location for temporary directory iceberg_absolute_write_data_path_data + Given final statement + """ + DROP TABLE IF EXISTS iceberg_absolute_write_data_path_test + """ + Given statement template + """ + CREATE TABLE iceberg_absolute_write_data_path_test ( + id INT, + value STRING + ) + USING iceberg + LOCATION {{ location.uri }} + TBLPROPERTIES ('write.data.path' = '{{ data_location.file_uri }}') + """ + Given statement + """ + INSERT INTO iceberg_absolute_write_data_path_test VALUES (1, 'v0') + """ + Then file tree in location matches + """ + 📂 metadata + 📄 *.metadata.json + 📄 *.metadata.json + 📄 snap-*.avro + """ + Then file tree in data_location matches + """ + 📄 *.parquet + """ + When query + """ + SELECT * FROM iceberg_absolute_write_data_path_test + """ + Then query result + | id | value | + | 1 | v0 | + Scenario: CREATE TABLE format-version selects metadata version without persisting reserved properties Given variable location for temporary directory iceberg_create_table_format_version Given final statement diff --git a/python/pysail/tests/spark/iceberg/test_iceberg_write.py b/python/pysail/tests/spark/iceberg/test_iceberg_write.py index 7b852fbf2..59c166a9b 100644 --- a/python/pysail/tests/spark/iceberg/test_iceberg_write.py +++ b/python/pysail/tests/spark/iceberg/test_iceberg_write.py @@ -137,6 +137,37 @@ def test_iceberg_write_append_mode(spark, sql_catalog): sql_catalog.drop_table(identifier) +def test_iceberg_write_honors_absolute_data_path(spark, sql_catalog, tmp_path): + identifier = "default.write_absolute_data_path" + table_path = tmp_path / "absolute_data_path_table" + data_path = tmp_path / "absolute_data_path_files" + table = sql_catalog.create_table( + identifier=identifier, + location=table_path.as_uri(), + schema=Schema( + NestedField(field_id=1, name="id", field_type=LongType(), required=False), + NestedField(field_id=2, name="value", field_type=StringType(), required=False), + ), + properties={"write.data.path": data_path.as_uri()}, + ) + try: + df = spark.createDataFrame([(1, "v0")], schema="id LONG, value STRING") + df.write.format("iceberg").mode("append").save(table.location()) + + data_files = set(data_path.rglob("*.parquet")) + assert data_files + assert not set(table_path.rglob("*.parquet")) + + static_table = StaticTable.from_metadata(table.location(), properties=pyiceberg_file_io_properties()) + manifest_data_files = {task.file.file_path for task in static_table.scan().plan_files()} + assert manifest_data_files == {path.absolute().as_uri() for path in data_files} + + rows = [tuple(row) for row in spark.read.format("iceberg").load(table.location()).collect()] + assert rows == [(1, "v0")] + finally: + sql_catalog.drop_table(identifier) + + def test_iceberg_sql_read_after_write(spark, sql_catalog): identifier = "default.write_sql_table" table = sql_catalog.create_table( From 160e53e43a28a5e02ea73ff897957103106b2d19 Mon Sep 17 00:00:00 2001 From: Heran Lin Date: Thu, 30 Jul 2026 16:53:16 +0800 Subject: [PATCH 095/131] feat: heartbeat for command execution (#2323) --- crates/sail-spark-connect/src/executor.rs | 282 +++++++++++++----- .../sail-spark-connect/src/proto/function.rs | 4 +- .../src/service/plan_executor.rs | 179 ++++++----- python/pysail/tests/spark/session/__init__.py | 0 .../tests/spark/session/test_heartbeat.py | 74 +++++ .../session/test_reattachable_execution.py | 31 ++ 6 files changed, 404 insertions(+), 166 deletions(-) create mode 100644 python/pysail/tests/spark/session/__init__.py create mode 100644 python/pysail/tests/spark/session/test_heartbeat.py create mode 100644 python/pysail/tests/spark/session/test_reattachable_execution.py diff --git a/crates/sail-spark-connect/src/executor.rs b/crates/sail-spark-connect/src/executor.rs index 33f8af93c..0ebc2ad21 100644 --- a/crates/sail-spark-connect/src/executor.rs +++ b/crates/sail-spark-connect/src/executor.rs @@ -7,12 +7,13 @@ use std::sync::{Arc, Mutex}; use std::time::Duration; use datafusion::arrow::array::RecordBatch; +use datafusion::arrow::datatypes::SchemaRef; use datafusion::arrow::ipc::writer::StreamWriter; use datafusion::execution::SendableRecordBatchStream; use fastrace::Span; use fastrace::future::FutureExt; use futures::Stream; -use futures::stream::{StreamExt, TryStreamExt}; +use futures::stream::StreamExt; use tokio::sync::{mpsc, oneshot}; use tokio::task::JoinHandle; use tonic::codegen::tokio_stream::wrappers::ReceiverStream; @@ -28,6 +29,7 @@ use crate::spark::connect::{ #[derive(Clone, Debug)] pub enum ExecutorBatch { + Heartbeat, ArrowBatch(ArrowBatch), SqlCommandResult(Box), WriteStreamOperationStartResult(Box), @@ -57,23 +59,30 @@ impl ExecutorOutput { } } -pub type ExecutorOutputStream = Pin + Send>>; +pub type ExecutorOutputStream = Pin> + Send>>; struct ExecutorBuffer { + capacity: usize, inner: VecDeque, } +// TODO: use "spark.connect.execute.reattachable.observerRetryBufferSize" +// TODO: limit the size based on serialized message size instead of element count +const EXECUTOR_BUFFER_CAPACITY: usize = 128; + impl ExecutorBuffer { - fn new() -> Self { - // TODO: use "spark.connect.execute.reattachable.observerRetryBufferSize" - // TODO: limit the size based on serialized message size instead of element count + fn new(capacity: usize) -> Self { Self { - inner: VecDeque::with_capacity(128), + capacity, + inner: VecDeque::with_capacity(capacity), } } fn add(&mut self, output: ExecutorOutput) { - if self.inner.len() >= self.inner.capacity() { + if self.capacity == 0 { + return; + } + if self.inner.len() >= self.capacity { self.inner.pop_front(); } self.inner.push_back(output); @@ -125,38 +134,76 @@ struct ExecutorTask { struct ExecutorTaskContext { stream: SendableRecordBatchStream, heartbeat_interval: Duration, + state: ExecutorTaskState, buffer: Arc>, } -impl ExecutorTaskContext { - fn new(stream: SendableRecordBatchStream, heartbeat_interval: Duration) -> Self { - Self { - stream, - heartbeat_interval, - buffer: Arc::new(Mutex::new(ExecutorBuffer::new())), +pub(crate) enum ExecutorMode { + Query, + Command { + completion: CommandCompletionHandler, + }, +} + +impl ExecutorMode { + pub(crate) fn command() -> Self { + Self::command_with_completion(|_, _| Ok(None)) + } + + pub(crate) fn command_with_completion( + completion: impl FnOnce(SchemaRef, Vec) -> SparkResult> + + Send + + 'static, + ) -> Self { + Self::Command { + completion: Box::new(completion), } } +} - async fn next(&mut self) -> SparkResult> { +pub(crate) enum ExecutorTaskState { + Query, + Command { + batches: Vec, + completion: Option, + }, +} + +enum ExecutorTaskItem { + Batch(Option), + Heartbeat, +} + +pub(crate) type CommandCompletionHandler = + Box) -> SparkResult> + Send>; + +impl ExecutorTaskContext { + async fn next( + stream: &mut SendableRecordBatchStream, + heartbeat_interval: Duration, + ) -> SparkResult { let span = Span::enter_with_local_parent("ExecutorTaskContext::next"); tokio::select! { - batch = self.stream.next().in_span(span) => Ok(batch.transpose()?), - _ = tokio::time::sleep(self.heartbeat_interval) => { - Ok(Some(RecordBatch::new_empty(self.stream.schema()))) + batch = stream.next().in_span(span) => Ok(ExecutorTaskItem::Batch(batch.transpose()?)), + _ = tokio::time::sleep(heartbeat_interval) => { + // FIXME: non-reattachable clients cannot refresh session activity by releasing heartbeat responses + Ok(ExecutorTaskItem::Heartbeat) } } } - fn save_output(&self, output: &ExecutorOutput) -> SparkResult<()> { - let mut buffer = self.buffer.lock()?; - buffer.add(output.clone()); + async fn send_output( + buffer: &Arc>, + tx: &mpsc::Sender>, + output: ExecutorOutput, + ) -> SparkResult<()> { + { + let mut buffer = buffer.lock()?; + buffer.add(output.clone()); + } + tx.send(Ok(output)).await?; Ok(()) } - - fn replay_outputs(&self) -> SparkResult> { - let buffer = self.buffer.lock()?; - Ok(buffer.iter().cloned().collect()) - } } enum ExecutorTaskResult { @@ -170,11 +217,29 @@ impl Executor { metadata: ExecutorMetadata, stream: SendableRecordBatchStream, heartbeat_interval: Duration, + mode: ExecutorMode, ) -> Self { + let state = match mode { + ExecutorMode::Query => ExecutorTaskState::Query, + ExecutorMode::Command { completion } => ExecutorTaskState::Command { + batches: vec![], + completion: Some(completion), + }, + }; + let buffer = if metadata.reattachable { + ExecutorBuffer::new(EXECUTOR_BUFFER_CAPACITY) + } else { + ExecutorBuffer::new(0) + }; Self { metadata, state: Mutex::new(ExecutorState::Pending { - context: ExecutorTaskContext::new(stream, heartbeat_interval), + context: ExecutorTaskContext { + stream, + heartbeat_interval, + state, + buffer: Arc::new(Mutex::new(buffer)), + }, span: Span::enter_with_local_parent("Executor::new"), }), } @@ -182,55 +247,139 @@ impl Executor { async fn run_internal( context: &mut ExecutorTaskContext, - tx: mpsc::Sender, + tx: &mpsc::Sender>, + reattachable: bool, ) -> SparkResult<()> { - for out in context.replay_outputs()? { - tx.send(out).await?; - } - let schema = to_spark_schema(context.stream.schema())?; - let out = ExecutorOutput::new(ExecutorBatch::Schema(Box::new(schema))); - context.save_output(&out)?; - tx.send(out).await?; - - let mut empty = true; - while let Some(batch) = context.next().await? { - let batch = to_arrow_batch(&batch)?; - let out = ExecutorOutput::new(ExecutorBatch::ArrowBatch(batch)); - context.save_output(&out)?; - tx.send(out).await?; - empty = false; + let outputs = { + let buffer = context.buffer.lock()?; + buffer.iter().cloned().collect::>() + }; + for output in outputs { + tx.send(Ok(output)).await?; } - if empty { - let batch = RecordBatch::new_empty(context.stream.schema()); - let batch = to_arrow_batch(&batch)?; - let out = ExecutorOutput::new(ExecutorBatch::ArrowBatch(batch)); - context.save_output(&out)?; - tx.send(out).await?; + match &mut context.state { + ExecutorTaskState::Query => { + let schema = to_spark_schema(context.stream.schema())?; + ExecutorTaskContext::send_output( + &context.buffer, + tx, + ExecutorOutput::new(ExecutorBatch::Schema(Box::new(schema))), + ) + .await?; + + let mut empty = true; + loop { + match ExecutorTaskContext::next(&mut context.stream, context.heartbeat_interval) + .await? + { + ExecutorTaskItem::Batch(Some(batch)) => { + let batch = to_arrow_batch(&batch)?; + ExecutorTaskContext::send_output( + &context.buffer, + tx, + ExecutorOutput::new(ExecutorBatch::ArrowBatch(batch)), + ) + .await?; + empty = false; + } + ExecutorTaskItem::Batch(None) => break, + ExecutorTaskItem::Heartbeat => { + ExecutorTaskContext::send_output( + &context.buffer, + tx, + ExecutorOutput::new(ExecutorBatch::Heartbeat), + ) + .await?; + } + } + } + if empty { + let batch = RecordBatch::new_empty(context.stream.schema()); + let batch = to_arrow_batch(&batch)?; + ExecutorTaskContext::send_output( + &context.buffer, + tx, + ExecutorOutput::new(ExecutorBatch::ArrowBatch(batch)), + ) + .await?; + } + if reattachable { + ExecutorTaskContext::send_output( + &context.buffer, + tx, + ExecutorOutput::complete(), + ) + .await?; + } + } + ExecutorTaskState::Command { + batches, + completion, + } => { + loop { + match ExecutorTaskContext::next(&mut context.stream, context.heartbeat_interval) + .await? + { + ExecutorTaskItem::Batch(Some(batch)) => { + batches.push(batch); + } + ExecutorTaskItem::Batch(None) => break, + ExecutorTaskItem::Heartbeat => { + ExecutorTaskContext::send_output( + &context.buffer, + tx, + ExecutorOutput::new(ExecutorBatch::Heartbeat), + ) + .await?; + } + } + } + let schema = context.stream.schema(); + let output = completion + .take() + .map(|completion| completion(schema, mem::take(batches))) + .transpose()? + .flatten(); + if let Some(output) = output { + ExecutorTaskContext::send_output(&context.buffer, tx, output).await?; + } + if reattachable { + ExecutorTaskContext::send_output( + &context.buffer, + tx, + ExecutorOutput::complete(), + ) + .await?; + } + } } - - let out = ExecutorOutput::new(ExecutorBatch::Complete); - context.save_output(&out)?; - tx.send(out).await?; Ok(()) } async fn run( mut context: ExecutorTaskContext, listener: oneshot::Receiver<()>, - tx: mpsc::Sender, + tx: mpsc::Sender>, + reattachable: bool, ) -> ExecutorTaskResult { let out = tokio::select! { - x = Executor::run_internal(&mut context, tx) => x, + x = Executor::run_internal(&mut context, &tx, reattachable) => x, _ = listener => return ExecutorTaskResult::Paused(context), }; match out { Ok(()) => ExecutorTaskResult::Completed, Err(SparkError::SendError(_)) => ExecutorTaskResult::Paused(context), - Err(e) => ExecutorTaskResult::Failed(e), + Err(e) => { + let _ = tx.send(Err(e)).await; + // TODO: track the original error in the task result + ExecutorTaskResult::Failed(SparkError::internal( + "task failed while executing the plan", + )) + } } } - pub(crate) fn start(&self) -> SparkResult> { + pub(crate) fn start(&self) -> SparkResult { let mut state = self.state.lock()?; let (context, span) = match mem::replace(state.deref_mut(), ExecutorState::Idle) { ExecutorState::Pending { context, span } => (context, span), @@ -256,9 +405,14 @@ impl Executor { let (tx, rx) = mpsc::channel(1); let (notifier, listener) = oneshot::channel(); let buffer = Arc::clone(&context.buffer); + let reattachable = self.metadata.reattachable; let handle = { let span = { Span::enter_with_parent("Executor::run", &span) }; - tokio::spawn(async move { Executor::run(context, listener, tx).in_span(span).await }) + tokio::spawn(async move { + Executor::run(context, listener, tx, reattachable) + .in_span(span) + .await + }) }; *state = ExecutorState::Running { task: ExecutorTask { @@ -268,7 +422,7 @@ impl Executor { }, span, }; - Ok(ReceiverStream::new(rx)) + Ok(Box::pin(ReceiverStream::new(rx))) } pub(crate) async fn pause_if_running(&self) -> SparkResult<()> { @@ -295,7 +449,7 @@ impl Executor { Ok(()) } - pub(crate) fn release(&self, response_id: Option) -> SparkResult<()> { + pub(crate) fn release(&self, response_id: String) -> SparkResult<()> { let state = self.state.lock()?; let buffer = match state.deref() { ExecutorState::Running { task, span: _ } => &task.buffer, @@ -304,19 +458,11 @@ impl Executor { return Ok(()); } }; - if let Some(response_id) = response_id { - buffer.lock()?.remove_until(&response_id); - } + buffer.lock()?.remove_until(&response_id); Ok(()) } } -pub(crate) async fn read_stream( - stream: SendableRecordBatchStream, -) -> SparkResult> { - stream.err_into().try_collect::>().await -} - pub(crate) fn to_arrow_batch(batch: &RecordBatch) -> SparkResult { let mut output = ArrowBatch::default(); { diff --git a/crates/sail-spark-connect/src/proto/function.rs b/crates/sail-spark-connect/src/proto/function.rs index 98618562d..877c04f73 100644 --- a/crates/sail-spark-connect/src/proto/function.rs +++ b/crates/sail-spark-connect/src/proto/function.rs @@ -7,6 +7,7 @@ mod tests { use datafusion::arrow::array::RecordBatch; use datafusion::arrow::error::ArrowError; use datafusion::arrow::util::display::{ArrayFormatter, FormatOptions}; + use futures::stream::TryStreamExt; use pyo3::Python; use sail_common::config::AppConfig; use sail_common::runtime::RuntimeManager; @@ -17,7 +18,6 @@ mod tests { use serde::{Deserialize, Serialize}; use crate::error::{SparkError, SparkResult}; - use crate::executor::read_stream; use crate::proto::data_type_json::JsonDataType; use crate::session::SparkSession; use crate::session_manager::create_spark_session_manager; @@ -91,7 +91,7 @@ mod tests { let (plan, _) = resolve_and_execute_plan(&context, spark.plan_config()?, plan).await?; let stream = service.runner().execute(&context, plan).await?; - read_stream(stream).await + stream.err_into().try_collect::>().await }); // TODO: validate the result against the expected output // TODO: handle non-deterministic results and error messages diff --git a/crates/sail-spark-connect/src/service/plan_executor.rs b/crates/sail-spark-connect/src/service/plan_executor.rs index 8c7e67ab8..b86a50ae6 100644 --- a/crates/sail-spark-connect/src/service/plan_executor.rs +++ b/crates/sail-spark-connect/src/service/plan_executor.rs @@ -14,11 +14,10 @@ use sail_common_datafusion::session::job::JobService; use sail_plan::resolve_and_execute_plan; use tonic::Status; use tonic::codegen::tokio_stream::Stream; -use tonic::codegen::tokio_stream::wrappers::ReceiverStream; use crate::error::{ProtoFieldExt, SparkError, SparkResult}; use crate::executor::{ - Executor, ExecutorBatch, ExecutorMetadata, ExecutorOutput, ExecutorOutputStream, read_stream, + Executor, ExecutorBatch, ExecutorMetadata, ExecutorMode, ExecutorOutput, ExecutorOutputStream, to_arrow_batch, }; use crate::session::SparkSession; @@ -62,12 +61,14 @@ impl Stream for ExecutePlanResponseStream { ) -> Poll>> { self.inner.as_mut().poll_next(cx).map(|poll| { poll.map(|item| { + let item = item.map_err(Status::from)?; let mut response = ExecutePlanResponse::default(); response.session_id.clone_from(&self.session_id); response.server_side_session_id.clone_from(&self.session_id); response.operation_id.clone_from(&self.operation_id.clone()); response.response_id = item.id; match item.batch { + ExecutorBatch::Heartbeat => {} ExecutorBatch::ArrowBatch(batch) => { response.response_type = Some(ResponseType::ArrowBatch(batch)); } @@ -109,19 +110,11 @@ impl Stream for ExecutePlanResponseStream { } } -enum ExecutePlanMode { - /// Execute the plan lazily as the client reads the response stream. - Lazy, - /// Execute the plan eagerly and return an empty response stream. - /// This is useful for executing command plans. - EagerSilent, -} - async fn handle_execute_plan( ctx: &SessionContext, plan: spec::Plan, metadata: ExecutorMetadata, - mode: ExecutePlanMode, + mode: ExecutorMode, ) -> SparkResult { let span = Span::root("handle_execute_plan", SpanContext::random()); let spark = ctx.extension::()?; @@ -132,31 +125,19 @@ async fn handle_execute_plan( let span = Span::enter_with_parent("JobRunner::execute", &span); service.runner().execute(ctx, plan).in_span(span).await? }; - let rx = match mode { - ExecutePlanMode::Lazy => { - let _guard = span.set_local_parent(); - let executor = Executor::new( - metadata, - stream, - spark.options().execution_heartbeat_interval, - ); - let rx = executor.start()?; - spark.add_executor(executor)?; - rx - } - ExecutePlanMode::EagerSilent => { - let _ = read_stream(stream).in_span(span).await?; - let (tx, rx) = tokio::sync::mpsc::channel(1); - if metadata.reattachable { - tx.send(ExecutorOutput::complete()).await?; - } - ReceiverStream::new(rx) - } - }; + let _guard = span.set_local_parent(); + let executor = Executor::new( + metadata, + stream, + spark.options().execution_heartbeat_interval, + mode, + ); + let rx = executor.start()?; + spark.add_executor(executor)?; Ok(ExecutePlanResponseStream::new( spark.session_id().to_string(), operation_id, - Box::pin(rx), + rx, )) } @@ -166,7 +147,7 @@ pub(crate) async fn handle_execute_relation( metadata: ExecutorMetadata, ) -> SparkResult { let plan = relation.try_into()?; - handle_execute_plan(ctx, plan, metadata, ExecutePlanMode::Lazy).await + handle_execute_plan(ctx, plan, metadata, ExecutorMode::Query).await } pub(crate) async fn handle_execute_register_function( @@ -177,7 +158,8 @@ pub(crate) async fn handle_execute_register_function( let plan = spec::Plan::Command(spec::CommandPlan::new(spec::CommandNode::RegisterFunction( udf.try_into()?, ))); - handle_execute_plan(ctx, plan, metadata, ExecutePlanMode::EagerSilent).await + let mode = ExecutorMode::command(); + handle_execute_plan(ctx, plan, metadata, mode).await } pub(crate) async fn handle_execute_write_operation( @@ -188,7 +170,8 @@ pub(crate) async fn handle_execute_write_operation( let plan = spec::Plan::Command(spec::CommandPlan::new(spec::CommandNode::Write( write.try_into()?, ))); - handle_execute_plan(ctx, plan, metadata, ExecutePlanMode::EagerSilent).await + let mode = ExecutorMode::command(); + handle_execute_plan(ctx, plan, metadata, mode).await } pub(crate) async fn handle_execute_create_dataframe_view( @@ -197,7 +180,8 @@ pub(crate) async fn handle_execute_create_dataframe_view( metadata: ExecutorMetadata, ) -> SparkResult { let plan = spec::Plan::Command(spec::CommandPlan::new(view.try_into()?)); - handle_execute_plan(ctx, plan, metadata, ExecutePlanMode::EagerSilent).await + let mode = ExecutorMode::command(); + handle_execute_plan(ctx, plan, metadata, mode).await } pub(crate) async fn handle_execute_write_operation_v2( @@ -208,7 +192,8 @@ pub(crate) async fn handle_execute_write_operation_v2( let plan = spec::Plan::Command(spec::CommandPlan::new(spec::CommandNode::WriteTo( write.try_into()?, ))); - handle_execute_plan(ctx, plan, metadata, ExecutePlanMode::EagerSilent).await + let mode = ExecutorMode::command(); + handle_execute_plan(ctx, plan, metadata, mode).await } pub(crate) async fn handle_execute_merge_into_table_command( @@ -217,7 +202,8 @@ pub(crate) async fn handle_execute_merge_into_table_command( metadata: ExecutorMetadata, ) -> SparkResult { let plan = spec::Plan::Command(spec::CommandPlan::new(command.try_into()?)); - handle_execute_plan(ctx, plan, metadata, ExecutePlanMode::EagerSilent).await + let mode = ExecutorMode::command(); + handle_execute_plan(ctx, plan, metadata, mode).await } /// Handles execution of a SQL command. @@ -228,7 +214,6 @@ pub(crate) async fn handle_execute_sql_command( metadata: ExecutorMetadata, ) -> SparkResult { let spark = ctx.extension::()?; - let service = ctx.extension::()?; let relation = if let Some(input) = sql.input { input } else { @@ -245,35 +230,40 @@ pub(crate) async fn handle_execute_sql_command( } }; let plan: spec::Plan = relation.clone().try_into()?; - let relation = match plan { - spec::Plan::Query(_) => relation, - command @ spec::Plan::Command(_) => { - let (plan, _) = resolve_and_execute_plan(ctx, spark.plan_config()?, command).await?; - let stream = service.runner().execute(ctx, plan).await?; - let schema = stream.schema(); - let data = read_stream(stream).await?; - let data = concat_batches(&schema, data.iter())?; - Relation { - common: None, - rel_type: Some(relation::RelType::LocalRelation(LocalRelation { - data: Some(to_arrow_batch(&data)?.data), - schema: None, - })), + match plan { + spec::Plan::Command(command) => { + let mode = ExecutorMode::command_with_completion(move |schema, data| { + let data = concat_batches(&schema, data.iter())?; + let relation = Relation { + common: None, + rel_type: Some(relation::RelType::LocalRelation(LocalRelation { + data: Some(to_arrow_batch(&data)?.data), + schema: None, + })), + }; + Ok(Some(ExecutorOutput::new(ExecutorBatch::SqlCommandResult( + Box::new(SqlCommandResult { + relation: Some(relation), + }), + )))) + }); + handle_execute_plan(ctx, spec::Plan::Command(command), metadata, mode).await + } + spec::Plan::Query(_) => { + let result = ExecutorBatch::SqlCommandResult(Box::new(SqlCommandResult { + relation: Some(relation), + })); + let mut output = vec![ExecutorOutput::new(result)]; + if metadata.reattachable { + output.push(ExecutorOutput::complete()); } + Ok(ExecutePlanResponseStream::new( + spark.session_id().to_string(), + metadata.operation_id, + Box::pin(stream::iter(output.into_iter().map(Ok))), + )) } - }; - let result = ExecutorBatch::SqlCommandResult(Box::new(SqlCommandResult { - relation: Some(relation), - })); - let mut output = vec![ExecutorOutput::new(result)]; - if metadata.reattachable { - output.push(ExecutorOutput::complete()); } - Ok(ExecutePlanResponseStream::new( - spark.session_id().to_string(), - metadata.operation_id, - Box::pin(stream::iter(output)), - )) } pub(crate) async fn handle_execute_write_stream_operation_start( @@ -305,7 +295,7 @@ pub(crate) async fn handle_execute_write_stream_operation_start( Ok(ExecutePlanResponseStream::new( spark.session_id().to_string(), operation_id, - Box::pin(stream::iter(output)), + Box::pin(stream::iter(output.into_iter().map(Ok))), )) } @@ -406,7 +396,7 @@ pub(crate) async fn handle_execute_streaming_query_command( Ok(ExecutePlanResponseStream::new( spark.session_id().to_string(), metadata.operation_id, - Box::pin(stream::iter(output)), + Box::pin(stream::iter(output.into_iter().map(Ok))), )) } @@ -489,7 +479,7 @@ pub(crate) async fn handle_execute_streaming_query_manager_command( Ok(ExecutePlanResponseStream::new( spark.session_id().to_string(), metadata.operation_id, - Box::pin(stream::iter(output)), + Box::pin(stream::iter(output.into_iter().map(Ok))), )) } @@ -501,7 +491,8 @@ pub(crate) async fn handle_execute_register_table_function( let plan = spec::Plan::Command(spec::CommandPlan::new( spec::CommandNode::RegisterTableFunction(udtf.try_into()?), )); - handle_execute_plan(ctx, plan, metadata, ExecutePlanMode::EagerSilent).await + let mode = ExecutorMode::command(); + handle_execute_plan(ctx, plan, metadata, mode).await } pub(crate) async fn handle_execute_streaming_query_listener_bus_command( @@ -519,7 +510,6 @@ pub(crate) async fn handle_execute_checkpoint_command( checkpoint: CheckpointCommand, metadata: ExecutorMetadata, ) -> SparkResult { - let spark = ctx.extension::()?; let CheckpointCommand { relation, local: _, @@ -543,24 +533,14 @@ pub(crate) async fn handle_execute_checkpoint_command( input: Box::new(query), }, )); - let (plan, _) = resolve_and_execute_plan(ctx, spark.plan_config()?, plan).await?; - let service = ctx.extension::()?; - let stream = service.runner().execute(ctx, plan).await?; - let _ = read_stream(stream).await?; - let result = ExecutorOutput::new(ExecutorBatch::CheckpointCommandResult(Box::new( - CheckpointCommandResult { - relation: Some(CachedRemoteRelation { relation_id }), - }, - ))); - let mut output = vec![result]; - if metadata.reattachable { - output.push(ExecutorOutput::complete()); - } - Ok(ExecutePlanResponseStream::new( - spark.session_id().to_string(), - metadata.operation_id, - Box::pin(stream::iter(output)), - )) + let mode = ExecutorMode::command_with_completion(move |_, _| { + Ok(Some(ExecutorOutput::new( + ExecutorBatch::CheckpointCommandResult(Box::new(CheckpointCommandResult { + relation: Some(CachedRemoteRelation { relation_id }), + })), + ))) + }); + handle_execute_plan(ctx, plan, metadata, mode).await } pub(crate) async fn handle_execute_remove_cached_remote_relation_command( @@ -579,7 +559,7 @@ pub(crate) async fn handle_execute_remove_cached_remote_relation_command( Ok(ExecutePlanResponseStream::new( spark.session_id().to_string(), metadata.operation_id, - Box::pin(stream::iter(output)), + Box::pin(stream::iter(output.into_iter().map(Ok))), )) } @@ -635,12 +615,14 @@ pub(crate) async fn handle_reattach_execute( ))); } executor.pause_if_running().await?; - executor.release(response_id)?; + if let Some(response_id) = response_id { + executor.release(response_id)?; + } let rx = executor.start()?; Ok(ExecutePlanResponseStream::new( spark.session_id().to_string(), operation_id, - Box::pin(rx), + rx, )) } @@ -650,10 +632,15 @@ pub(crate) async fn handle_release_execute( response_id: Option, ) -> SparkResult<()> { let spark = ctx.extension::()?; - // Some operations may not have an executor (e.g. DDL statements), - // so it is a no-op if the executor is not found. - if let Some(executor) = spark.get_executor(operation_id.as_str())? { + let executor = spark.get_executor(operation_id.as_str())?; + // TODO: clean up non-reattachable executors which the client does not release explicitly + let Some(executor) = executor.filter(|executor| executor.metadata.reattachable) else { + return Ok(()); + }; + if let Some(response_id) = response_id { executor.release(response_id)?; + } else if let Some(executor) = spark.remove_executor(operation_id.as_str())? { + executor.pause_if_running().await?; } Ok(()) } @@ -717,6 +704,6 @@ pub(crate) async fn handle_execute_register_datasource( Ok(ExecutePlanResponseStream::new( spark.session_id().to_string(), metadata.operation_id, - Box::pin(stream::iter(output)), + Box::pin(stream::iter(output.into_iter().map(Ok))), )) } diff --git a/python/pysail/tests/spark/session/__init__.py b/python/pysail/tests/spark/session/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/python/pysail/tests/spark/session/test_heartbeat.py b/python/pysail/tests/spark/session/test_heartbeat.py new file mode 100644 index 000000000..24c5bbdc0 --- /dev/null +++ b/python/pysail/tests/spark/session/test_heartbeat.py @@ -0,0 +1,74 @@ +import time + +import pyspark.sql.functions as F # noqa: N812 +import pytest +from pyspark.errors.exceptions.connect import SparkConnectGrpcException +from pyspark.sql.types import LongType, Row + +from pysail.testing.spark.session import spark_connect_server, spark_session_factory + +pytestmark = pytest.mark.skip(reason="slow tests that should be run manually") + + +@pytest.fixture(scope="module") +def bad_sessions(): + envs = { + "SAIL_SPARK__SESSION_TIMEOUT_SECS": "2", + "SAIL_SPARK__EXECUTION_HEARTBEAT_INTERVAL_SECS": "4", + } + with spark_connect_server(envs=envs) as server, spark_session_factory(server.remote) as sessions: + yield sessions + + +@pytest.fixture(scope="module") +def good_sessions(): + envs = { + "SAIL_SPARK__SESSION_TIMEOUT_SECS": "2", + "SAIL_SPARK__EXECUTION_HEARTBEAT_INTERVAL_SECS": "1", + } + with spark_connect_server(envs=envs) as server, spark_session_factory(server.remote) as sessions: + yield sessions + + +@pytest.fixture(scope="module") +def identity(): + @F.udf(LongType()) + def _identity(value): + time.sleep(5) + return value + + return _identity + + +@pytest.fixture(scope="module") +def data(identity): + def _data(spark): + return spark.range(1).select(identity("id").alias("id")) + + return _data + + +def test_query_fails_when_session_timeout_precedes_heartbeat(bad_sessions, data): + spark = bad_sessions.create() + with pytest.raises(SparkConnectGrpcException): + data(spark).collect() + + +def test_command_fails_when_session_timeout_precedes_heartbeat(bad_sessions, data): + spark = bad_sessions.create() + with pytest.raises(SparkConnectGrpcException): + data(spark).write.format("noop").mode("overwrite").save() + + +def test_query_succeeds_when_heartbeat_precedes_session_timeout(good_sessions, data): + spark = good_sessions.create() + assert data(spark).collect() == [Row(id=0)] + # Send another request to make sure the session is still alive. + assert spark.range(1).count() == 1 + + +def test_command_succeeds_when_heartbeat_precedes_session_timeout(good_sessions, data): + spark = good_sessions.create() + data(spark).write.format("noop").mode("overwrite").save() + # Send another request to make sure the session is still alive. + assert spark.range(1).count() == 1 diff --git a/python/pysail/tests/spark/session/test_reattachable_execution.py b/python/pysail/tests/spark/session/test_reattachable_execution.py new file mode 100644 index 000000000..8d6a6ae8c --- /dev/null +++ b/python/pysail/tests/spark/session/test_reattachable_execution.py @@ -0,0 +1,31 @@ +import pyspark.sql.connect.functions as F # noqa: N812 +import pytest +from pyspark.errors.exceptions.connect import SparkConnectGrpcException + +from pysail.testing.spark.session import spark_session_factory +from pysail.testing.spark.utils.common import is_jvm_spark + +pytestmark = pytest.mark.skipif(is_jvm_spark(), reason="Sail only") + + +@pytest.fixture(scope="module") +def fail(): + @F.udf("long") + def _fail(_value): + msg = "expected failure" + raise RuntimeError(msg) + + return _fail + + +def test_execution_error_without_reattachment(remote, fail): + """Error handling for reattachable execution (the default) is exercised frequently in other tests. + This test ensures that the error can still be propagated for (non-default) non-reattachable execution. + """ + with spark_session_factory(remote) as sessions: + spark = sessions.create() + spark._client.disable_reattachable_execute() # noqa: SLF001 + with pytest.raises(SparkConnectGrpcException, match="expected failure"): + spark.range(1).select(fail("id")).write.format("noop").mode("overwrite").save() + with pytest.raises(SparkConnectGrpcException, match="expected failure"): + spark.range(1).select(fail("id")).collect() From 88cf927a14cf57eca80486e66979c8277de0c90b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 30 Jul 2026 17:11:31 +0800 Subject: [PATCH 096/131] deps(npm): bump eslint from 10.7.0 to 10.8.0 in the all group (#2317) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package.json | 2 +- pnpm-lock.yaml | 151 +++++++++++++++++++++++-------------------------- 2 files changed, 71 insertions(+), 82 deletions(-) diff --git a/package.json b/package.json index ef5adfab8..d585f60db 100644 --- a/package.json +++ b/package.json @@ -19,7 +19,7 @@ "@typescript-eslint/parser": "^8.65.0", "@viz-js/viz": "^3.28.0", "autoprefixer": "^10.5.4", - "eslint": "^10.7.0", + "eslint": "^10.8.0", "eslint-config-prettier": "^10.1.8", "eslint-plugin-vue": "^10.10.0", "fast-glob": "^3.3.3", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ce67ed668..a08f8a60d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -10,7 +10,7 @@ importers: devDependencies: '@eslint/js': specifier: ^10.0.1 - version: 10.0.1(eslint@10.7.0(jiti@2.7.0)) + version: 10.0.1(eslint@10.8.0(jiti@2.7.0)) '@ianvs/prettier-plugin-sort-imports': specifier: ^4.7.1 version: 4.7.1(@vue/compiler-sfc@3.5.40)(prettier@3.9.6) @@ -28,10 +28,10 @@ importers: version: 24.12.2 '@typescript-eslint/eslint-plugin': specifier: ^8.65.0 - version: 8.65.0(@typescript-eslint/parser@8.65.0(eslint@10.7.0(jiti@2.7.0))(typescript@6.0.3))(eslint@10.7.0(jiti@2.7.0))(typescript@6.0.3) + version: 8.65.0(@typescript-eslint/parser@8.65.0(eslint@10.8.0(jiti@2.7.0))(typescript@6.0.3))(eslint@10.8.0(jiti@2.7.0))(typescript@6.0.3) '@typescript-eslint/parser': specifier: ^8.65.0 - version: 8.65.0(eslint@10.7.0(jiti@2.7.0))(typescript@6.0.3) + version: 8.65.0(eslint@10.8.0(jiti@2.7.0))(typescript@6.0.3) '@viz-js/viz': specifier: ^3.28.0 version: 3.28.0 @@ -39,14 +39,14 @@ importers: specifier: ^10.5.4 version: 10.5.4(postcss@8.5.22) eslint: - specifier: ^10.7.0 - version: 10.7.0(jiti@2.7.0) + specifier: ^10.8.0 + version: 10.8.0(jiti@2.7.0) eslint-config-prettier: specifier: ^10.1.8 - version: 10.1.8(eslint@10.7.0(jiti@2.7.0)) + version: 10.1.8(eslint@10.8.0(jiti@2.7.0)) eslint-plugin-vue: specifier: ^10.10.0 - version: 10.10.0(@typescript-eslint/parser@8.65.0(eslint@10.7.0(jiti@2.7.0))(typescript@6.0.3))(eslint@10.7.0(jiti@2.7.0))(vue-eslint-parser@10.4.1(eslint@10.7.0(jiti@2.7.0))) + version: 10.10.0(@typescript-eslint/parser@8.65.0(eslint@10.8.0(jiti@2.7.0))(typescript@6.0.3))(eslint@10.8.0(jiti@2.7.0))(vue-eslint-parser@10.4.1(eslint@10.8.0(jiti@2.7.0))) fast-glob: specifier: ^3.3.3 version: 3.3.3 @@ -88,7 +88,7 @@ importers: version: 6.0.3 typescript-eslint: specifier: ^8.65.0 - version: 8.65.0(eslint@10.7.0(jiti@2.7.0))(typescript@6.0.3) + version: 8.65.0(eslint@10.8.0(jiti@2.7.0))(typescript@6.0.3) vega: specifier: ^6.3.1 version: 6.3.1 @@ -103,7 +103,7 @@ importers: version: 3.5.40(typescript@6.0.3) vue-eslint-parser: specifier: ^10.4.1 - version: 10.4.1(eslint@10.7.0(jiti@2.7.0)) + version: 10.4.1(eslint@10.8.0(jiti@2.7.0)) vue-tsc: specifier: ^3.3.8 version: 3.3.8(typescript@6.0.3) @@ -407,12 +407,6 @@ packages: peerDependencies: eslint: ^6.0.0 || ^7.0.0 || >=8.0.0 - '@eslint-community/eslint-utils@4.9.1': - resolution: {integrity: sha512-phrYmNiYppR7znFEdqgfWHXR6NCkZEK7hwWDHZUjit/2/U0r6XvkDl0SYnoM51Hq7FhCGdLDT6zxCCOY1hexsQ==} - engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} - peerDependencies: - eslint: ^6.0.0 || ^7.0.0 || >=8.0.0 - '@eslint-community/regexpp@4.12.2': resolution: {integrity: sha512-EriSTlt5OC9/7SXkRSCAhfSxxoSUgBm33OH+IkwbdpgoqsSsUg7y3uh+IICI/Qg4BBWr3U2i39RpmycbxMq4ew==} engines: {node: ^12.0.0 || ^14.0.0 || >=16.0.0} @@ -421,8 +415,8 @@ packages: resolution: {integrity: sha512-Y3kKLvC1dvTOT+oGlqNQ1XLqK6D1HU2YXPc52NmAlJZbMMWDzGYXMiPRJ8TYD39muD/OTjlZmNJ4ib7dvSrMBA==} engines: {node: ^20.19.0 || ^22.13.0 || >=24} - '@eslint/config-helpers@0.6.0': - resolution: {integrity: sha512-ii6Bw9jJ2zi2cWA2Z+9/QZ/+3DX6kwaV5Q986D/CdP3Lap3w/pgQZ373FV7byY/i7L4IRH/G43I5dz1ClsCbpA==} + '@eslint/config-helpers@0.7.0': + resolution: {integrity: sha512-DObd/KKUsU+FaFv4PLxSRenpXfQWmPXXP3pPZ6/K1PCrMu2vQpMDMuQe/BqYeoLcz8ro0bVDF1RxOJgfVEdhUw==} engines: {node: ^20.19.0 || ^22.13.0 || >=24} '@eslint/core@1.2.1': @@ -978,8 +972,8 @@ packages: peerDependencies: acorn: ^6.0.0 || ^7.0.0 || ^8.0.0 - acorn@8.17.0: - resolution: {integrity: sha512-xRQbDb9BnwDafYNn6Vwl839DYVjqXYb1XVGtWAZ1kcDc6iwAL4hg3B1dZlRiuENFeO2H53gFG3in621AdERVAg==} + acorn@8.18.0: + resolution: {integrity: sha512-lGq+9yr1/GuAWaVYIHRjvvySG5/4VfKIvC8EWxStPdcDh/Ka7FG3twP6v4d5BkravUilhIAsG4Qj83t02LWUPQ==} engines: {node: '>=0.4.0'} hasBin: true @@ -1021,9 +1015,9 @@ packages: boolbase@1.0.0: resolution: {integrity: sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==} - brace-expansion@5.0.7: - resolution: {integrity: sha512-7oFy703dxfY3/NLxC1fh2SUCQ0H9rmAY+5EpDVfXjUTTs+HEwR2nYaqLv+GWcTsumwxPfiz6CzCNkwXwBUwqCA==} - engines: {node: 18 || 20 || >=22} + brace-expansion@5.0.8: + resolution: {integrity: sha512-JZyDyq3D4AUifKTPOB7DELf6XsB3WdPuNxCtob1vFXPsSXhdAiHBWJ/tJ8HAc9aH84BK+5JFZLNkJKx3G9kzQg==} + engines: {node: 20 || >=22} braces@3.0.3: resolution: {integrity: sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==} @@ -1278,8 +1272,8 @@ packages: resolution: {integrity: sha512-tD40eHxA35h0PEIZNeIjkHoDR4YjjJp34biM0mDvplBe//mB+IHCqHDGV7pxF+7MklTvighcCPPZC7ynWyjdTA==} engines: {node: ^20.19.0 || ^22.13.0 || >=24} - eslint@10.7.0: - resolution: {integrity: sha512-GVTD7s1vdIl6UYvAfriOPeY1Df8LIZjfofLvHwde+erDHGGuHyuM6xoxRxmHiebhYuD2p1vN4wWh0XzPARSGDQ==} + eslint@10.8.0: + resolution: {integrity: sha512-nuKKvN+oIBO0koN7Tm7dlkmnkc21mtt0QJLwAKzjLq14y6lRTdVG36MZHJ8eQHwdJMwZbQNMlPOYedMq/oVJvQ==} engines: {node: ^20.19.0 || ^22.13.0 || >=24} hasBin: true peerDependencies: @@ -1361,8 +1355,8 @@ packages: resolution: {integrity: sha512-f7ccFPK3SXFHpx15UIGyRJ/FJQctuKZ0zVuN3frBo4HnK3cay9VEW0R6yPYFHC0AgqhukPzKjq22t5DmAyqGyw==} engines: {node: '>=16'} - flatted@3.4.2: - resolution: {integrity: sha512-PjDse7RzhcPkIJwy5t7KPWQSZ9cAbzQXcafsetQoD7sOJRQlGikNbx7yZp2OotDnJyrDcbyRq3Ttb18iYOqkxA==} + flatted@3.4.3: + resolution: {integrity: sha512-/zipXxyO6rGvuNGDiULY9MvEGSkb2gaG4GGH4ygMi0ZZzyMHdUZBmntJmx5x1G2VuPytCwGN4xsJP6cw+sK+vQ==} focus-trap@7.8.0: resolution: {integrity: sha512-/yNdlIkpWbM0ptxno3ONTuf+2g318kh2ez3KSeZN5dZ8YC6AAmgeWz+GasYYiBJPFaYcSAPeu4GfhUaChzIJXA==} @@ -1613,8 +1607,8 @@ packages: resolution: {integrity: sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==} engines: {node: '>=8.6'} - minimatch@10.2.5: - resolution: {integrity: sha512-MULkVLfKGYDFYejP07QOurDLLQpcjk7Fw+7jXS2R2czRQzR56yHRveU5NDJEOviH+hETZKSkIk5c+T23GjFUMg==} + minimatch@10.2.6: + resolution: {integrity: sha512-vpLQEs+VLCr1nU0BXS07maYoFwlDAH0gngQuuttxIwutDFEMHq2blX+8vpgxDdK3J1PwjCJiep77OitTZ4Ll1A==} engines: {node: 18 || 20 || >=22} minisearch@7.2.0: @@ -2441,14 +2435,9 @@ snapshots: '@esbuild/win32-x64@0.21.5': optional: true - '@eslint-community/eslint-utils@4.10.1(eslint@10.7.0(jiti@2.7.0))': - dependencies: - eslint: 10.7.0(jiti@2.7.0) - eslint-visitor-keys: 3.4.3 - - '@eslint-community/eslint-utils@4.9.1(eslint@10.7.0(jiti@2.7.0))': + '@eslint-community/eslint-utils@4.10.1(eslint@10.8.0(jiti@2.7.0))': dependencies: - eslint: 10.7.0(jiti@2.7.0) + eslint: 10.8.0(jiti@2.7.0) eslint-visitor-keys: 3.4.3 '@eslint-community/regexpp@4.12.2': {} @@ -2457,11 +2446,11 @@ snapshots: dependencies: '@eslint/object-schema': 3.0.5 debug: 4.4.3 - minimatch: 10.2.5 + minimatch: 10.2.6 transitivePeerDependencies: - supports-color - '@eslint/config-helpers@0.6.0': + '@eslint/config-helpers@0.7.0': dependencies: '@eslint/core': 1.2.1 @@ -2469,9 +2458,9 @@ snapshots: dependencies: '@types/json-schema': 7.0.15 - '@eslint/js@10.0.1(eslint@10.7.0(jiti@2.7.0))': + '@eslint/js@10.0.1(eslint@10.8.0(jiti@2.7.0))': optionalDependencies: - eslint: 10.7.0(jiti@2.7.0) + eslint: 10.8.0(jiti@2.7.0) '@eslint/object-schema@3.0.5': {} @@ -2766,15 +2755,15 @@ snapshots: '@types/web-bluetooth@0.0.21': {} - '@typescript-eslint/eslint-plugin@8.65.0(@typescript-eslint/parser@8.65.0(eslint@10.7.0(jiti@2.7.0))(typescript@6.0.3))(eslint@10.7.0(jiti@2.7.0))(typescript@6.0.3)': + '@typescript-eslint/eslint-plugin@8.65.0(@typescript-eslint/parser@8.65.0(eslint@10.8.0(jiti@2.7.0))(typescript@6.0.3))(eslint@10.8.0(jiti@2.7.0))(typescript@6.0.3)': dependencies: '@eslint-community/regexpp': 4.12.2 - '@typescript-eslint/parser': 8.65.0(eslint@10.7.0(jiti@2.7.0))(typescript@6.0.3) + '@typescript-eslint/parser': 8.65.0(eslint@10.8.0(jiti@2.7.0))(typescript@6.0.3) '@typescript-eslint/scope-manager': 8.65.0 - '@typescript-eslint/type-utils': 8.65.0(eslint@10.7.0(jiti@2.7.0))(typescript@6.0.3) - '@typescript-eslint/utils': 8.65.0(eslint@10.7.0(jiti@2.7.0))(typescript@6.0.3) + '@typescript-eslint/type-utils': 8.65.0(eslint@10.8.0(jiti@2.7.0))(typescript@6.0.3) + '@typescript-eslint/utils': 8.65.0(eslint@10.8.0(jiti@2.7.0))(typescript@6.0.3) '@typescript-eslint/visitor-keys': 8.65.0 - eslint: 10.7.0(jiti@2.7.0) + eslint: 10.8.0(jiti@2.7.0) ignore: 7.0.6 natural-compare: 1.4.0 ts-api-utils: 2.5.0(typescript@6.0.3) @@ -2782,14 +2771,14 @@ snapshots: transitivePeerDependencies: - supports-color - '@typescript-eslint/parser@8.65.0(eslint@10.7.0(jiti@2.7.0))(typescript@6.0.3)': + '@typescript-eslint/parser@8.65.0(eslint@10.8.0(jiti@2.7.0))(typescript@6.0.3)': dependencies: '@typescript-eslint/scope-manager': 8.65.0 '@typescript-eslint/types': 8.65.0 '@typescript-eslint/typescript-estree': 8.65.0(typescript@6.0.3) '@typescript-eslint/visitor-keys': 8.65.0 debug: 4.4.3 - eslint: 10.7.0(jiti@2.7.0) + eslint: 10.8.0(jiti@2.7.0) typescript: 6.0.3 transitivePeerDependencies: - supports-color @@ -2812,13 +2801,13 @@ snapshots: dependencies: typescript: 6.0.3 - '@typescript-eslint/type-utils@8.65.0(eslint@10.7.0(jiti@2.7.0))(typescript@6.0.3)': + '@typescript-eslint/type-utils@8.65.0(eslint@10.8.0(jiti@2.7.0))(typescript@6.0.3)': dependencies: '@typescript-eslint/types': 8.65.0 '@typescript-eslint/typescript-estree': 8.65.0(typescript@6.0.3) - '@typescript-eslint/utils': 8.65.0(eslint@10.7.0(jiti@2.7.0))(typescript@6.0.3) + '@typescript-eslint/utils': 8.65.0(eslint@10.8.0(jiti@2.7.0))(typescript@6.0.3) debug: 4.4.3 - eslint: 10.7.0(jiti@2.7.0) + eslint: 10.8.0(jiti@2.7.0) ts-api-utils: 2.5.0(typescript@6.0.3) typescript: 6.0.3 transitivePeerDependencies: @@ -2833,7 +2822,7 @@ snapshots: '@typescript-eslint/types': 8.65.0 '@typescript-eslint/visitor-keys': 8.65.0 debug: 4.4.3 - minimatch: 10.2.5 + minimatch: 10.2.6 semver: 7.8.5 tinyglobby: 0.2.17 ts-api-utils: 2.5.0(typescript@6.0.3) @@ -2841,13 +2830,13 @@ snapshots: transitivePeerDependencies: - supports-color - '@typescript-eslint/utils@8.65.0(eslint@10.7.0(jiti@2.7.0))(typescript@6.0.3)': + '@typescript-eslint/utils@8.65.0(eslint@10.8.0(jiti@2.7.0))(typescript@6.0.3)': dependencies: - '@eslint-community/eslint-utils': 4.10.1(eslint@10.7.0(jiti@2.7.0)) + '@eslint-community/eslint-utils': 4.10.1(eslint@10.8.0(jiti@2.7.0)) '@typescript-eslint/scope-manager': 8.65.0 '@typescript-eslint/types': 8.65.0 '@typescript-eslint/typescript-estree': 8.65.0(typescript@6.0.3) - eslint: 10.7.0(jiti@2.7.0) + eslint: 10.8.0(jiti@2.7.0) typescript: 6.0.3 transitivePeerDependencies: - supports-color @@ -2989,11 +2978,11 @@ snapshots: transitivePeerDependencies: - typescript - acorn-jsx@5.3.2(acorn@8.17.0): + acorn-jsx@5.3.2(acorn@8.18.0): dependencies: - acorn: 8.17.0 + acorn: 8.18.0 - acorn@8.17.0: {} + acorn@8.18.0: {} ajv@6.15.0: dependencies: @@ -3044,7 +3033,7 @@ snapshots: boolbase@1.0.0: {} - brace-expansion@5.0.7: + brace-expansion@5.0.8: dependencies: balanced-match: 4.0.4 @@ -3268,22 +3257,22 @@ snapshots: escape-string-regexp@4.0.0: {} - eslint-config-prettier@10.1.8(eslint@10.7.0(jiti@2.7.0)): + eslint-config-prettier@10.1.8(eslint@10.8.0(jiti@2.7.0)): dependencies: - eslint: 10.7.0(jiti@2.7.0) + eslint: 10.8.0(jiti@2.7.0) - eslint-plugin-vue@10.10.0(@typescript-eslint/parser@8.65.0(eslint@10.7.0(jiti@2.7.0))(typescript@6.0.3))(eslint@10.7.0(jiti@2.7.0))(vue-eslint-parser@10.4.1(eslint@10.7.0(jiti@2.7.0))): + eslint-plugin-vue@10.10.0(@typescript-eslint/parser@8.65.0(eslint@10.8.0(jiti@2.7.0))(typescript@6.0.3))(eslint@10.8.0(jiti@2.7.0))(vue-eslint-parser@10.4.1(eslint@10.8.0(jiti@2.7.0))): dependencies: - '@eslint-community/eslint-utils': 4.10.1(eslint@10.7.0(jiti@2.7.0)) - eslint: 10.7.0(jiti@2.7.0) + '@eslint-community/eslint-utils': 4.10.1(eslint@10.8.0(jiti@2.7.0)) + eslint: 10.8.0(jiti@2.7.0) natural-compare: 1.4.0 nth-check: 2.1.1 postcss-selector-parser: 7.1.4 semver: 7.8.5 - vue-eslint-parser: 10.4.1(eslint@10.7.0(jiti@2.7.0)) + vue-eslint-parser: 10.4.1(eslint@10.8.0(jiti@2.7.0)) xml-name-validator: 5.0.0 optionalDependencies: - '@typescript-eslint/parser': 8.65.0(eslint@10.7.0(jiti@2.7.0))(typescript@6.0.3) + '@typescript-eslint/parser': 8.65.0(eslint@10.8.0(jiti@2.7.0))(typescript@6.0.3) eslint-scope@9.1.2: dependencies: @@ -3296,12 +3285,12 @@ snapshots: eslint-visitor-keys@5.0.1: {} - eslint@10.7.0(jiti@2.7.0): + eslint@10.8.0(jiti@2.7.0): dependencies: - '@eslint-community/eslint-utils': 4.9.1(eslint@10.7.0(jiti@2.7.0)) + '@eslint-community/eslint-utils': 4.10.1(eslint@10.8.0(jiti@2.7.0)) '@eslint-community/regexpp': 4.12.2 '@eslint/config-array': 0.23.5 - '@eslint/config-helpers': 0.6.0 + '@eslint/config-helpers': 0.7.0 '@eslint/core': 1.2.1 '@eslint/plugin-kit': 0.7.2 '@humanfs/node': 0.16.8 @@ -3325,7 +3314,7 @@ snapshots: imurmurhash: 0.1.4 is-glob: 4.0.3 json-stable-stringify-without-jsonify: 1.0.1 - minimatch: 10.2.5 + minimatch: 10.2.6 natural-compare: 1.4.0 optionator: 0.9.4 optionalDependencies: @@ -3335,8 +3324,8 @@ snapshots: espree@11.2.0: dependencies: - acorn: 8.17.0 - acorn-jsx: 5.3.2(acorn@8.17.0) + acorn: 8.18.0 + acorn-jsx: 5.3.2(acorn@8.18.0) eslint-visitor-keys: 5.0.1 esprima@4.0.1: {} @@ -3396,10 +3385,10 @@ snapshots: flat-cache@4.0.1: dependencies: - flatted: 3.4.2 + flatted: 3.4.3 keyv: 4.5.4 - flatted@3.4.2: {} + flatted@3.4.3: {} focus-trap@7.8.0: dependencies: @@ -3625,9 +3614,9 @@ snapshots: braces: 3.0.3 picomatch: 2.3.2 - minimatch@10.2.5: + minimatch@10.2.6: dependencies: - brace-expansion: 5.0.7 + brace-expansion: 5.0.8 minisearch@7.2.0: {} @@ -3897,13 +3886,13 @@ snapshots: dependencies: prelude-ls: 1.2.1 - typescript-eslint@8.65.0(eslint@10.7.0(jiti@2.7.0))(typescript@6.0.3): + typescript-eslint@8.65.0(eslint@10.8.0(jiti@2.7.0))(typescript@6.0.3): dependencies: - '@typescript-eslint/eslint-plugin': 8.65.0(@typescript-eslint/parser@8.65.0(eslint@10.7.0(jiti@2.7.0))(typescript@6.0.3))(eslint@10.7.0(jiti@2.7.0))(typescript@6.0.3) - '@typescript-eslint/parser': 8.65.0(eslint@10.7.0(jiti@2.7.0))(typescript@6.0.3) + '@typescript-eslint/eslint-plugin': 8.65.0(@typescript-eslint/parser@8.65.0(eslint@10.8.0(jiti@2.7.0))(typescript@6.0.3))(eslint@10.8.0(jiti@2.7.0))(typescript@6.0.3) + '@typescript-eslint/parser': 8.65.0(eslint@10.8.0(jiti@2.7.0))(typescript@6.0.3) '@typescript-eslint/typescript-estree': 8.65.0(typescript@6.0.3) - '@typescript-eslint/utils': 8.65.0(eslint@10.7.0(jiti@2.7.0))(typescript@6.0.3) - eslint: 10.7.0(jiti@2.7.0) + '@typescript-eslint/utils': 8.65.0(eslint@10.8.0(jiti@2.7.0))(typescript@6.0.3) + eslint: 10.8.0(jiti@2.7.0) typescript: 6.0.3 transitivePeerDependencies: - supports-color @@ -4246,10 +4235,10 @@ snapshots: vscode-uri@3.1.0: {} - vue-eslint-parser@10.4.1(eslint@10.7.0(jiti@2.7.0)): + vue-eslint-parser@10.4.1(eslint@10.8.0(jiti@2.7.0)): dependencies: debug: 4.4.3 - eslint: 10.7.0(jiti@2.7.0) + eslint: 10.8.0(jiti@2.7.0) eslint-scope: 9.1.2 eslint-visitor-keys: 5.0.1 espree: 11.2.0 From 3456d2a9adf18253d777f0dbacdaf9957177b189 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 30 Jul 2026 17:13:04 +0800 Subject: [PATCH 097/131] deps(cargo): bump the all group with 2 updates (#2319) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9d93b3680..3d428a32f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -144,7 +144,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -155,7 +155,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -973,9 +973,9 @@ dependencies = [ [[package]] name = "aws-smithy-runtime-api" -version = "1.13.0" +version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22ed1ebe6e0a95ea84570225f5a8208dec4b8f77e61a9b0d6f51773fcb4612f0" +checksum = "3b98f2e1fd67ec06618f9c291e5e495a468e60519e44c9c1979cd0521f3affdb" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api-macros", @@ -2892,7 +2892,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -2970,9 +2970,9 @@ dependencies = [ [[package]] name = "either" -version = "1.16.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" +checksum = "9e5e8f6c15a24b9a3ee5efec809ccd006d3b30e8b3bb63c39af737c7f87daa1d" [[package]] name = "encode_unicode" @@ -3057,7 +3057,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -4605,7 +4605,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -5381,7 +5381,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -7264,7 +7264,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -8373,7 +8373,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b55fb86dfd3a2f5f76ea78310a88f96c4ea21a3031f8d212443d56123fd0521" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -8681,7 +8681,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -8761,7 +8761,7 @@ dependencies = [ "cfg-if", "libc", "psm", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -8951,7 +8951,7 @@ dependencies = [ "getrandom 0.4.3", "once_cell", "rustix 1.1.4", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -10014,7 +10014,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] From 2d31ebec5909b2889749a4c73b562dabb0ae6cb0 Mon Sep 17 00:00:00 2001 From: Xiaolong Liang Date: Thu, 30 Jul 2026 17:57:30 +0800 Subject: [PATCH 098/131] refactor: move distributed EXPLAIN output to CODEGEN mode (#2316) --- .../sail-common-datafusion/src/session/job.rs | 6 +- crates/sail-execution/src/job_runner.rs | 28 +- crates/sail-plan/src/explain.rs | 16 +- python/pysail/testing/spark/steps/plan.py | 2 +- .../__snapshots__/features/explain.yaml | 23 + .../delta/__snapshots__/features/delete.yaml | 85 + .../delta/__snapshots__/features/merge.yaml | 285 + .../__snapshots__/features/explain.yaml | 326 +- .../features/lakehouse_commit.yaml | 839 ++ .../features/scalar_subquery.yaml | 11743 ++++++++++++++-- .../test_shuffle_storage.plan.yaml | 560 + .../spark/execution/features/explain.feature | 6 +- .../features/lakehouse_commit.feature | 4 +- .../features/scalar_subquery.feature | 46 +- .../spark/execution/test_shuffle_storage.py | 2 +- 15 files changed, 13111 insertions(+), 860 deletions(-) diff --git a/crates/sail-common-datafusion/src/session/job.rs b/crates/sail-common-datafusion/src/session/job.rs index eff899800..c5d1b7f88 100644 --- a/crates/sail-common-datafusion/src/session/job.rs +++ b/crates/sail-common-datafusion/src/session/job.rs @@ -22,10 +22,8 @@ pub trait JobRunner: StateObservable + Send + Sync + 'static plan: Arc, ) -> Result; - /// Returns an execution-mode-specific plan for EXPLAIN output. - fn explain(&self, _plan: Arc) -> Result> { - Ok(None) - } + /// Returns the distributed execution plan for EXPLAIN output. + fn explain(&self, plan: Arc) -> Result; async fn stop(&self, history: oneshot::Sender); } diff --git a/crates/sail-execution/src/job_runner.rs b/crates/sail-execution/src/job_runner.rs index 7f6fa36ad..048daf5ef 100644 --- a/crates/sail-execution/src/job_runner.rs +++ b/crates/sail-execution/src/job_runner.rs @@ -17,6 +17,17 @@ use crate::driver::{DriverActor, DriverEvent, DriverHandle, DriverOptions}; use crate::job_graph::{JobGraph, JobGraphOptions}; use crate::shuffle::ShuffleBackendKind; +fn explain_job_graph(plan: Arc, use_blocking_shuffle: bool) -> Result { + JobGraph::try_new( + plan, + JobGraphOptions { + use_blocking_shuffle, + }, + ) + .map(|graph| graph.to_string()) + .map_err(|e| DataFusionError::External(Box::new(e))) +} + pub struct LocalJobRunner { next_job_id: AtomicU64, stopped: AtomicBool, @@ -46,6 +57,10 @@ impl StateObservable for LocalJobRunner { #[tonic::async_trait] impl JobRunner for LocalJobRunner { + fn explain(&self, plan: Arc) -> Result { + explain_job_graph(plan, false) + } + async fn execute( &self, ctx: &SessionContext, @@ -114,18 +129,11 @@ impl StateObservable for ClusterJobRunner { #[tonic::async_trait] impl JobRunner for ClusterJobRunner { - fn explain(&self, plan: Arc) -> Result> { - JobGraph::try_new( + fn explain(&self, plan: Arc) -> Result { + explain_job_graph( plan, - JobGraphOptions { - use_blocking_shuffle: matches!( - &self.shuffle_backend, - ShuffleBackendKind::Storage { .. } - ), - }, + matches!(&self.shuffle_backend, ShuffleBackendKind::Storage { .. }), ) - .map(|graph| Some(graph.to_string())) - .map_err(|e| DataFusionError::External(Box::new(e))) } /// Executes a plan on the cluster. This is where the cool stuff happens. diff --git a/crates/sail-plan/src/explain.rs b/crates/sail-plan/src/explain.rs index 670f213b9..a7a7630ac 100644 --- a/crates/sail-plan/src/explain.rs +++ b/crates/sail-plan/src/explain.rs @@ -378,7 +378,7 @@ fn distributed_plan_string( let plan = physical.as_ref()?; let service = ctx.extension::().ok()?; match service.runner().explain(Arc::clone(plan)) { - Ok(plan) => plan, + Ok(plan) => Some(plan), Err(err) => Some(format!("Distributed plan error: {err}")), } } @@ -427,9 +427,8 @@ async fn explain_from_collected( collected.logical_string(&collected.optimized_logical, PlanType::FinalLogicalPlan); let mut physical = PhysicalStrings::default(); - let distributed_plan = distributed_plan_string(ctx, &collected.physical_plan); - let mut sections = match options.kind { + let sections = match options.kind { ExplainKind::Simple => { let physical_for_mode = if options.analyze { physical.full_with_metrics(&collected) @@ -466,7 +465,7 @@ async fn explain_from_collected( ] } ExplainKind::Codegen => { - vec![ + let mut sections = vec![ render_section( "Codegen", "Whole-stage codegen is not supported; showing physical plan instead.", @@ -483,7 +482,11 @@ async fn explain_from_collected( physical.plain(&collected, options.verbose) }, ), - ] + ]; + if let Some(plan) = distributed_plan_string(ctx, &collected.physical_plan) { + sections.push(render_section("Distributed Plan", &plan)); + } + sections } ExplainKind::Cost => { vec![ @@ -505,9 +508,6 @@ async fn explain_from_collected( vec![render_section("Physical Plan", physical.full(&collected))] } }; - if let Some(plan) = distributed_plan { - sections.push(render_section("Distributed Plan", &plan)); - } let output = sections.join("\n\n"); Ok(ExplainString { diff --git a/python/pysail/testing/spark/steps/plan.py b/python/pysail/testing/spark/steps/plan.py index f37e5005f..5f85c29d3 100644 --- a/python/pysail/testing/spark/steps/plan.py +++ b/python/pysail/testing/spark/steps/plan.py @@ -36,7 +36,7 @@ def _normalize_metrics_block(match: re.Match[str]) -> str: # Normalize temp paths / file URIs that appear in plans. pytest_tmp_prefix = re.compile( - r"(^|[\s\[\(=,:{\"])" + r"(^|[\s\[\(=,:{\"]|[A-Za-z][A-Za-z0-9+.\-]+://)" r"(?!\[)" r"(?:(?:[A-Za-z]:)?/|private/|tmp/)" r"(?:[^ \t\r\n\),\]/]+/)*" diff --git a/python/pysail/tests/spark/analyzer/__snapshots__/features/explain.yaml b/python/pysail/tests/spark/analyzer/__snapshots__/features/explain.yaml index a4dec45ab..f611952f1 100644 --- a/python/pysail/tests/spark/analyzer/__snapshots__/features/explain.yaml +++ b/python/pysail/tests/spark/analyzer/__snapshots__/features/explain.yaml @@ -359,6 +359,29 @@ data: AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3)] ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] DataSourceExec: partitions=1, partition_sizes=[1] + + + == Distributed Plan == + === stage 0 === + inputs=[] + mode=Pipelined + partitions=1 + distribution=Hash(keys=[#2@0], channels=4) + placement=Worker + AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3)] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + + === stage 1 === + inputs=[StageInput(stage=0, mode=Shuffle)] + mode=Pipelined + partitions=4 + distribution=RoundRobin(channels=1) + placement=Worker + ProjectionExec: expr=[#4@0 as k, #5@1 as SUM(v)] + ProjectionExec: expr=[#2@0 as #4, sum(t.#3)@1 as #5] + AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[sum(t.#3)] + StageInputExec: input=0, partitioning=Hash([#2@0], 4) --- name: "test_explain_cost_shows_logical_plans_with_stats" data: diff --git a/python/pysail/tests/spark/delta/__snapshots__/features/delete.yaml b/python/pysail/tests/spark/delta/__snapshots__/features/delete.yaml index 2d4ac11e0..9d8710cf6 100644 --- a/python/pysail/tests/spark/delta/__snapshots__/features/delete.yaml +++ b/python/pysail/tests/spark/delta/__snapshots__/features/delete.yaml @@ -434,6 +434,91 @@ data: ProjectionExec: expr=[CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, path) END as __sail_file_path, CASE WHEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, size) END IS NOT NULL THEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, size) END ELSE 0 END as size_bytes, CASE WHEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, modificationTime) END IS NOT NULL THEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, modificationTime) END ELSE 0 END as modification_time, __sail_delta_log_version@2 as _commit_version, CASE WHEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, modificationTime) END IS NOT NULL THEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, modificationTime) END ELSE 0 END as _commit_timestamp, CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, stats) END as stats_json, CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, deletionVector) END as deletionVector, CASE WHEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, path) END IS NOT NULL THEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, path) END ELSE CASE WHEN remove@1 IS NOT NULL THEN get_field(remove@1, path) END END as __sail_delta_replay_path, remove@1 IS NOT NULL as __sail_delta_is_remove, __sail_delta_log_version@2 as __sail_delta_log_version] RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=2 DataSourceExec: file_groups={2 groups: [[/delete_explain_codegen/_delta_log/00000000000000000000.json], [/delete_explain_codegen/_delta_log/00000000000000000001.json]]}, projection=[add, remove, __sail_delta_log_version], file_type=json + + + == Distributed Plan == + === stage 0 === + inputs=[] + mode=Pipelined + partitions=2 + distribution=RoundRobin(channels=4) + placement=Worker + DataSourceExec: file_groups={2 groups: [[/delete_explain_codegen/_delta_log/00000000000000000000.json], [/delete_explain_codegen/_delta_log/00000000000000000001.json]]}, projection=[add, remove, __sail_delta_log_version], file_type=json + + === stage 1 === + inputs=[StageInput(stage=0, mode=Shuffle)] + mode=Pipelined + partitions=4 + distribution=Hash(keys=[__sail_delta_replay_path@7], channels=4) + placement=Worker + ProjectionExec: expr=[CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, path) END as __sail_file_path, CASE WHEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, size) END IS NOT NULL THEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, size) END ELSE 0 END as size_bytes, CASE WHEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, modificationTime) END IS NOT NULL THEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, modificationTime) END ELSE 0 END as modification_time, __sail_delta_log_version@2 as _commit_version, CASE WHEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, modificationTime) END IS NOT NULL THEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, modificationTime) END ELSE 0 END as _commit_timestamp, CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, stats) END as stats_json, CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, deletionVector) END as deletionVector, CASE WHEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, path) END IS NOT NULL THEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, path) END ELSE CASE WHEN remove@1 IS NOT NULL THEN get_field(remove@1, path) END END as __sail_delta_replay_path, remove@1 IS NOT NULL as __sail_delta_is_remove, __sail_delta_log_version@2 as __sail_delta_log_version] + StageInputExec: input=0, partitioning=RoundRobinBatch(4) + + === stage 2 === + inputs=[] + mode=Pipelined + partitions=2 + distribution=RoundRobin(channels=4) + placement=Worker + DataSourceExec: file_groups={2 groups: [[/delete_explain_codegen/_delta_log/00000000000000000000.json], [/delete_explain_codegen/_delta_log/00000000000000000001.json]]}, projection=[add, remove, __sail_delta_log_version], file_type=json + + === stage 3 === + inputs=[StageInput(stage=2, mode=Shuffle)] + mode=Pipelined + partitions=4 + distribution=Hash(keys=[__sail_delta_replay_path@7], channels=4) + placement=Worker + ProjectionExec: expr=[CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, path) END as __sail_file_path, CASE WHEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, size) END IS NOT NULL THEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, size) END ELSE 0 END as size_bytes, CASE WHEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, modificationTime) END IS NOT NULL THEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, modificationTime) END ELSE 0 END as modification_time, __sail_delta_log_version@2 as _commit_version, CASE WHEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, modificationTime) END IS NOT NULL THEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, modificationTime) END ELSE 0 END as _commit_timestamp, CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, stats) END as stats_json, CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, deletionVector) END as deletionVector, CASE WHEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, path) END IS NOT NULL THEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, path) END ELSE CASE WHEN remove@1 IS NOT NULL THEN get_field(remove@1, path) END END as __sail_delta_replay_path, remove@1 IS NOT NULL as __sail_delta_is_remove, __sail_delta_log_version@2 as __sail_delta_log_version] + StageInputExec: input=0, partitioning=RoundRobinBatch(4) + + === stage 4 === + inputs=[StageInput(stage=3, mode=Shuffle)] + mode=Pipelined + partitions=4 + distribution=RoundRobin(channels=1) + placement=Worker + DeltaDiscoveryExec(table_path=file:///delete_explain_codegen/) + FilterExec: (get_field(stats_parsed@7, minValues, department) IS NULL OR get_field(stats_parsed@7, maxValues, department) IS NULL OR get_field(stats_parsed@7, minValues, department) <= Engineering AND get_field(stats_parsed@7, maxValues, department) >= Engineering OR get_field(stats_parsed@7, minValues, department) IS NULL OR get_field(stats_parsed@7, maxValues, department) IS NULL OR get_field(stats_parsed@7, minValues, department) <= Marketing AND get_field(stats_parsed@7, maxValues, department) >= Marketing) AND (get_field(stats_parsed@7, nullCount, active) IS NULL OR get_field(stats_parsed@7, numRecords) IS NULL OR get_field(stats_parsed@7, nullCount, active) < get_field(stats_parsed@7, numRecords)) + DeltaMetadataStatsExec(output=stats_parsed, columns=[id, name, age, department, salary]) + DeltaLogReplayExec(table_path=file:///delete_explain_codegen/, version=1) + SortExec: expr=[__sail_delta_replay_path@7 ASC NULLS LAST, _commit_version@3 DESC NULLS LAST, __sail_delta_is_remove@8 ASC NULLS LAST], preserve_partitioning=[true] + StageInputExec: input=0, partitioning=Hash([__sail_delta_replay_path@7], 4) + + === stage 5 === + inputs=[StageInput(stage=1, mode=Shuffle), StageInput(stage=4, mode=Shuffle)] + mode=Pipelined + partitions=5 + distribution=RoundRobin(channels=1) + placement=Worker + UnionExec + DeltaWriterExec(table_path=file:///delete_explain_codegen/) + FilterExec: NOT salary@4 + 1 > 70000 AND (department@3 = Engineering OR department@3 = Marketing) AND active@5 IS NOT NULL + DeltaScanByAddsExec(table_path=file:///delete_explain_codegen/, version=1, projection=None, limit=None, pushdown=false) + DeltaDiscoveryExec(table_path=file:///delete_explain_codegen/) + FilterExec: (get_field(stats_parsed@7, minValues, department) IS NULL OR get_field(stats_parsed@7, maxValues, department) IS NULL OR get_field(stats_parsed@7, minValues, department) <= Engineering AND get_field(stats_parsed@7, maxValues, department) >= Engineering OR get_field(stats_parsed@7, minValues, department) IS NULL OR get_field(stats_parsed@7, maxValues, department) IS NULL OR get_field(stats_parsed@7, minValues, department) <= Marketing AND get_field(stats_parsed@7, maxValues, department) >= Marketing) AND (get_field(stats_parsed@7, nullCount, active) IS NULL OR get_field(stats_parsed@7, numRecords) IS NULL OR get_field(stats_parsed@7, nullCount, active) < get_field(stats_parsed@7, numRecords)) + DeltaMetadataStatsExec(output=stats_parsed, columns=[id, name, age, department, salary]) + DeltaLogReplayExec(table_path=file:///delete_explain_codegen/, version=1) + SortExec: expr=[__sail_delta_replay_path@7 ASC NULLS LAST, _commit_version@3 DESC NULLS LAST, __sail_delta_is_remove@8 ASC NULLS LAST], preserve_partitioning=[true] + StageInputExec: input=0, partitioning=Hash([__sail_delta_replay_path@7], 4) + DeltaRemoveActionsExec + StageInputExec: input=1, partitioning=UnknownPartitioning(1) + + === stage 6 === + inputs=[StageInput(stage=5, mode=Shuffle)] + mode=Pipelined + partitions=1 + distribution=RoundRobin(channels=1) + placement=Driver + DeltaCommitExec(table_path=file:///delete_explain_codegen/) + StageInputExec: input=0, partitioning=UnknownPartitioning(1) + + === stage 7 === + inputs=[StageInput(stage=6, mode=Forward)] + mode=Pipelined + partitions=1 + distribution=RoundRobin(channels=1) + placement=Worker + StageInputExec: input=0, partitioning=UnknownPartitioning(1) --- name: "test_explain_includes_json_delta_log_commits_when_present" data: diff --git a/python/pysail/tests/spark/delta/__snapshots__/features/merge.yaml b/python/pysail/tests/spark/delta/__snapshots__/features/merge.yaml index 66505cfc9..a6763089b 100644 --- a/python/pysail/tests/spark/delta/__snapshots__/features/merge.yaml +++ b/python/pysail/tests/spark/delta/__snapshots__/features/merge.yaml @@ -2415,6 +2415,291 @@ data: ProjectionExec: expr=[CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, path) END as __sail_file_path, CASE WHEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, size) END IS NOT NULL THEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, size) END ELSE 0 END as size_bytes, CASE WHEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, modificationTime) END IS NOT NULL THEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, modificationTime) END ELSE 0 END as modification_time, __sail_delta_log_version@2 as _commit_version, CASE WHEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, modificationTime) END IS NOT NULL THEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, modificationTime) END ELSE 0 END as _commit_timestamp, CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, stats) END as stats_json, CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, deletionVector) END as deletionVector, CASE WHEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, path) END IS NOT NULL THEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, path) END ELSE CASE WHEN remove@1 IS NOT NULL THEN get_field(remove@1, path) END END as __sail_delta_replay_path, remove@1 IS NOT NULL as __sail_delta_is_remove, __sail_delta_log_version@2 as __sail_delta_log_version] RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=2 DataSourceExec: file_groups={2 groups: [[/merge_explain_codegen/_delta_log/00000000000000000000.json], [/merge_explain_codegen/_delta_log/00000000000000000001.json]]}, projection=[add, remove, __sail_delta_log_version], file_type=json + + + == Distributed Plan == + === stage 0 === + inputs=[] + mode=Pipelined + partitions=1 + distribution=Hash(keys=[__sail_src_id@0], channels=1) + placement=Worker + ProjectionExec: expr=[column1@0 as __sail_src_id, column2@1 as __sail_src_category, column3@2 as __sail_src_amount, column4@3 as __sail_src_note, true as __sail_merge_source_row_present] + FilterExec: column3@2 + 1 > 10 AND (column1@0 = 1 OR column1@0 = 2 OR column1@0 = 4) + DataSourceExec: partitions=1, partition_sizes=[1] + + === stage 1 === + inputs=[] + mode=Pipelined + partitions=1 + distribution=Hash(keys=[id@0], channels=1) + placement=Worker + ProjectionExec: expr=[id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path, __sail_merge_target_row_id@5 as __sail_merge_target_row_id, true as __sail_merge_target_row_present] + MonotonicIdExec: col=__sail_merge_target_row_id + DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet + + === stage 2 === + inputs=[StageInput(stage=0, mode=Shuffle), StageInput(stage=1, mode=Shuffle)] + mode=Pipelined + partitions=1 + distribution=Hash(keys=[__sail_merge_target_row_id@5], channels=4) + placement=Worker + HashJoinExec: mode=Partitioned, join_type=Full, on=[(__sail_src_id@0, id@0)], filter=category@0 IS NOT NULL, projection=[id@5, category@6, amount@7, note@8, __sail_file_path@9, __sail_merge_target_row_id@10, __sail_merge_target_row_present@11, __sail_src_id@0, __sail_src_category@1, __sail_src_amount@2, __sail_src_note@3, __sail_merge_source_row_present@4] + StageInputExec: input=0, partitioning=Hash([__sail_src_id@0], 1) + StageInputExec: input=1, partitioning=Hash([id@0], 1) + + === stage 3 === + inputs=[] + mode=Pipelined + partitions=1 + distribution=RoundRobin(channels=4) + placement=Worker + FilterExec: NULL IS NULL AND column3@1 + 1 > 10 AND (column1@0 = 1 OR column1@0 = 2 OR column1@0 = 4), projection=[] + DataSourceExec: partitions=1, partition_sizes=[1] + + === stage 4 === + inputs=[StageInput(stage=3, mode=Shuffle)] + mode=Pipelined + partitions=4 + distribution=RoundRobin(channels=1) + placement=Worker + AggregateExec: mode=Partial, gby=[], aggr=[count(1) as __sail_merge_source_metric] + StageInputExec: input=0, partitioning=RoundRobinBatch(4) + + === stage 5 === + inputs=[] + mode=Pipelined + partitions=1 + distribution=Hash(keys=[__sail_src_id@0], channels=1) + placement=Worker + ProjectionExec: expr=[column1@0 as __sail_src_id, column2@1 as __sail_src_category, column3@2 as __sail_src_amount, column4@3 as __sail_src_note, true as __sail_merge_source_row_present] + FilterExec: column3@2 + 1 > 10 AND (column1@0 = 1 OR column1@0 = 2 OR column1@0 = 4) + DataSourceExec: partitions=1, partition_sizes=[1] + + === stage 6 === + inputs=[] + mode=Pipelined + partitions=1 + distribution=Hash(keys=[id@0], channels=1) + placement=Worker + ProjectionExec: expr=[id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path, __sail_merge_target_row_id@5 as __sail_merge_target_row_id, true as __sail_merge_target_row_present] + MonotonicIdExec: col=__sail_merge_target_row_id + DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet + + === stage 7 === + inputs=[StageInput(stage=5, mode=Shuffle), StageInput(stage=6, mode=Shuffle)] + mode=Pipelined + partitions=1 + distribution=Hash(keys=[__sail_merge_target_row_id@5], channels=4) + placement=Worker + HashJoinExec: mode=Partitioned, join_type=Full, on=[(__sail_src_id@0, id@0)], filter=category@0 IS NOT NULL, projection=[id@5, category@6, amount@7, note@8, __sail_file_path@9, __sail_merge_target_row_id@10, __sail_merge_target_row_present@11, __sail_src_id@0, __sail_src_category@1, __sail_src_amount@2, __sail_src_note@3, __sail_merge_source_row_present@4] + StageInputExec: input=0, partitioning=Hash([__sail_src_id@0], 1) + StageInputExec: input=1, partitioning=Hash([id@0], 1) + + === stage 8 === + inputs=[StageInput(stage=7, mode=Shuffle)] + mode=Pipelined + partitions=4 + distribution=Hash(keys=[__sail_file_path@0], channels=4) + placement=Worker + AggregateExec: mode=Partial, gby=[__sail_file_path@0 as __sail_file_path], aggr=[] + FilterExec: __sail_merge_target_row_present@3 IS NOT NULL AND (__sail_merge_source_row_present@5 IS NOT NULL AND id@0 = __sail_src_id@4 AND category@1 IS NOT NULL AND (category@1 = vip OR category@1 = stale) OR __sail_merge_source_row_present@5 IS NULL AND category@1 = stale), projection=[__sail_file_path@2] + ProjectionExec: expr=[id@0 as id, category@1 as category, __sail_file_path@4 as __sail_file_path, __sail_merge_target_row_present@6 as __sail_merge_target_row_present, __sail_src_id@7 as __sail_src_id, __sail_merge_source_row_present@11 as __sail_merge_source_row_present] + MergeCardinalityCheckExec: row_id=__sail_merge_target_row_id, target_present=__sail_merge_target_row_present, source_present=__sail_merge_source_row_present + StageInputExec: input=0, partitioning=Hash([__sail_merge_target_row_id@5], 4) + + === stage 9 === + inputs=[StageInput(stage=8, mode=Shuffle)] + mode=Pipelined + partitions=4 + distribution=RoundRobin(channels=1) + placement=Worker + AggregateExec: mode=FinalPartitioned, gby=[__sail_file_path@0 as __sail_file_path], aggr=[] + StageInputExec: input=0, partitioning=Hash([__sail_file_path@0], 4) + + === stage 10 === + inputs=[] + mode=Pipelined + partitions=1 + distribution=Hash(keys=[__sail_src_id@0], channels=1) + placement=Worker + ProjectionExec: expr=[column1@0 as __sail_src_id, column2@1 as __sail_src_category, column3@2 as __sail_src_amount, column4@3 as __sail_src_note, true as __sail_merge_source_row_present] + FilterExec: column3@2 + 1 > 10 AND (column1@0 = 1 OR column1@0 = 2 OR column1@0 = 4) + DataSourceExec: partitions=1, partition_sizes=[1] + + === stage 11 === + inputs=[] + mode=Pipelined + partitions=1 + distribution=Hash(keys=[id@0], channels=1) + placement=Worker + ProjectionExec: expr=[id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path, __sail_merge_target_row_id@5 as __sail_merge_target_row_id, true as __sail_merge_target_row_present] + MonotonicIdExec: col=__sail_merge_target_row_id + DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet + + === stage 12 === + inputs=[StageInput(stage=10, mode=Shuffle), StageInput(stage=11, mode=Shuffle)] + mode=Pipelined + partitions=1 + distribution=Hash(keys=[__sail_merge_target_row_id@5], channels=4) + placement=Worker + HashJoinExec: mode=Partitioned, join_type=Full, on=[(__sail_src_id@0, id@0)], filter=category@0 IS NOT NULL, projection=[id@5, category@6, amount@7, note@8, __sail_file_path@9, __sail_merge_target_row_id@10, __sail_merge_target_row_present@11, __sail_src_id@0, __sail_src_category@1, __sail_src_amount@2, __sail_src_note@3, __sail_merge_source_row_present@4] + StageInputExec: input=0, partitioning=Hash([__sail_src_id@0], 1) + StageInputExec: input=1, partitioning=Hash([id@0], 1) + + === stage 13 === + inputs=[] + mode=Pipelined + partitions=1 + distribution=RoundRobin(channels=4) + placement=Worker + FilterExec: column3@1 + 1 > 10 AND (column1@0 = 1 OR column1@0 = 2 OR column1@0 = 4), projection=[] + DataSourceExec: partitions=1, partition_sizes=[1] + + === stage 14 === + inputs=[StageInput(stage=13, mode=Shuffle)] + mode=Pipelined + partitions=4 + distribution=RoundRobin(channels=1) + placement=Worker + AggregateExec: mode=Partial, gby=[], aggr=[count(1) as __sail_merge_source_metric] + StageInputExec: input=0, partitioning=RoundRobinBatch(4) + + === stage 15 === + inputs=[] + mode=Pipelined + partitions=1 + distribution=Hash(keys=[__sail_src_id@0], channels=1) + placement=Worker + ProjectionExec: expr=[column1@0 as __sail_src_id, column2@1 as __sail_src_category, column3@2 as __sail_src_amount, column4@3 as __sail_src_note, true as __sail_merge_source_row_present] + FilterExec: column3@2 + 1 > 10 AND (column1@0 = 1 OR column1@0 = 2 OR column1@0 = 4) + DataSourceExec: partitions=1, partition_sizes=[1] + + === stage 16 === + inputs=[] + mode=Pipelined + partitions=1 + distribution=Hash(keys=[id@0], channels=1) + placement=Worker + ProjectionExec: expr=[id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path, __sail_merge_target_row_id@5 as __sail_merge_target_row_id, true as __sail_merge_target_row_present] + MonotonicIdExec: col=__sail_merge_target_row_id + DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet + + === stage 17 === + inputs=[StageInput(stage=15, mode=Shuffle), StageInput(stage=16, mode=Shuffle)] + mode=Pipelined + partitions=1 + distribution=Hash(keys=[__sail_merge_target_row_id@5], channels=4) + placement=Worker + HashJoinExec: mode=Partitioned, join_type=Full, on=[(__sail_src_id@0, id@0)], filter=category@0 IS NOT NULL, projection=[id@5, category@6, amount@7, note@8, __sail_file_path@9, __sail_merge_target_row_id@10, __sail_merge_target_row_present@11, __sail_src_id@0, __sail_src_category@1, __sail_src_amount@2, __sail_src_note@3, __sail_merge_source_row_present@4] + StageInputExec: input=0, partitioning=Hash([__sail_src_id@0], 1) + StageInputExec: input=1, partitioning=Hash([id@0], 1) + + === stage 18 === + inputs=[StageInput(stage=17, mode=Shuffle)] + mode=Pipelined + partitions=4 + distribution=Hash(keys=[__sail_file_path@0], channels=4) + placement=Worker + AggregateExec: mode=Partial, gby=[__sail_file_path@0 as __sail_file_path], aggr=[] + FilterExec: __sail_merge_target_row_present@3 IS NOT NULL AND (__sail_merge_source_row_present@5 IS NOT NULL AND id@0 = __sail_src_id@4 AND category@1 IS NOT NULL AND (category@1 = vip OR category@1 = stale) OR __sail_merge_source_row_present@5 IS NULL AND category@1 = stale), projection=[__sail_file_path@2] + ProjectionExec: expr=[id@0 as id, category@1 as category, __sail_file_path@4 as __sail_file_path, __sail_merge_target_row_present@6 as __sail_merge_target_row_present, __sail_src_id@7 as __sail_src_id, __sail_merge_source_row_present@11 as __sail_merge_source_row_present] + MergeCardinalityCheckExec: row_id=__sail_merge_target_row_id, target_present=__sail_merge_target_row_present, source_present=__sail_merge_source_row_present + StageInputExec: input=0, partitioning=Hash([__sail_merge_target_row_id@5], 4) + + === stage 19 === + inputs=[StageInput(stage=18, mode=Shuffle)] + mode=Pipelined + partitions=4 + distribution=RoundRobin(channels=1) + placement=Worker + AggregateExec: mode=FinalPartitioned, gby=[__sail_file_path@0 as __sail_file_path], aggr=[] + StageInputExec: input=0, partitioning=Hash([__sail_file_path@0], 4) + + === stage 20 === + inputs=[] + mode=Pipelined + partitions=2 + distribution=RoundRobin(channels=4) + placement=Worker + DataSourceExec: file_groups={2 groups: [[/merge_explain_codegen/_delta_log/00000000000000000000.json], [/merge_explain_codegen/_delta_log/00000000000000000001.json]]}, projection=[add, remove, __sail_delta_log_version], file_type=json + + === stage 21 === + inputs=[StageInput(stage=20, mode=Shuffle)] + mode=Pipelined + partitions=4 + distribution=Hash(keys=[__sail_delta_replay_path@7], channels=4) + placement=Worker + ProjectionExec: expr=[CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, path) END as __sail_file_path, CASE WHEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, size) END IS NOT NULL THEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, size) END ELSE 0 END as size_bytes, CASE WHEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, modificationTime) END IS NOT NULL THEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, modificationTime) END ELSE 0 END as modification_time, __sail_delta_log_version@2 as _commit_version, CASE WHEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, modificationTime) END IS NOT NULL THEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, modificationTime) END ELSE 0 END as _commit_timestamp, CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, stats) END as stats_json, CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, deletionVector) END as deletionVector, CASE WHEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, path) END IS NOT NULL THEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, path) END ELSE CASE WHEN remove@1 IS NOT NULL THEN get_field(remove@1, path) END END as __sail_delta_replay_path, remove@1 IS NOT NULL as __sail_delta_is_remove, __sail_delta_log_version@2 as __sail_delta_log_version] + StageInputExec: input=0, partitioning=RoundRobinBatch(4) + + === stage 22 === + inputs=[StageInput(stage=19, mode=Broadcast), StageInput(stage=21, mode=Shuffle)] + mode=Pipelined + partitions=4 + distribution=RoundRobin(channels=1) + placement=Worker + DeltaDiscoveryExec(table_path=file:///merge_explain_codegen/) + HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(__sail_file_path@0, __sail_file_path@0)], projection=[__sail_file_path@1, size_bytes@2, modification_time@3, _commit_version@4, _commit_timestamp@5, stats_json@6, deletionVector@7] + StageInputExec: input=0, partitioning=UnknownPartitioning(1) + DeltaLogReplayExec(table_path=file:///merge_explain_codegen/, version=1) + SortExec: expr=[__sail_delta_replay_path@7 ASC NULLS LAST, _commit_version@3 DESC NULLS LAST, __sail_delta_is_remove@8 ASC NULLS LAST], preserve_partitioning=[true] + StageInputExec: input=1, partitioning=Hash([__sail_delta_replay_path@7], 4) + + === stage 23 === + inputs=[StageInput(stage=2, mode=Shuffle), StageInput(stage=4, mode=Shuffle), StageInput(stage=9, mode=Broadcast), StageInput(stage=12, mode=Shuffle), StageInput(stage=14, mode=Shuffle), StageInput(stage=22, mode=Shuffle)] + mode=Pipelined + partitions=11 + distribution=RoundRobin(channels=1) + placement=Worker + UnionExec + DeltaWriterExec(table_path=file:///merge_explain_codegen/) + UnionExec + UnionExec + ProjectionExec: expr=[CASE WHEN __common_expr_1@0 THEN __sail_src_id@10 ELSE id@5 END as id, CASE WHEN __common_expr_1@0 THEN __sail_src_category@11 ELSE category@6 END as category, CASE WHEN __common_expr_2@1 THEN __sail_src_amount@12 + 1 WHEN __common_expr_1@0 THEN __sail_src_amount@12 ELSE amount@7 END as amount, CASE WHEN __common_expr_2@1 THEN spark_concat(__sail_src_note@13, _, id@5) WHEN __common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale THEN spark_concat(note@8, _orphan) WHEN __common_expr_1@0 THEN spark_concat(insert_, __sail_src_note@13) ELSE note@8 END as note, CASE WHEN __common_expr_4@3 THEN 3 WHEN __common_expr_2@1 THEN 6 WHEN __common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale THEN 8 WHEN __common_expr_5@4 AND category@6 = stale THEN 5 ELSE 0 END as __sail_operation_type, NULL as __sail_merge_source_metric] + ProjectionExec: expr=[__common_expr_6@0 AND __sail_merge_source_row_present@12 IS NOT NULL as __common_expr_1, __common_expr_7@1 AND category@4 = vip as __common_expr_2, __common_expr_8@2 as __common_expr_3, __common_expr_6@0 as __common_expr_4, __common_expr_7@1 as __common_expr_5, id@3 as id, category@4 as category, amount@5 as amount, note@6 as note, __sail_file_path@7 as __sail_file_path, __sail_src_id@8 as __sail_src_id, __sail_src_category@9 as __sail_src_category, __sail_src_amount@10 as __sail_src_amount, __sail_src_note@11 as __sail_src_note, __sail_merge_source_row_present@12 as __sail_merge_source_row_present] + ProjectionExec: expr=[__sail_merge_target_row_present@6 IS NULL as __common_expr_6, __common_expr_9@0 AND __sail_merge_source_row_present@11 IS NOT NULL as __common_expr_7, __common_expr_9@0 as __common_expr_8, id@1 as id, category@2 as category, amount@3 as amount, note@4 as note, __sail_file_path@5 as __sail_file_path, __sail_src_id@7 as __sail_src_id, __sail_src_category@8 as __sail_src_category, __sail_src_amount@9 as __sail_src_amount, __sail_src_note@10 as __sail_src_note, __sail_merge_source_row_present@11 as __sail_merge_source_row_present] + ProjectionExec: expr=[__sail_merge_target_row_present@5 IS NOT NULL as __common_expr_9, id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path, __sail_merge_target_row_present@5 as __sail_merge_target_row_present, __sail_src_id@6 as __sail_src_id, __sail_src_category@7 as __sail_src_category, __sail_src_amount@8 as __sail_src_amount, __sail_src_note@9 as __sail_src_note, __sail_merge_source_row_present@10 as __sail_merge_source_row_present] + FilterExec: __sail_file_path@4 IS NULL AND (__sail_merge_target_row_present@5 IS NOT NULL OR __sail_merge_target_row_present@5 IS NULL AND __sail_merge_source_row_present@10 IS NOT NULL) + ProjectionExec: expr=[id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path, __sail_merge_target_row_present@6 as __sail_merge_target_row_present, __sail_src_id@7 as __sail_src_id, __sail_src_category@8 as __sail_src_category, __sail_src_amount@9 as __sail_src_amount, __sail_src_note@10 as __sail_src_note, __sail_merge_source_row_present@11 as __sail_merge_source_row_present] + MergeCardinalityCheckExec: row_id=__sail_merge_target_row_id, target_present=__sail_merge_target_row_present, source_present=__sail_merge_source_row_present + StageInputExec: input=0, partitioning=Hash([__sail_merge_target_row_id@5], 4) + ProjectionExec: expr=[NULL as id, NULL as category, NULL as amount, NULL as note, 9 as __sail_operation_type, __sail_merge_source_metric@0 as __sail_merge_source_metric] + AggregateExec: mode=Final, gby=[], aggr=[count(1) as __sail_merge_source_metric] + StageInputExec: input=1, partitioning=UnknownPartitioning(1) + HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(__sail_file_path@0, __sail_file_path@4)], projection=[id@1, category@2, amount@3, note@4, __sail_operation_type@6, __sail_merge_source_metric@7] + StageInputExec: input=2, partitioning=UnknownPartitioning(1) + UnionExec + ProjectionExec: expr=[CASE WHEN __common_expr_1@0 THEN __sail_src_id@10 ELSE id@5 END as id, CASE WHEN __common_expr_1@0 THEN __sail_src_category@11 ELSE category@6 END as category, CASE WHEN __common_expr_2@1 THEN __sail_src_amount@12 + 1 WHEN __common_expr_1@0 THEN __sail_src_amount@12 ELSE amount@7 END as amount, CASE WHEN __common_expr_2@1 THEN spark_concat(__sail_src_note@13, _, id@5) WHEN __common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale THEN spark_concat(note@8, _orphan) WHEN __common_expr_1@0 THEN spark_concat(insert_, __sail_src_note@13) ELSE note@8 END as note, __sail_file_path@9 as __sail_file_path, CASE WHEN __common_expr_4@3 THEN 3 WHEN __common_expr_2@1 THEN 6 WHEN __common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale THEN 8 WHEN __common_expr_5@4 AND category@6 = stale THEN 5 ELSE 0 END as __sail_operation_type, NULL as __sail_merge_source_metric] + ProjectionExec: expr=[__common_expr_6@0 AND __sail_merge_source_row_present@12 IS NOT NULL as __common_expr_1, __common_expr_7@1 AND category@4 = vip as __common_expr_2, __common_expr_8@2 as __common_expr_3, __common_expr_6@0 as __common_expr_4, __common_expr_7@1 as __common_expr_5, id@3 as id, category@4 as category, amount@5 as amount, note@6 as note, __sail_file_path@7 as __sail_file_path, __sail_src_id@8 as __sail_src_id, __sail_src_category@9 as __sail_src_category, __sail_src_amount@10 as __sail_src_amount, __sail_src_note@11 as __sail_src_note, __sail_merge_source_row_present@12 as __sail_merge_source_row_present] + ProjectionExec: expr=[__sail_merge_target_row_present@6 IS NULL as __common_expr_6, __common_expr_9@0 AND __sail_merge_source_row_present@11 IS NOT NULL as __common_expr_7, __common_expr_9@0 as __common_expr_8, id@1 as id, category@2 as category, amount@3 as amount, note@4 as note, __sail_file_path@5 as __sail_file_path, __sail_src_id@7 as __sail_src_id, __sail_src_category@8 as __sail_src_category, __sail_src_amount@9 as __sail_src_amount, __sail_src_note@10 as __sail_src_note, __sail_merge_source_row_present@11 as __sail_merge_source_row_present] + ProjectionExec: expr=[__sail_merge_target_row_present@5 IS NOT NULL as __common_expr_9, id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path, __sail_merge_target_row_present@5 as __sail_merge_target_row_present, __sail_src_id@6 as __sail_src_id, __sail_src_category@7 as __sail_src_category, __sail_src_amount@8 as __sail_src_amount, __sail_src_note@9 as __sail_src_note, __sail_merge_source_row_present@10 as __sail_merge_source_row_present] + FilterExec: __sail_merge_target_row_present@5 IS NOT NULL OR __sail_merge_target_row_present@5 IS NULL AND __sail_merge_source_row_present@10 IS NOT NULL + ProjectionExec: expr=[id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path, __sail_merge_target_row_present@6 as __sail_merge_target_row_present, __sail_src_id@7 as __sail_src_id, __sail_src_category@8 as __sail_src_category, __sail_src_amount@9 as __sail_src_amount, __sail_src_note@10 as __sail_src_note, __sail_merge_source_row_present@11 as __sail_merge_source_row_present] + MergeCardinalityCheckExec: row_id=__sail_merge_target_row_id, target_present=__sail_merge_target_row_present, source_present=__sail_merge_source_row_present + StageInputExec: input=3, partitioning=Hash([__sail_merge_target_row_id@5], 4) + ProjectionExec: expr=[NULL as id, NULL as category, NULL as amount, NULL as note, NULL as __sail_file_path, 9 as __sail_operation_type, __sail_merge_source_metric@0 as __sail_merge_source_metric] + AggregateExec: mode=Final, gby=[], aggr=[count(1) as __sail_merge_source_metric] + StageInputExec: input=4, partitioning=UnknownPartitioning(1) + DeltaRemoveActionsExec + StageInputExec: input=5, partitioning=UnknownPartitioning(1) + + === stage 24 === + inputs=[StageInput(stage=23, mode=Shuffle)] + mode=Pipelined + partitions=1 + distribution=RoundRobin(channels=1) + placement=Driver + DeltaCommitExec(table_path=file:///merge_explain_codegen/) + StageInputExec: input=0, partitioning=UnknownPartitioning(1) + + === stage 25 === + inputs=[StageInput(stage=24, mode=Forward)] + mode=Pipelined + partitions=1 + distribution=RoundRobin(channels=1) + placement=Worker + StageInputExec: input=0, partitioning=UnknownPartitioning(1) --- name: "test_explain_extended_does_not_include_mergecardinalitycheck_when_source_is_grouped_by_join_keys" data: diff --git a/python/pysail/tests/spark/execution/__snapshots__/features/explain.yaml b/python/pysail/tests/spark/execution/__snapshots__/features/explain.yaml index b27297335..d1400dbe4 100644 --- a/python/pysail/tests/spark/execution/__snapshots__/features/explain.yaml +++ b/python/pysail/tests/spark/execution/__snapshots__/features/explain.yaml @@ -1,8 +1,332 @@ # serializer version: 1 --- -name: "test_explain_includes_the_distributed_execution_plan_in_localcluster_mode" +name: "test_explain_codegen_includes_the_distributed_execution_plan" data: |- + == Codegen == + Whole-stage codegen is not supported; showing physical plan instead. + + == Plan Steps == + initial_logical_plan: + Projection: t.#2 AS #4, sum(t.#3) AS total AS #5 + Aggregate: groupBy=[[t.#2]], aggr=[[sum(t.#3)]] + SubqueryAlias: t + Projection: #0 AS #2, #1 AS #3 + Projection: column1 AS #0, column2 AS #1 + Values: (Int32(1), Int32(2)), (Int32(1), Int32(3)) + + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE + + logical_plan after resolve_grouping_function: + SAME TEXT AS ABOVE + + logical_plan after type_coercion: + Projection: t.#2 AS #4, sum(t.#3) AS total AS #5 + Aggregate: groupBy=[[t.#2]], aggr=[[sum(CAST(t.#3 AS Int64))]] + SubqueryAlias: t + Projection: #0 AS #2, #1 AS #3 + Projection: column1 AS #0, column2 AS #1 + Values: (Int32(1), Int32(2)), (Int32(1), Int32(3)) + + analyzed_logical_plan: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_projection: + SAME TEXT AS ABOVE + + logical_plan after rewrite_set_comparison: + SAME TEXT AS ABOVE + + logical_plan after optimize_unions: + SAME TEXT AS ABOVE + + logical_plan after unions_to_filter: + SAME TEXT AS ABOVE + + logical_plan after simplify_expressions: + SAME TEXT AS ABOVE + + logical_plan after replace_distinct_aggregate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_predicate_subquery: + SAME TEXT AS ABOVE + + logical_plan after scalar_subquery_to_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_join: + SAME TEXT AS ABOVE + + logical_plan after extract_equijoin_predicate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_duplicated_expr: + SAME TEXT AS ABOVE + + logical_plan after eliminate_filter: + SAME TEXT AS ABOVE + + logical_plan after eliminate_cross_join: + SAME TEXT AS ABOVE + + logical_plan after eliminate_limit: + SAME TEXT AS ABOVE + + logical_plan after propagate_empty_relation: + SAME TEXT AS ABOVE + + logical_plan after filter_null_join_keys: + SAME TEXT AS ABOVE + + logical_plan after eliminate_outer_join: + SAME TEXT AS ABOVE + + logical_plan after push_down_limit: + SAME TEXT AS ABOVE + + logical_plan after push_down_filter: + SAME TEXT AS ABOVE + + logical_plan after single_distinct_aggregation_to_group_by: + SAME TEXT AS ABOVE + + logical_plan after eliminate_group_by_constant: + SAME TEXT AS ABOVE + + logical_plan after common_sub_expression_eliminate: + SAME TEXT AS ABOVE + + logical_plan after extract_leaf_expressions: + SAME TEXT AS ABOVE + + logical_plan after optimize_projections: + Projection: t.#2 AS #4, sum(t.#3) AS total AS #5 + Aggregate: groupBy=[[t.#2]], aggr=[[sum(CAST(t.#3 AS Int64))]] + SubqueryAlias: t + Projection: column1 AS #2, column2 AS #3 + Values: (Int32(1), Int32(2)), (Int32(1), Int32(3)) + + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_projection: + SAME TEXT AS ABOVE + + logical_plan after rewrite_set_comparison: + SAME TEXT AS ABOVE + + logical_plan after optimize_unions: + SAME TEXT AS ABOVE + + logical_plan after unions_to_filter: + SAME TEXT AS ABOVE + + logical_plan after simplify_expressions: + SAME TEXT AS ABOVE + + logical_plan after replace_distinct_aggregate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_predicate_subquery: + SAME TEXT AS ABOVE + + logical_plan after scalar_subquery_to_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_join: + SAME TEXT AS ABOVE + + logical_plan after extract_equijoin_predicate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_duplicated_expr: + SAME TEXT AS ABOVE + + logical_plan after eliminate_filter: + SAME TEXT AS ABOVE + + logical_plan after eliminate_cross_join: + SAME TEXT AS ABOVE + + logical_plan after eliminate_limit: + SAME TEXT AS ABOVE + + logical_plan after propagate_empty_relation: + SAME TEXT AS ABOVE + + logical_plan after filter_null_join_keys: + SAME TEXT AS ABOVE + + logical_plan after eliminate_outer_join: + SAME TEXT AS ABOVE + + logical_plan after push_down_limit: + SAME TEXT AS ABOVE + + logical_plan after push_down_filter: + SAME TEXT AS ABOVE + + logical_plan after single_distinct_aggregation_to_group_by: + SAME TEXT AS ABOVE + + logical_plan after eliminate_group_by_constant: + SAME TEXT AS ABOVE + + logical_plan after common_sub_expression_eliminate: + SAME TEXT AS ABOVE + + logical_plan after extract_leaf_expressions: + SAME TEXT AS ABOVE + + logical_plan after optimize_projections: + SAME TEXT AS ABOVE + + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE + + logical_plan: + Projection: t.#2 AS #4, sum(t.#3) AS total AS #5 + Aggregate: groupBy=[[t.#2]], aggr=[[sum(CAST(t.#3 AS Int64))]] + SubqueryAlias: t + Projection: column1 AS #2, column2 AS #3 + Values: (Int32(1), Int32(2)), (Int32(1), Int32(3)) + + initial_physical_plan: + ProjectionExec: expr=[#2@0 as #4, sum(t.#3)@1 as #5] + AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[sum(t.#3)] + AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3)] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + + + initial_physical_plan_with_stats: + ProjectionExec: expr=[#2@0 as #4, sum(t.#3)@1 as #5], statistics=[Rows=Inexact(2), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]] + AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[sum(t.#3)], statistics=[Rows=Inexact(2), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]] + AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3)], statistics=[Rows=Inexact(2), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3], statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]] + DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]] + + + initial_physical_plan_with_schema: + ProjectionExec: expr=[#2@0 as #4, sum(t.#3)@1 as #5], schema=[#4:Int32;N, #5:Int64;N] + AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[sum(t.#3)], schema=[#2:Int32;N, sum(t.#3):Int64;N] + AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3)], schema=[#2:Int32;N, sum(t.#3)[sum]:Int64;N] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3], schema=[#2:Int32;N, #3:Int32;N] + DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N, column2:Int32;N] + + + physical_plan after OutputRequirements: + OutputRequirementExec: order_by=[], dist_by=Unspecified + ProjectionExec: expr=[#2@0 as #4, sum(t.#3)@1 as #5] + AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[sum(t.#3)] + AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3)] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after aggregate_statistics: + SAME TEXT AS ABOVE + + physical_plan after join_selection: + SAME TEXT AS ABOVE + + physical_plan after LimitedDistinctAggregation: + SAME TEXT AS ABOVE + + physical_plan after FilterPushdown: + SAME TEXT AS ABOVE + + physical_plan after EnforceDistribution: + OutputRequirementExec: order_by=[], dist_by=Unspecified + ProjectionExec: expr=[#2@0 as #4, sum(t.#3)@1 as #5] + AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[sum(t.#3)] + RepartitionExec: partitioning=Hash([#2@0], 4), input_partitions=1 + AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3)] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after CombinePartialFinalAggregate: + SAME TEXT AS ABOVE + + physical_plan after EnforceSorting: + SAME TEXT AS ABOVE + + physical_plan after OptimizeAggregateOrder: + SAME TEXT AS ABOVE + + physical_plan after WindowTopN: + SAME TEXT AS ABOVE + + physical_plan after ProjectionPushdown: + SAME TEXT AS ABOVE + + physical_plan after OutputRequirements: + ProjectionExec: expr=[#2@0 as #4, sum(t.#3)@1 as #5] + AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[sum(t.#3)] + RepartitionExec: partitioning=Hash([#2@0], 4), input_partitions=1 + AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3)] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after LimitAggregation: + SAME TEXT AS ABOVE + + physical_plan after LimitPushPastWindows: + SAME TEXT AS ABOVE + + physical_plan after HashJoinBuffering: + SAME TEXT AS ABOVE + + physical_plan after LimitPushdown: + SAME TEXT AS ABOVE + + physical_plan after TopKRepartition: + SAME TEXT AS ABOVE + + physical_plan after ProjectionPushdown: + SAME TEXT AS ABOVE + + physical_plan after PushdownSort: + SAME TEXT AS ABOVE + + physical_plan after EnsureCooperative: + SAME TEXT AS ABOVE + + physical_plan after FilterPushdown(Post): + SAME TEXT AS ABOVE + + physical_plan after RewriteExplicitRepartition: + SAME TEXT AS ABOVE + + physical_plan after RewriteCollectLeftHashJoin: + SAME TEXT AS ABOVE + + physical_plan after EnforceBarrierPartitioning: + SAME TEXT AS ABOVE + + physical_plan after SanityCheckPlan: + SAME TEXT AS ABOVE + + physical_plan: + ProjectionExec: expr=[#4@0 as k, #5@1 as total] + ProjectionExec: expr=[#2@0 as #4, sum(t.#3)@1 as #5] + AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[sum(t.#3)] + RepartitionExec: partitioning=Hash([#2@0], 4), input_partitions=1 + AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3)] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + + == Physical Plan == ProjectionExec: expr=[#4@0 as k, #5@1 as total] ProjectionExec: expr=[#2@0 as #4, sum(t.#3)@1 as #5] diff --git a/python/pysail/tests/spark/execution/__snapshots__/features/lakehouse_commit.yaml b/python/pysail/tests/spark/execution/__snapshots__/features/lakehouse_commit.yaml index 3d364b4c1..67c35a94f 100644 --- a/python/pysail/tests/spark/execution/__snapshots__/features/lakehouse_commit.yaml +++ b/python/pysail/tests/spark/execution/__snapshots__/features/lakehouse_commit.yaml @@ -3,6 +3,440 @@ name: "test_delta_commit_runs_on_the_driver_after_parallel_file_writing" data: |- + == Codegen == + Whole-stage codegen is not supported; showing physical plan instead. + + == Plan Steps == + initial_logical_plan: + Barrier + DeltaWrite: options=DeltaWriteNodeOptions { path: "/distributed_delta_commit", mode: Append, partition_by: [], bucket_by: None, sort_order: [], options: [TablePropertyList { items: [] }, OptionList { items: [("path", "/distributed_delta_commit")] }], lakehouse_table: Some(LakehouseExecutionContext { catalog_provider_id: CatalogProviderId("sail"), catalog_table: ["distributed_delta_commit"], table_identity: CatalogTableIdentity { table_id: None, table_uri: Some("/distributed_delta_commit") }, operation: Write, format: Delta, authority: CatalogRegistered { lifecycle: External, pointer: StorageDiscovery, commit: Filesystem }, pointer: StorageDiscovery, commit: Filesystem, scan: ClientTableFormat, access_session: None, rest_session: None, versioned_catalog: None, cross_format: None, governance: None, capability_fingerprint: CapabilityFingerprint("sail:distributed_delta_commit:Delta:CatalogRegistered { lifecycle: External, pointer: StorageDiscovery, commit: Filesystem }") }) } + Projection: CASE WHEN Boolean(true) THEN id ELSE Int64(NULL) END AS id + Projection: #1 AS id + Projection: range.#0 AS #1 + Projection: range.id AS #0 + TableScan: range + + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE + + logical_plan after resolve_grouping_function: + SAME TEXT AS ABOVE + + logical_plan after type_coercion: + SAME TEXT AS ABOVE + + analyzed_logical_plan: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_projection: + SAME TEXT AS ABOVE + + logical_plan after rewrite_set_comparison: + SAME TEXT AS ABOVE + + logical_plan after optimize_unions: + SAME TEXT AS ABOVE + + logical_plan after unions_to_filter: + SAME TEXT AS ABOVE + + logical_plan after simplify_expressions: + Barrier + DeltaWrite: options=DeltaWriteNodeOptions { path: "/distributed_delta_commit", mode: Append, partition_by: [], bucket_by: None, sort_order: [], options: [TablePropertyList { items: [] }, OptionList { items: [("path", "/distributed_delta_commit")] }], lakehouse_table: Some(LakehouseExecutionContext { catalog_provider_id: CatalogProviderId("sail"), catalog_table: ["distributed_delta_commit"], table_identity: CatalogTableIdentity { table_id: None, table_uri: Some("/distributed_delta_commit") }, operation: Write, format: Delta, authority: CatalogRegistered { lifecycle: External, pointer: StorageDiscovery, commit: Filesystem }, pointer: StorageDiscovery, commit: Filesystem, scan: ClientTableFormat, access_session: None, rest_session: None, versioned_catalog: None, cross_format: None, governance: None, capability_fingerprint: CapabilityFingerprint("sail:distributed_delta_commit:Delta:CatalogRegistered { lifecycle: External, pointer: StorageDiscovery, commit: Filesystem }") }) } + Projection: id AS id + Projection: #1 AS id + Projection: range.#0 AS #1 + Projection: range.id AS #0 + TableScan: range + + logical_plan after replace_distinct_aggregate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_predicate_subquery: + SAME TEXT AS ABOVE + + logical_plan after scalar_subquery_to_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_join: + SAME TEXT AS ABOVE + + logical_plan after extract_equijoin_predicate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_duplicated_expr: + SAME TEXT AS ABOVE + + logical_plan after eliminate_filter: + SAME TEXT AS ABOVE + + logical_plan after eliminate_cross_join: + SAME TEXT AS ABOVE + + logical_plan after eliminate_limit: + SAME TEXT AS ABOVE + + logical_plan after propagate_empty_relation: + SAME TEXT AS ABOVE + + logical_plan after filter_null_join_keys: + SAME TEXT AS ABOVE + + logical_plan after eliminate_outer_join: + SAME TEXT AS ABOVE + + logical_plan after push_down_limit: + SAME TEXT AS ABOVE + + logical_plan after push_down_filter: + SAME TEXT AS ABOVE + + logical_plan after single_distinct_aggregation_to_group_by: + SAME TEXT AS ABOVE + + logical_plan after eliminate_group_by_constant: + SAME TEXT AS ABOVE + + logical_plan after common_sub_expression_eliminate: + SAME TEXT AS ABOVE + + logical_plan after extract_leaf_expressions: + SAME TEXT AS ABOVE + + logical_plan after optimize_projections: + Barrier + DeltaWrite: options=DeltaWriteNodeOptions { path: "/distributed_delta_commit", mode: Append, partition_by: [], bucket_by: None, sort_order: [], options: [TablePropertyList { items: [] }, OptionList { items: [("path", "/distributed_delta_commit")] }], lakehouse_table: Some(LakehouseExecutionContext { catalog_provider_id: CatalogProviderId("sail"), catalog_table: ["distributed_delta_commit"], table_identity: CatalogTableIdentity { table_id: None, table_uri: Some("/distributed_delta_commit") }, operation: Write, format: Delta, authority: CatalogRegistered { lifecycle: External, pointer: StorageDiscovery, commit: Filesystem }, pointer: StorageDiscovery, commit: Filesystem, scan: ClientTableFormat, access_session: None, rest_session: None, versioned_catalog: None, cross_format: None, governance: None, capability_fingerprint: CapabilityFingerprint("sail:distributed_delta_commit:Delta:CatalogRegistered { lifecycle: External, pointer: StorageDiscovery, commit: Filesystem }") }) } + Projection: #1 AS id + Projection: range.id AS #1 + TableScan: range projection=[id] + + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_projection: + SAME TEXT AS ABOVE + + logical_plan after rewrite_set_comparison: + SAME TEXT AS ABOVE + + logical_plan after optimize_unions: + SAME TEXT AS ABOVE + + logical_plan after unions_to_filter: + SAME TEXT AS ABOVE + + logical_plan after simplify_expressions: + SAME TEXT AS ABOVE + + logical_plan after replace_distinct_aggregate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_predicate_subquery: + SAME TEXT AS ABOVE + + logical_plan after scalar_subquery_to_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_join: + SAME TEXT AS ABOVE + + logical_plan after extract_equijoin_predicate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_duplicated_expr: + SAME TEXT AS ABOVE + + logical_plan after eliminate_filter: + SAME TEXT AS ABOVE + + logical_plan after eliminate_cross_join: + SAME TEXT AS ABOVE + + logical_plan after eliminate_limit: + SAME TEXT AS ABOVE + + logical_plan after propagate_empty_relation: + SAME TEXT AS ABOVE + + logical_plan after filter_null_join_keys: + SAME TEXT AS ABOVE + + logical_plan after eliminate_outer_join: + SAME TEXT AS ABOVE + + logical_plan after push_down_limit: + SAME TEXT AS ABOVE + + logical_plan after push_down_filter: + SAME TEXT AS ABOVE + + logical_plan after single_distinct_aggregation_to_group_by: + SAME TEXT AS ABOVE + + logical_plan after eliminate_group_by_constant: + SAME TEXT AS ABOVE + + logical_plan after common_sub_expression_eliminate: + SAME TEXT AS ABOVE + + logical_plan after extract_leaf_expressions: + SAME TEXT AS ABOVE + + logical_plan after optimize_projections: + Barrier + DeltaWrite: options=DeltaWriteNodeOptions { path: "/distributed_delta_commit", mode: Append, partition_by: [], bucket_by: None, sort_order: [], options: [TablePropertyList { items: [] }, OptionList { items: [("path", "/distributed_delta_commit")] }], lakehouse_table: Some(LakehouseExecutionContext { catalog_provider_id: CatalogProviderId("sail"), catalog_table: ["distributed_delta_commit"], table_identity: CatalogTableIdentity { table_id: None, table_uri: Some("/distributed_delta_commit") }, operation: Write, format: Delta, authority: CatalogRegistered { lifecycle: External, pointer: StorageDiscovery, commit: Filesystem }, pointer: StorageDiscovery, commit: Filesystem, scan: ClientTableFormat, access_session: None, rest_session: None, versioned_catalog: None, cross_format: None, governance: None, capability_fingerprint: CapabilityFingerprint("sail:distributed_delta_commit:Delta:CatalogRegistered { lifecycle: External, pointer: StorageDiscovery, commit: Filesystem }") }) } + Projection: range.id AS id + TableScan: range projection=[id] + + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_projection: + SAME TEXT AS ABOVE + + logical_plan after rewrite_set_comparison: + SAME TEXT AS ABOVE + + logical_plan after optimize_unions: + SAME TEXT AS ABOVE + + logical_plan after unions_to_filter: + SAME TEXT AS ABOVE + + logical_plan after simplify_expressions: + SAME TEXT AS ABOVE + + logical_plan after replace_distinct_aggregate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_predicate_subquery: + SAME TEXT AS ABOVE + + logical_plan after scalar_subquery_to_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_join: + SAME TEXT AS ABOVE + + logical_plan after extract_equijoin_predicate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_duplicated_expr: + SAME TEXT AS ABOVE + + logical_plan after eliminate_filter: + SAME TEXT AS ABOVE + + logical_plan after eliminate_cross_join: + SAME TEXT AS ABOVE + + logical_plan after eliminate_limit: + SAME TEXT AS ABOVE + + logical_plan after propagate_empty_relation: + SAME TEXT AS ABOVE + + logical_plan after filter_null_join_keys: + SAME TEXT AS ABOVE + + logical_plan after eliminate_outer_join: + SAME TEXT AS ABOVE + + logical_plan after push_down_limit: + SAME TEXT AS ABOVE + + logical_plan after push_down_filter: + SAME TEXT AS ABOVE + + logical_plan after single_distinct_aggregation_to_group_by: + SAME TEXT AS ABOVE + + logical_plan after eliminate_group_by_constant: + SAME TEXT AS ABOVE + + logical_plan after common_sub_expression_eliminate: + SAME TEXT AS ABOVE + + logical_plan after extract_leaf_expressions: + SAME TEXT AS ABOVE + + logical_plan after optimize_projections: + SAME TEXT AS ABOVE + + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE + + logical_plan: + Barrier + DeltaWrite: options=DeltaWriteNodeOptions { path: "/distributed_delta_commit", mode: Append, partition_by: [], bucket_by: None, sort_order: [], options: [TablePropertyList { items: [] }, OptionList { items: [("path", "/distributed_delta_commit")] }], lakehouse_table: Some(LakehouseExecutionContext { catalog_provider_id: CatalogProviderId("sail"), catalog_table: ["distributed_delta_commit"], table_identity: CatalogTableIdentity { table_id: None, table_uri: Some("/distributed_delta_commit") }, operation: Write, format: Delta, authority: CatalogRegistered { lifecycle: External, pointer: StorageDiscovery, commit: Filesystem }, pointer: StorageDiscovery, commit: Filesystem, scan: ClientTableFormat, access_session: None, rest_session: None, versioned_catalog: None, cross_format: None, governance: None, capability_fingerprint: CapabilityFingerprint("sail:distributed_delta_commit:Delta:CatalogRegistered { lifecycle: External, pointer: StorageDiscovery, commit: Filesystem }") }) } + Projection: range.id AS id + TableScan: range projection=[id] + + initial_physical_plan: + DeltaCommitExec(table_path=file:///distributed_delta_commit/) + CoalescePartitionsExec + DeltaWriterExec(table_path=file:///distributed_delta_commit/) + SortExec: expr=[1 ASC], preserve_partitioning=[false] + RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=4 + ProjectionExec: expr=[id@0 as id] + ProjectionExec: expr=[id@0 as id] + RangeExec + + + initial_physical_plan_with_stats: + DeltaCommitExec(table_path=file:///distributed_delta_commit/), statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:)]] + CoalescePartitionsExec, statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:)]] + DeltaWriterExec(table_path=file:///distributed_delta_commit/), statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:)]] + SortExec: expr=[1 ASC], preserve_partitioning=[false], statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:)]] + RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=4, statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:)]] + ProjectionExec: expr=[id@0 as id], statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:)]] + ProjectionExec: expr=[id@0 as id], statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:)]] + RangeExec, statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:)]] + + + initial_physical_plan_with_schema: + DeltaCommitExec(table_path=file:///distributed_delta_commit/), schema=[count:UInt64;N] + CoalescePartitionsExec, schema=[action:Union(Dense, 0: ("add": non-null Struct("path": non-null Utf8, "partitionValues": non-null Map("key_value": non-null Struct("key": non-null Utf8, "value": Utf8), unsorted), "size": non-null Int64, "modificationTime": non-null Int64, "dataChange": non-null Boolean, "stats": Utf8, "tags": Map("key_value": non-null Struct("key": non-null Utf8, "value": Utf8), unsorted), "deletionVector": Struct("storageType": non-null Utf8, "pathOrInlineDv": non-null Utf8, "offset": Int32, "sizeInBytes": non-null Int32, "cardinality": non-null Int64), "baseRowId": Int64, "defaultRowCommitVersion": Int64, "clusteringProvider": Utf8)), 1: ("remove": non-null Struct("path": non-null Utf8, "dataChange": non-null Boolean, "deletionTimestamp": Int64, "extendedFileMetadata": Boolean, "partitionValues": Map("key_value": non-null Struct("key": non-null Utf8, "value": Utf8), unsorted), "size": Int64, "stats": Utf8, "tags": Map("key_value": non-null Struct("key": non-null Utf8, "value": Utf8), unsorted), "deletionVector": Struct("storageType": non-null Utf8, "pathOrInlineDv": non-null Utf8, "offset": Int32, "sizeInBytes": non-null Int32, "cardinality": non-null Int64), "baseRowId": Int64, "defaultRowCommitVersion": Int64)), 2: ("protocol": non-null Struct("minReaderVersion": non-null Int32, "minWriterVersion": non-null Int32, "readerFeatures": List(Utf8, field: 'element'), "writerFeatures": List(Utf8, field: 'element'))), 3: ("metadata": non-null Struct("id": non-null Utf8, "name": Utf8, "description": Utf8, "format": non-null Struct("provider": non-null Utf8, "options": non-null Map("key_value": non-null Struct("key": non-null Utf8, "value": Utf8), unsorted)), "schemaString": non-null Utf8, "partitionColumns": non-null List(Utf8, field: 'element'), "createdTime": Int64, "configuration": non-null Map("key_value": non-null Struct("key": non-null Utf8, "value": Utf8), unsorted))), 4: ("commit_meta": non-null Struct("commit_row_count": non-null UInt64, "operation_json": Utf8, "operation_metrics_json": Utf8)))] + DeltaWriterExec(table_path=file:///distributed_delta_commit/), schema=[action:Union(Dense, 0: ("add": non-null Struct("path": non-null Utf8, "partitionValues": non-null Map("key_value": non-null Struct("key": non-null Utf8, "value": Utf8), unsorted), "size": non-null Int64, "modificationTime": non-null Int64, "dataChange": non-null Boolean, "stats": Utf8, "tags": Map("key_value": non-null Struct("key": non-null Utf8, "value": Utf8), unsorted), "deletionVector": Struct("storageType": non-null Utf8, "pathOrInlineDv": non-null Utf8, "offset": Int32, "sizeInBytes": non-null Int32, "cardinality": non-null Int64), "baseRowId": Int64, "defaultRowCommitVersion": Int64, "clusteringProvider": Utf8)), 1: ("remove": non-null Struct("path": non-null Utf8, "dataChange": non-null Boolean, "deletionTimestamp": Int64, "extendedFileMetadata": Boolean, "partitionValues": Map("key_value": non-null Struct("key": non-null Utf8, "value": Utf8), unsorted), "size": Int64, "stats": Utf8, "tags": Map("key_value": non-null Struct("key": non-null Utf8, "value": Utf8), unsorted), "deletionVector": Struct("storageType": non-null Utf8, "pathOrInlineDv": non-null Utf8, "offset": Int32, "sizeInBytes": non-null Int32, "cardinality": non-null Int64), "baseRowId": Int64, "defaultRowCommitVersion": Int64)), 2: ("protocol": non-null Struct("minReaderVersion": non-null Int32, "minWriterVersion": non-null Int32, "readerFeatures": List(Utf8, field: 'element'), "writerFeatures": List(Utf8, field: 'element'))), 3: ("metadata": non-null Struct("id": non-null Utf8, "name": Utf8, "description": Utf8, "format": non-null Struct("provider": non-null Utf8, "options": non-null Map("key_value": non-null Struct("key": non-null Utf8, "value": Utf8), unsorted)), "schemaString": non-null Utf8, "partitionColumns": non-null List(Utf8, field: 'element'), "createdTime": Int64, "configuration": non-null Map("key_value": non-null Struct("key": non-null Utf8, "value": Utf8), unsorted))), 4: ("commit_meta": non-null Struct("commit_row_count": non-null UInt64, "operation_json": Utf8, "operation_metrics_json": Utf8)))] + SortExec: expr=[1 ASC], preserve_partitioning=[false], schema=[id:Int64] + RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=4, schema=[id:Int64] + ProjectionExec: expr=[id@0 as id], schema=[id:Int64] + ProjectionExec: expr=[id@0 as id], schema=[id:Int64] + RangeExec, schema=[id:Int64] + + + physical_plan after OutputRequirements: + OutputRequirementExec: order_by=[], dist_by=Unspecified + DeltaCommitExec(table_path=file:///distributed_delta_commit/) + CoalescePartitionsExec + DeltaWriterExec(table_path=file:///distributed_delta_commit/) + SortExec: expr=[1 ASC], preserve_partitioning=[false] + RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=4 + ProjectionExec: expr=[id@0 as id] + ProjectionExec: expr=[id@0 as id] + RangeExec + + + physical_plan after aggregate_statistics: + SAME TEXT AS ABOVE + + physical_plan after join_selection: + SAME TEXT AS ABOVE + + physical_plan after LimitedDistinctAggregation: + SAME TEXT AS ABOVE + + physical_plan after FilterPushdown: + SAME TEXT AS ABOVE + + physical_plan after EnforceDistribution: + OutputRequirementExec: order_by=[], dist_by=Unspecified + DeltaCommitExec(table_path=file:///distributed_delta_commit/) + CoalescePartitionsExec + DeltaWriterExec(table_path=file:///distributed_delta_commit/) + RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 + SortExec: expr=[1 ASC], preserve_partitioning=[false] + CoalescePartitionsExec + ProjectionExec: expr=[id@0 as id] + ProjectionExec: expr=[id@0 as id] + RangeExec + + + physical_plan after CombinePartialFinalAggregate: + SAME TEXT AS ABOVE + + physical_plan after EnforceSorting: + OutputRequirementExec: order_by=[], dist_by=Unspecified + DeltaCommitExec(table_path=file:///distributed_delta_commit/) + CoalescePartitionsExec + DeltaWriterExec(table_path=file:///distributed_delta_commit/) + ProjectionExec: expr=[id@0 as id] + ProjectionExec: expr=[id@0 as id] + RangeExec + + + physical_plan after OptimizeAggregateOrder: + SAME TEXT AS ABOVE + + physical_plan after WindowTopN: + SAME TEXT AS ABOVE + + physical_plan after ProjectionPushdown: + OutputRequirementExec: order_by=[], dist_by=Unspecified + DeltaCommitExec(table_path=file:///distributed_delta_commit/) + CoalescePartitionsExec + DeltaWriterExec(table_path=file:///distributed_delta_commit/) + ProjectionExec: expr=[id@0 as id] + RangeExec + + + physical_plan after OutputRequirements: + DeltaCommitExec(table_path=file:///distributed_delta_commit/) + CoalescePartitionsExec + DeltaWriterExec(table_path=file:///distributed_delta_commit/) + ProjectionExec: expr=[id@0 as id] + RangeExec + + + physical_plan after LimitAggregation: + SAME TEXT AS ABOVE + + physical_plan after LimitPushPastWindows: + SAME TEXT AS ABOVE + + physical_plan after HashJoinBuffering: + SAME TEXT AS ABOVE + + physical_plan after LimitPushdown: + SAME TEXT AS ABOVE + + physical_plan after TopKRepartition: + SAME TEXT AS ABOVE + + physical_plan after ProjectionPushdown: + DeltaCommitExec(table_path=file:///distributed_delta_commit/) + CoalescePartitionsExec + DeltaWriterExec(table_path=file:///distributed_delta_commit/) + RangeExec + + + physical_plan after PushdownSort: + SAME TEXT AS ABOVE + + physical_plan after EnsureCooperative: + SAME TEXT AS ABOVE + + physical_plan after FilterPushdown(Post): + SAME TEXT AS ABOVE + + physical_plan after RewriteExplicitRepartition: + SAME TEXT AS ABOVE + + physical_plan after RewriteCollectLeftHashJoin: + SAME TEXT AS ABOVE + + physical_plan after EnforceBarrierPartitioning: + SAME TEXT AS ABOVE + + physical_plan after SanityCheckPlan: + SAME TEXT AS ABOVE + + physical_plan: + DeltaCommitExec(table_path=file:///distributed_delta_commit/) + CoalescePartitionsExec + DeltaWriterExec(table_path=file:///distributed_delta_commit/) + RangeExec + + == Physical Plan == DeltaCommitExec(table_path=file:///distributed_delta_commit/) CoalescePartitionsExec @@ -40,6 +474,411 @@ data: name: "test_iceberg_commit_runs_on_the_driver_after_parallel_file_writing" data: |- + == Codegen == + Whole-stage codegen is not supported; showing physical plan instead. + + == Plan Steps == + initial_logical_plan: + Barrier + IcebergWrite: options=IcebergWriteNodeOptions { path: "file:///distributed_iceberg_commit", mode: Append, partition_by: [], bucket_by: None, sort_order: [], options: [TablePropertyList { items: [("metadata_location", "file:///distributed_iceberg_commit/metadata/v1.metadata.json")] }, OptionList { items: [("path", "file:///distributed_iceberg_commit")] }], lakehouse_table: Some(LakehouseExecutionContext { catalog_provider_id: CatalogProviderId("sail"), catalog_table: ["distributed_iceberg_commit"], table_identity: CatalogTableIdentity { table_id: None, table_uri: Some("file:///distributed_iceberg_commit") }, operation: Write, format: Iceberg, authority: CatalogRegistered { lifecycle: External, pointer: StorageDiscovery, commit: Filesystem }, pointer: StorageDiscovery, commit: Filesystem, scan: ClientTableFormat, access_session: None, rest_session: None, versioned_catalog: None, cross_format: None, governance: None, capability_fingerprint: CapabilityFingerprint("sail:distributed_iceberg_commit:Iceberg:CatalogRegistered { lifecycle: External, pointer: StorageDiscovery, commit: Filesystem }") }) } + Projection: CASE WHEN Boolean(true) THEN id ELSE Int64(NULL) END AS id + Projection: #1 AS id + Projection: range.#0 AS #1 + Projection: range.id AS #0 + TableScan: range + + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE + + logical_plan after resolve_grouping_function: + SAME TEXT AS ABOVE + + logical_plan after type_coercion: + SAME TEXT AS ABOVE + + analyzed_logical_plan: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_projection: + SAME TEXT AS ABOVE + + logical_plan after rewrite_set_comparison: + SAME TEXT AS ABOVE + + logical_plan after optimize_unions: + SAME TEXT AS ABOVE + + logical_plan after unions_to_filter: + SAME TEXT AS ABOVE + + logical_plan after simplify_expressions: + Barrier + IcebergWrite: options=IcebergWriteNodeOptions { path: "file:///distributed_iceberg_commit", mode: Append, partition_by: [], bucket_by: None, sort_order: [], options: [TablePropertyList { items: [("metadata_location", "file:///distributed_iceberg_commit/metadata/v1.metadata.json")] }, OptionList { items: [("path", "file:///distributed_iceberg_commit")] }], lakehouse_table: Some(LakehouseExecutionContext { catalog_provider_id: CatalogProviderId("sail"), catalog_table: ["distributed_iceberg_commit"], table_identity: CatalogTableIdentity { table_id: None, table_uri: Some("file:///distributed_iceberg_commit") }, operation: Write, format: Iceberg, authority: CatalogRegistered { lifecycle: External, pointer: StorageDiscovery, commit: Filesystem }, pointer: StorageDiscovery, commit: Filesystem, scan: ClientTableFormat, access_session: None, rest_session: None, versioned_catalog: None, cross_format: None, governance: None, capability_fingerprint: CapabilityFingerprint("sail:distributed_iceberg_commit:Iceberg:CatalogRegistered { lifecycle: External, pointer: StorageDiscovery, commit: Filesystem }") }) } + Projection: id AS id + Projection: #1 AS id + Projection: range.#0 AS #1 + Projection: range.id AS #0 + TableScan: range + + logical_plan after replace_distinct_aggregate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_predicate_subquery: + SAME TEXT AS ABOVE + + logical_plan after scalar_subquery_to_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_join: + SAME TEXT AS ABOVE + + logical_plan after extract_equijoin_predicate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_duplicated_expr: + SAME TEXT AS ABOVE + + logical_plan after eliminate_filter: + SAME TEXT AS ABOVE + + logical_plan after eliminate_cross_join: + SAME TEXT AS ABOVE + + logical_plan after eliminate_limit: + SAME TEXT AS ABOVE + + logical_plan after propagate_empty_relation: + SAME TEXT AS ABOVE + + logical_plan after filter_null_join_keys: + SAME TEXT AS ABOVE + + logical_plan after eliminate_outer_join: + SAME TEXT AS ABOVE + + logical_plan after push_down_limit: + SAME TEXT AS ABOVE + + logical_plan after push_down_filter: + SAME TEXT AS ABOVE + + logical_plan after single_distinct_aggregation_to_group_by: + SAME TEXT AS ABOVE + + logical_plan after eliminate_group_by_constant: + SAME TEXT AS ABOVE + + logical_plan after common_sub_expression_eliminate: + SAME TEXT AS ABOVE + + logical_plan after extract_leaf_expressions: + SAME TEXT AS ABOVE + + logical_plan after optimize_projections: + Barrier + IcebergWrite: options=IcebergWriteNodeOptions { path: "file:///distributed_iceberg_commit", mode: Append, partition_by: [], bucket_by: None, sort_order: [], options: [TablePropertyList { items: [("metadata_location", "file:///distributed_iceberg_commit/metadata/v1.metadata.json")] }, OptionList { items: [("path", "file:///distributed_iceberg_commit")] }], lakehouse_table: Some(LakehouseExecutionContext { catalog_provider_id: CatalogProviderId("sail"), catalog_table: ["distributed_iceberg_commit"], table_identity: CatalogTableIdentity { table_id: None, table_uri: Some("file:///distributed_iceberg_commit") }, operation: Write, format: Iceberg, authority: CatalogRegistered { lifecycle: External, pointer: StorageDiscovery, commit: Filesystem }, pointer: StorageDiscovery, commit: Filesystem, scan: ClientTableFormat, access_session: None, rest_session: None, versioned_catalog: None, cross_format: None, governance: None, capability_fingerprint: CapabilityFingerprint("sail:distributed_iceberg_commit:Iceberg:CatalogRegistered { lifecycle: External, pointer: StorageDiscovery, commit: Filesystem }") }) } + Projection: #1 AS id + Projection: range.id AS #1 + TableScan: range projection=[id] + + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_projection: + SAME TEXT AS ABOVE + + logical_plan after rewrite_set_comparison: + SAME TEXT AS ABOVE + + logical_plan after optimize_unions: + SAME TEXT AS ABOVE + + logical_plan after unions_to_filter: + SAME TEXT AS ABOVE + + logical_plan after simplify_expressions: + SAME TEXT AS ABOVE + + logical_plan after replace_distinct_aggregate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_predicate_subquery: + SAME TEXT AS ABOVE + + logical_plan after scalar_subquery_to_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_join: + SAME TEXT AS ABOVE + + logical_plan after extract_equijoin_predicate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_duplicated_expr: + SAME TEXT AS ABOVE + + logical_plan after eliminate_filter: + SAME TEXT AS ABOVE + + logical_plan after eliminate_cross_join: + SAME TEXT AS ABOVE + + logical_plan after eliminate_limit: + SAME TEXT AS ABOVE + + logical_plan after propagate_empty_relation: + SAME TEXT AS ABOVE + + logical_plan after filter_null_join_keys: + SAME TEXT AS ABOVE + + logical_plan after eliminate_outer_join: + SAME TEXT AS ABOVE + + logical_plan after push_down_limit: + SAME TEXT AS ABOVE + + logical_plan after push_down_filter: + SAME TEXT AS ABOVE + + logical_plan after single_distinct_aggregation_to_group_by: + SAME TEXT AS ABOVE + + logical_plan after eliminate_group_by_constant: + SAME TEXT AS ABOVE + + logical_plan after common_sub_expression_eliminate: + SAME TEXT AS ABOVE + + logical_plan after extract_leaf_expressions: + SAME TEXT AS ABOVE + + logical_plan after optimize_projections: + Barrier + IcebergWrite: options=IcebergWriteNodeOptions { path: "file:///distributed_iceberg_commit", mode: Append, partition_by: [], bucket_by: None, sort_order: [], options: [TablePropertyList { items: [("metadata_location", "file:///distributed_iceberg_commit/metadata/v1.metadata.json")] }, OptionList { items: [("path", "file:///distributed_iceberg_commit")] }], lakehouse_table: Some(LakehouseExecutionContext { catalog_provider_id: CatalogProviderId("sail"), catalog_table: ["distributed_iceberg_commit"], table_identity: CatalogTableIdentity { table_id: None, table_uri: Some("file:///distributed_iceberg_commit") }, operation: Write, format: Iceberg, authority: CatalogRegistered { lifecycle: External, pointer: StorageDiscovery, commit: Filesystem }, pointer: StorageDiscovery, commit: Filesystem, scan: ClientTableFormat, access_session: None, rest_session: None, versioned_catalog: None, cross_format: None, governance: None, capability_fingerprint: CapabilityFingerprint("sail:distributed_iceberg_commit:Iceberg:CatalogRegistered { lifecycle: External, pointer: StorageDiscovery, commit: Filesystem }") }) } + Projection: range.id AS id + TableScan: range projection=[id] + + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_projection: + SAME TEXT AS ABOVE + + logical_plan after rewrite_set_comparison: + SAME TEXT AS ABOVE + + logical_plan after optimize_unions: + SAME TEXT AS ABOVE + + logical_plan after unions_to_filter: + SAME TEXT AS ABOVE + + logical_plan after simplify_expressions: + SAME TEXT AS ABOVE + + logical_plan after replace_distinct_aggregate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_predicate_subquery: + SAME TEXT AS ABOVE + + logical_plan after scalar_subquery_to_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_join: + SAME TEXT AS ABOVE + + logical_plan after extract_equijoin_predicate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_duplicated_expr: + SAME TEXT AS ABOVE + + logical_plan after eliminate_filter: + SAME TEXT AS ABOVE + + logical_plan after eliminate_cross_join: + SAME TEXT AS ABOVE + + logical_plan after eliminate_limit: + SAME TEXT AS ABOVE + + logical_plan after propagate_empty_relation: + SAME TEXT AS ABOVE + + logical_plan after filter_null_join_keys: + SAME TEXT AS ABOVE + + logical_plan after eliminate_outer_join: + SAME TEXT AS ABOVE + + logical_plan after push_down_limit: + SAME TEXT AS ABOVE + + logical_plan after push_down_filter: + SAME TEXT AS ABOVE + + logical_plan after single_distinct_aggregation_to_group_by: + SAME TEXT AS ABOVE + + logical_plan after eliminate_group_by_constant: + SAME TEXT AS ABOVE + + logical_plan after common_sub_expression_eliminate: + SAME TEXT AS ABOVE + + logical_plan after extract_leaf_expressions: + SAME TEXT AS ABOVE + + logical_plan after optimize_projections: + SAME TEXT AS ABOVE + + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE + + logical_plan: + Barrier + IcebergWrite: options=IcebergWriteNodeOptions { path: "file:///distributed_iceberg_commit", mode: Append, partition_by: [], bucket_by: None, sort_order: [], options: [TablePropertyList { items: [("metadata_location", "file:///distributed_iceberg_commit/metadata/v1.metadata.json")] }, OptionList { items: [("path", "file:///distributed_iceberg_commit")] }], lakehouse_table: Some(LakehouseExecutionContext { catalog_provider_id: CatalogProviderId("sail"), catalog_table: ["distributed_iceberg_commit"], table_identity: CatalogTableIdentity { table_id: None, table_uri: Some("file:///distributed_iceberg_commit") }, operation: Write, format: Iceberg, authority: CatalogRegistered { lifecycle: External, pointer: StorageDiscovery, commit: Filesystem }, pointer: StorageDiscovery, commit: Filesystem, scan: ClientTableFormat, access_session: None, rest_session: None, versioned_catalog: None, cross_format: None, governance: None, capability_fingerprint: CapabilityFingerprint("sail:distributed_iceberg_commit:Iceberg:CatalogRegistered { lifecycle: External, pointer: StorageDiscovery, commit: Filesystem }") }) } + Projection: range.id AS id + TableScan: range projection=[id] + + initial_physical_plan: + IcebergCommitExec(table_path=file:///distributed_iceberg_commit/) + IcebergWriterExec(table_path=file:///distributed_iceberg_commit/) + RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=4 + ProjectionExec: expr=[id@0 as id] + RangeExec + + + initial_physical_plan_with_stats: + IcebergCommitExec(table_path=file:///distributed_iceberg_commit/), statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:)]] + IcebergWriterExec(table_path=file:///distributed_iceberg_commit/), statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:)]] + RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=4, statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:)]] + ProjectionExec: expr=[id@0 as id], statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:)]] + RangeExec, statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:)]] + + + initial_physical_plan_with_schema: + IcebergCommitExec(table_path=file:///distributed_iceberg_commit/), schema=[count:UInt64;N] + IcebergWriterExec(table_path=file:///distributed_iceberg_commit/), schema=[action:Union(Dense, 0: ("add": non-null Struct("content": non-null Utf8, "file_path": non-null Utf8, "file_format": non-null Utf8, "partition": non-null List(non-null Union(Dense, 0: ("Boolean": non-null Boolean), 1: ("Int": non-null Int32), 2: ("Long": non-null Int64), 3: ("Float": non-null Float32), 4: ("Double": non-null Float64), 5: ("String": non-null Utf8), 6: ("Int128": non-null Utf8), 7: ("UInt128": non-null Utf8), 8: ("Binary": non-null List(non-null UInt8, field: 'element'))), field: 'element'), "record_count": non-null UInt64, "file_size_in_bytes": non-null UInt64, "column_sizes": non-null Map("entries": non-null Struct("keys": non-null Int32, "values": non-null UInt64), unsorted), "value_counts": non-null Map("entries": non-null Struct("keys": non-null Int32, "values": non-null UInt64), unsorted), "null_value_counts": non-null Map("entries": non-null Struct("keys": non-null Int32, "values": non-null UInt64), unsorted), "split_offsets": non-null List(non-null Int64, field: 'element'), "partition_spec_id": non-null Int32)), 1: ("delete": non-null Struct("file_path": non-null Utf8)), 2: ("commit_meta": non-null Struct("table_uri": non-null Utf8, "row_count": non-null UInt64, "operation": non-null Utf8, "requirements_json": non-null Utf8, "table_properties_json": non-null Utf8, "lakehouse_table_json": Utf8, "schema_json": Utf8, "partition_spec_json": Utf8)))] + RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=4, schema=[id:Int64] + ProjectionExec: expr=[id@0 as id], schema=[id:Int64] + RangeExec, schema=[id:Int64] + + + physical_plan after OutputRequirements: + OutputRequirementExec: order_by=[], dist_by=Unspecified + IcebergCommitExec(table_path=file:///distributed_iceberg_commit/) + IcebergWriterExec(table_path=file:///distributed_iceberg_commit/) + RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=4 + ProjectionExec: expr=[id@0 as id] + RangeExec + + + physical_plan after aggregate_statistics: + SAME TEXT AS ABOVE + + physical_plan after join_selection: + SAME TEXT AS ABOVE + + physical_plan after LimitedDistinctAggregation: + SAME TEXT AS ABOVE + + physical_plan after FilterPushdown: + SAME TEXT AS ABOVE + + physical_plan after EnforceDistribution: + OutputRequirementExec: order_by=[], dist_by=Unspecified + IcebergCommitExec(table_path=file:///distributed_iceberg_commit/) + CoalescePartitionsExec + IcebergWriterExec(table_path=file:///distributed_iceberg_commit/) + ProjectionExec: expr=[id@0 as id] + RangeExec + + + physical_plan after CombinePartialFinalAggregate: + SAME TEXT AS ABOVE + + physical_plan after EnforceSorting: + SAME TEXT AS ABOVE + + physical_plan after OptimizeAggregateOrder: + SAME TEXT AS ABOVE + + physical_plan after WindowTopN: + SAME TEXT AS ABOVE + + physical_plan after ProjectionPushdown: + OutputRequirementExec: order_by=[], dist_by=Unspecified + IcebergCommitExec(table_path=file:///distributed_iceberg_commit/) + CoalescePartitionsExec + IcebergWriterExec(table_path=file:///distributed_iceberg_commit/) + RangeExec + + + physical_plan after OutputRequirements: + IcebergCommitExec(table_path=file:///distributed_iceberg_commit/) + CoalescePartitionsExec + IcebergWriterExec(table_path=file:///distributed_iceberg_commit/) + RangeExec + + + physical_plan after LimitAggregation: + SAME TEXT AS ABOVE + + physical_plan after LimitPushPastWindows: + SAME TEXT AS ABOVE + + physical_plan after HashJoinBuffering: + SAME TEXT AS ABOVE + + physical_plan after LimitPushdown: + SAME TEXT AS ABOVE + + physical_plan after TopKRepartition: + SAME TEXT AS ABOVE + + physical_plan after ProjectionPushdown: + SAME TEXT AS ABOVE + + physical_plan after PushdownSort: + SAME TEXT AS ABOVE + + physical_plan after EnsureCooperative: + SAME TEXT AS ABOVE + + physical_plan after FilterPushdown(Post): + SAME TEXT AS ABOVE + + physical_plan after RewriteExplicitRepartition: + SAME TEXT AS ABOVE + + physical_plan after RewriteCollectLeftHashJoin: + SAME TEXT AS ABOVE + + physical_plan after EnforceBarrierPartitioning: + SAME TEXT AS ABOVE + + physical_plan after SanityCheckPlan: + SAME TEXT AS ABOVE + + physical_plan: + IcebergCommitExec(table_path=file:///distributed_iceberg_commit/) + CoalescePartitionsExec + IcebergWriterExec(table_path=file:///distributed_iceberg_commit/) + RangeExec + + == Physical Plan == IcebergCommitExec(table_path=file:///distributed_iceberg_commit/) CoalescePartitionsExec diff --git a/python/pysail/tests/spark/execution/__snapshots__/features/scalar_subquery.yaml b/python/pysail/tests/spark/execution/__snapshots__/features/scalar_subquery.yaml index 7e98ab73a..80a55c1c3 100644 --- a/python/pysail/tests/spark/execution/__snapshots__/features/scalar_subquery.yaml +++ b/python/pysail/tests/spark/execution/__snapshots__/features/scalar_subquery.yaml @@ -3,6 +3,586 @@ name: "test_correlated_scalar_subquery_in_filter" data: |- + == Codegen == + Whole-stage codegen is not supported; showing physical plan instead. + + == Plan Steps == + initial_logical_plan: + Sort: #9 ASC NULLS FIRST + Projection: outer_t.#2 AS #9, outer_t.#3 AS #10 + Filter: outer_t.#3 = () + Subquery: + Projection: max(inner_t.#7) AS #8 + Aggregate: groupBy=[[]], aggr=[[max(inner_t.#7)]] + Filter: inner_t.#6 = outer_ref(outer_t.#2) + SubqueryAlias: inner_t + Projection: #4 AS #6, #5 AS #7 + Projection: column1 AS #4, column2 AS #5 + Values: (Int32(1), Int32(4)), (Int32(1), Int32(2)), (Int32(2), Int32(3)), (Int32(2), Int32(1)) + SubqueryAlias: outer_t + Projection: #0 AS #2, #1 AS #3 + Projection: column1 AS #0, column2 AS #1 + Values: (Int32(1), Int32(2)), (Int32(1), Int32(4)), (Int32(2), Int32(1)), (Int32(2), Int32(3)) + + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE + + logical_plan after resolve_grouping_function: + SAME TEXT AS ABOVE + + logical_plan after type_coercion: + SAME TEXT AS ABOVE + + analyzed_logical_plan: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_projection: + SAME TEXT AS ABOVE + + logical_plan after rewrite_set_comparison: + SAME TEXT AS ABOVE + + logical_plan after optimize_unions: + SAME TEXT AS ABOVE + + logical_plan after unions_to_filter: + SAME TEXT AS ABOVE + + logical_plan after simplify_expressions: + SAME TEXT AS ABOVE + + logical_plan after replace_distinct_aggregate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_predicate_subquery: + SAME TEXT AS ABOVE + + logical_plan after scalar_subquery_to_join: + Sort: #9 ASC NULLS FIRST + Projection: outer_t.#2 AS #9, outer_t.#3 AS #10 + Projection: outer_t.#2, outer_t.#3 + Filter: outer_t.#3 = __scalar_sq_1.#8 + Left Join: Filter: __scalar_sq_1.#6 = outer_t.#2 + SubqueryAlias: outer_t + Projection: #0 AS #2, #1 AS #3 + Projection: column1 AS #0, column2 AS #1 + Values: (Int32(1), Int32(2)), (Int32(1), Int32(4)), (Int32(2), Int32(1)), (Int32(2), Int32(3)) + SubqueryAlias: __scalar_sq_1 + Projection: max(inner_t.#7) AS #8, inner_t.#6, __always_true + Aggregate: groupBy=[[inner_t.#6, Boolean(true) AS __always_true]], aggr=[[max(inner_t.#7)]] + SubqueryAlias: inner_t + Projection: #4 AS #6, #5 AS #7 + Projection: column1 AS #4, column2 AS #5 + Values: (Int32(1), Int32(4)), (Int32(1), Int32(2)), (Int32(2), Int32(3)), (Int32(2), Int32(1)) + + logical_plan after decorrelate_lateral_join: + SAME TEXT AS ABOVE + + logical_plan after extract_equijoin_predicate: + Sort: #9 ASC NULLS FIRST + Projection: outer_t.#2 AS #9, outer_t.#3 AS #10 + Projection: outer_t.#2, outer_t.#3 + Filter: outer_t.#3 = __scalar_sq_1.#8 + Left Join: outer_t.#2 = __scalar_sq_1.#6 + SubqueryAlias: outer_t + Projection: #0 AS #2, #1 AS #3 + Projection: column1 AS #0, column2 AS #1 + Values: (Int32(1), Int32(2)), (Int32(1), Int32(4)), (Int32(2), Int32(1)), (Int32(2), Int32(3)) + SubqueryAlias: __scalar_sq_1 + Projection: max(inner_t.#7) AS #8, inner_t.#6, __always_true + Aggregate: groupBy=[[inner_t.#6, Boolean(true) AS __always_true]], aggr=[[max(inner_t.#7)]] + SubqueryAlias: inner_t + Projection: #4 AS #6, #5 AS #7 + Projection: column1 AS #4, column2 AS #5 + Values: (Int32(1), Int32(4)), (Int32(1), Int32(2)), (Int32(2), Int32(3)), (Int32(2), Int32(1)) + + logical_plan after eliminate_duplicated_expr: + SAME TEXT AS ABOVE + + logical_plan after eliminate_filter: + SAME TEXT AS ABOVE + + logical_plan after eliminate_cross_join: + SAME TEXT AS ABOVE + + logical_plan after eliminate_limit: + SAME TEXT AS ABOVE + + logical_plan after propagate_empty_relation: + SAME TEXT AS ABOVE + + logical_plan after filter_null_join_keys: + SAME TEXT AS ABOVE + + logical_plan after eliminate_outer_join: + Sort: #9 ASC NULLS FIRST + Projection: outer_t.#2 AS #9, outer_t.#3 AS #10 + Projection: outer_t.#2, outer_t.#3 + Filter: outer_t.#3 = __scalar_sq_1.#8 + Inner Join: outer_t.#2 = __scalar_sq_1.#6 + SubqueryAlias: outer_t + Projection: #0 AS #2, #1 AS #3 + Projection: column1 AS #0, column2 AS #1 + Values: (Int32(1), Int32(2)), (Int32(1), Int32(4)), (Int32(2), Int32(1)), (Int32(2), Int32(3)) + SubqueryAlias: __scalar_sq_1 + Projection: max(inner_t.#7) AS #8, inner_t.#6, __always_true + Aggregate: groupBy=[[inner_t.#6, Boolean(true) AS __always_true]], aggr=[[max(inner_t.#7)]] + SubqueryAlias: inner_t + Projection: #4 AS #6, #5 AS #7 + Projection: column1 AS #4, column2 AS #5 + Values: (Int32(1), Int32(4)), (Int32(1), Int32(2)), (Int32(2), Int32(3)), (Int32(2), Int32(1)) + + logical_plan after push_down_limit: + SAME TEXT AS ABOVE + + logical_plan after push_down_filter: + Sort: #9 ASC NULLS FIRST + Projection: outer_t.#2 AS #9, outer_t.#3 AS #10 + Projection: outer_t.#2, outer_t.#3 + Inner Join: outer_t.#2 = __scalar_sq_1.#6 Filter: outer_t.#3 = __scalar_sq_1.#8 + SubqueryAlias: outer_t + Projection: #0 AS #2, #1 AS #3 + Projection: column1 AS #0, column2 AS #1 + Values: (Int32(1), Int32(2)), (Int32(1), Int32(4)), (Int32(2), Int32(1)), (Int32(2), Int32(3)) + SubqueryAlias: __scalar_sq_1 + Projection: max(inner_t.#7) AS #8, inner_t.#6, __always_true + Aggregate: groupBy=[[inner_t.#6, Boolean(true) AS __always_true]], aggr=[[max(inner_t.#7)]] + SubqueryAlias: inner_t + Projection: #4 AS #6, #5 AS #7 + Projection: column1 AS #4, column2 AS #5 + Values: (Int32(1), Int32(4)), (Int32(1), Int32(2)), (Int32(2), Int32(3)), (Int32(2), Int32(1)) + + logical_plan after single_distinct_aggregation_to_group_by: + SAME TEXT AS ABOVE + + logical_plan after eliminate_group_by_constant: + Sort: #9 ASC NULLS FIRST + Projection: outer_t.#2 AS #9, outer_t.#3 AS #10 + Projection: outer_t.#2, outer_t.#3 + Inner Join: outer_t.#2 = __scalar_sq_1.#6 Filter: outer_t.#3 = __scalar_sq_1.#8 + SubqueryAlias: outer_t + Projection: #0 AS #2, #1 AS #3 + Projection: column1 AS #0, column2 AS #1 + Values: (Int32(1), Int32(2)), (Int32(1), Int32(4)), (Int32(2), Int32(1)), (Int32(2), Int32(3)) + SubqueryAlias: __scalar_sq_1 + Projection: max(inner_t.#7) AS #8, inner_t.#6, __always_true + Projection: inner_t.#6, Boolean(true) AS __always_true, max(inner_t.#7) + Aggregate: groupBy=[[inner_t.#6]], aggr=[[max(inner_t.#7)]] + SubqueryAlias: inner_t + Projection: #4 AS #6, #5 AS #7 + Projection: column1 AS #4, column2 AS #5 + Values: (Int32(1), Int32(4)), (Int32(1), Int32(2)), (Int32(2), Int32(3)), (Int32(2), Int32(1)) + + logical_plan after common_sub_expression_eliminate: + SAME TEXT AS ABOVE + + logical_plan after extract_leaf_expressions: + SAME TEXT AS ABOVE + + logical_plan after optimize_projections: + Sort: #9 ASC NULLS FIRST + Projection: outer_t.#2 AS #9, outer_t.#3 AS #10 + Inner Join: outer_t.#2 = __scalar_sq_1.#6 Filter: outer_t.#3 = __scalar_sq_1.#8 + SubqueryAlias: outer_t + Projection: column1 AS #2, column2 AS #3 + Values: (Int32(1), Int32(2)), (Int32(1), Int32(4)), (Int32(2), Int32(1)), (Int32(2), Int32(3)) + SubqueryAlias: __scalar_sq_1 + Projection: max(inner_t.#7) AS #8, inner_t.#6 + Aggregate: groupBy=[[inner_t.#6]], aggr=[[max(inner_t.#7)]] + SubqueryAlias: inner_t + Projection: column1 AS #6, column2 AS #7 + Values: (Int32(1), Int32(4)), (Int32(1), Int32(2)), (Int32(2), Int32(3)), (Int32(2), Int32(1)) + + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_projection: + SAME TEXT AS ABOVE + + logical_plan after rewrite_set_comparison: + SAME TEXT AS ABOVE + + logical_plan after optimize_unions: + SAME TEXT AS ABOVE + + logical_plan after unions_to_filter: + SAME TEXT AS ABOVE + + logical_plan after simplify_expressions: + SAME TEXT AS ABOVE + + logical_plan after replace_distinct_aggregate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_predicate_subquery: + SAME TEXT AS ABOVE + + logical_plan after scalar_subquery_to_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_join: + SAME TEXT AS ABOVE + + logical_plan after extract_equijoin_predicate: + Sort: #9 ASC NULLS FIRST + Projection: outer_t.#2 AS #9, outer_t.#3 AS #10 + Inner Join: outer_t.#2 = __scalar_sq_1.#6, outer_t.#3 = __scalar_sq_1.#8 + SubqueryAlias: outer_t + Projection: column1 AS #2, column2 AS #3 + Values: (Int32(1), Int32(2)), (Int32(1), Int32(4)), (Int32(2), Int32(1)), (Int32(2), Int32(3)) + SubqueryAlias: __scalar_sq_1 + Projection: max(inner_t.#7) AS #8, inner_t.#6 + Aggregate: groupBy=[[inner_t.#6]], aggr=[[max(inner_t.#7)]] + SubqueryAlias: inner_t + Projection: column1 AS #6, column2 AS #7 + Values: (Int32(1), Int32(4)), (Int32(1), Int32(2)), (Int32(2), Int32(3)), (Int32(2), Int32(1)) + + logical_plan after eliminate_duplicated_expr: + SAME TEXT AS ABOVE + + logical_plan after eliminate_filter: + SAME TEXT AS ABOVE + + logical_plan after eliminate_cross_join: + SAME TEXT AS ABOVE + + logical_plan after eliminate_limit: + SAME TEXT AS ABOVE + + logical_plan after propagate_empty_relation: + SAME TEXT AS ABOVE + + logical_plan after filter_null_join_keys: + SAME TEXT AS ABOVE + + logical_plan after eliminate_outer_join: + SAME TEXT AS ABOVE + + logical_plan after push_down_limit: + SAME TEXT AS ABOVE + + logical_plan after push_down_filter: + SAME TEXT AS ABOVE + + logical_plan after single_distinct_aggregation_to_group_by: + SAME TEXT AS ABOVE + + logical_plan after eliminate_group_by_constant: + SAME TEXT AS ABOVE + + logical_plan after common_sub_expression_eliminate: + SAME TEXT AS ABOVE + + logical_plan after extract_leaf_expressions: + SAME TEXT AS ABOVE + + logical_plan after optimize_projections: + SAME TEXT AS ABOVE + + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_projection: + SAME TEXT AS ABOVE + + logical_plan after rewrite_set_comparison: + SAME TEXT AS ABOVE + + logical_plan after optimize_unions: + SAME TEXT AS ABOVE + + logical_plan after unions_to_filter: + SAME TEXT AS ABOVE + + logical_plan after simplify_expressions: + SAME TEXT AS ABOVE + + logical_plan after replace_distinct_aggregate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_predicate_subquery: + SAME TEXT AS ABOVE + + logical_plan after scalar_subquery_to_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_join: + SAME TEXT AS ABOVE + + logical_plan after extract_equijoin_predicate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_duplicated_expr: + SAME TEXT AS ABOVE + + logical_plan after eliminate_filter: + SAME TEXT AS ABOVE + + logical_plan after eliminate_cross_join: + SAME TEXT AS ABOVE + + logical_plan after eliminate_limit: + SAME TEXT AS ABOVE + + logical_plan after propagate_empty_relation: + SAME TEXT AS ABOVE + + logical_plan after filter_null_join_keys: + SAME TEXT AS ABOVE + + logical_plan after eliminate_outer_join: + SAME TEXT AS ABOVE + + logical_plan after push_down_limit: + SAME TEXT AS ABOVE + + logical_plan after push_down_filter: + SAME TEXT AS ABOVE + + logical_plan after single_distinct_aggregation_to_group_by: + SAME TEXT AS ABOVE + + logical_plan after eliminate_group_by_constant: + SAME TEXT AS ABOVE + + logical_plan after common_sub_expression_eliminate: + SAME TEXT AS ABOVE + + logical_plan after extract_leaf_expressions: + SAME TEXT AS ABOVE + + logical_plan after optimize_projections: + SAME TEXT AS ABOVE + + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE + + logical_plan: + Sort: #9 ASC NULLS FIRST + Projection: outer_t.#2 AS #9, outer_t.#3 AS #10 + Inner Join: outer_t.#2 = __scalar_sq_1.#6, outer_t.#3 = __scalar_sq_1.#8 + SubqueryAlias: outer_t + Projection: column1 AS #2, column2 AS #3 + Values: (Int32(1), Int32(2)), (Int32(1), Int32(4)), (Int32(2), Int32(1)), (Int32(2), Int32(3)) + SubqueryAlias: __scalar_sq_1 + Projection: max(inner_t.#7) AS #8, inner_t.#6 + Aggregate: groupBy=[[inner_t.#6]], aggr=[[max(inner_t.#7)]] + SubqueryAlias: inner_t + Projection: column1 AS #6, column2 AS #7 + Values: (Int32(1), Int32(4)), (Int32(1), Int32(2)), (Int32(2), Int32(3)), (Int32(2), Int32(1)) + + initial_physical_plan: + SortExec: expr=[#9@0 ASC], preserve_partitioning=[false] + ProjectionExec: expr=[#2@0 as #9, #3@1 as #10] + HashJoinExec: mode=Auto, join_type=Inner, on=[(#2@0, #6@1), (#3@1, #8@0)] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[max(inner_t.#7)@1 as #8, #6@0 as #6] + AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + ProjectionExec: expr=[column1@0 as #6, column2@1 as #7] + DataSourceExec: partitions=1, partition_sizes=[1] + + + initial_physical_plan_with_stats: + SortExec: expr=[#9@0 ASC], preserve_partitioning=[false], statistics=[Rows=Inexact(4), Bytes=Inexact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]] + ProjectionExec: expr=[#2@0 as #9, #3@1 as #10], statistics=[Rows=Inexact(4), Bytes=Inexact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]] + HashJoinExec: mode=Auto, join_type=Inner, on=[(#2@0, #6@1), (#3@1, #8@0)], statistics=[Rows=Inexact(4), Bytes=Absent, [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0)),(Col[2]:),(Col[3]:)]] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3], statistics=[Rows=Exact(4), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]] + DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(4), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]] + ProjectionExec: expr=[max(inner_t.#7)@1 as #8, #6@0 as #6], statistics=[Rows=Inexact(4), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]] + AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)], statistics=[Rows=Inexact(4), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]] + AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)], statistics=[Rows=Inexact(4), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]] + ProjectionExec: expr=[column1@0 as #6, column2@1 as #7], statistics=[Rows=Exact(4), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]] + DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(4), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]] + + + initial_physical_plan_with_schema: + SortExec: expr=[#9@0 ASC], preserve_partitioning=[false], schema=[#9:Int32;N, #10:Int32;N] + ProjectionExec: expr=[#2@0 as #9, #3@1 as #10], schema=[#9:Int32;N, #10:Int32;N] + HashJoinExec: mode=Auto, join_type=Inner, on=[(#2@0, #6@1), (#3@1, #8@0)], schema=[#2:Int32;N, #3:Int32;N, #8:Int32;N, #6:Int32;N] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3], schema=[#2:Int32;N, #3:Int32;N] + DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N, column2:Int32;N] + ProjectionExec: expr=[max(inner_t.#7)@1 as #8, #6@0 as #6], schema=[#8:Int32;N, #6:Int32;N] + AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)], schema=[#6:Int32;N, max(inner_t.#7):Int32;N] + AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)], schema=[#6:Int32;N, max(inner_t.#7)[value]:Int32;N] + ProjectionExec: expr=[column1@0 as #6, column2@1 as #7], schema=[#6:Int32;N, #7:Int32;N] + DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N, column2:Int32;N] + + + physical_plan after OutputRequirements: + OutputRequirementExec: order_by=[(#9@0, asc)], dist_by=SinglePartition + SortExec: expr=[#9@0 ASC], preserve_partitioning=[false] + ProjectionExec: expr=[#2@0 as #9, #3@1 as #10] + HashJoinExec: mode=Auto, join_type=Inner, on=[(#2@0, #6@1), (#3@1, #8@0)] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[max(inner_t.#7)@1 as #8, #6@0 as #6] + AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + ProjectionExec: expr=[column1@0 as #6, column2@1 as #7] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after aggregate_statistics: + SAME TEXT AS ABOVE + + physical_plan after join_selection: + OutputRequirementExec: order_by=[(#9@0, asc)], dist_by=SinglePartition + SortExec: expr=[#9@0 ASC], preserve_partitioning=[false] + ProjectionExec: expr=[#2@0 as #9, #3@1 as #10] + HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(#2@0, #6@1), (#3@1, #8@0)] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[max(inner_t.#7)@1 as #8, #6@0 as #6] + AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + ProjectionExec: expr=[column1@0 as #6, column2@1 as #7] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after LimitedDistinctAggregation: + SAME TEXT AS ABOVE + + physical_plan after FilterPushdown: + SAME TEXT AS ABOVE + + physical_plan after EnforceDistribution: + OutputRequirementExec: order_by=[(#9@0, asc)], dist_by=SinglePartition + SortExec: expr=[#9@0 ASC], preserve_partitioning=[false] + CoalescePartitionsExec + ProjectionExec: expr=[#2@0 as #9, #3@1 as #10] + HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(#2@0, #6@1), (#3@1, #8@0)] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[max(inner_t.#7)@1 as #8, #6@0 as #6] + AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + RepartitionExec: partitioning=Hash([#6@0], 4), input_partitions=1 + AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + ProjectionExec: expr=[column1@0 as #6, column2@1 as #7] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after CombinePartialFinalAggregate: + SAME TEXT AS ABOVE + + physical_plan after EnforceSorting: + OutputRequirementExec: order_by=[(#9@0, asc)], dist_by=SinglePartition + SortPreservingMergeExec: [#9@0 ASC] + SortExec: expr=[#9@0 ASC], preserve_partitioning=[true] + ProjectionExec: expr=[#2@0 as #9, #3@1 as #10] + HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(#2@0, #6@1), (#3@1, #8@0)] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[max(inner_t.#7)@1 as #8, #6@0 as #6] + AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + RepartitionExec: partitioning=Hash([#6@0], 4), input_partitions=1 + AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + ProjectionExec: expr=[column1@0 as #6, column2@1 as #7] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after OptimizeAggregateOrder: + SAME TEXT AS ABOVE + + physical_plan after WindowTopN: + SAME TEXT AS ABOVE + + physical_plan after ProjectionPushdown: + OutputRequirementExec: order_by=[(#9@0, asc)], dist_by=SinglePartition + SortPreservingMergeExec: [#9@0 ASC] + SortExec: expr=[#9@0 ASC], preserve_partitioning=[true] + ProjectionExec: expr=[#2@0 as #9, #3@1 as #10] + HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(#2@0, #6@1), (#3@1, #8@0)], projection=[#2@0, #3@1] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[max(inner_t.#7)@1 as #8, #6@0 as #6] + AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + RepartitionExec: partitioning=Hash([#6@0], 4), input_partitions=1 + AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + ProjectionExec: expr=[column1@0 as #6, column2@1 as #7] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after OutputRequirements: + SortPreservingMergeExec: [#9@0 ASC] + SortExec: expr=[#9@0 ASC], preserve_partitioning=[true] + ProjectionExec: expr=[#2@0 as #9, #3@1 as #10] + HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(#2@0, #6@1), (#3@1, #8@0)], projection=[#2@0, #3@1] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[max(inner_t.#7)@1 as #8, #6@0 as #6] + AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + RepartitionExec: partitioning=Hash([#6@0], 4), input_partitions=1 + AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + ProjectionExec: expr=[column1@0 as #6, column2@1 as #7] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after LimitAggregation: + SAME TEXT AS ABOVE + + physical_plan after LimitPushPastWindows: + SAME TEXT AS ABOVE + + physical_plan after HashJoinBuffering: + SAME TEXT AS ABOVE + + physical_plan after LimitPushdown: + SAME TEXT AS ABOVE + + physical_plan after TopKRepartition: + SAME TEXT AS ABOVE + + physical_plan after ProjectionPushdown: + SAME TEXT AS ABOVE + + physical_plan after PushdownSort: + SAME TEXT AS ABOVE + + physical_plan after EnsureCooperative: + SAME TEXT AS ABOVE + + physical_plan after FilterPushdown(Post): + SAME TEXT AS ABOVE + + physical_plan after RewriteExplicitRepartition: + SAME TEXT AS ABOVE + + physical_plan after RewriteCollectLeftHashJoin: + SAME TEXT AS ABOVE + + physical_plan after EnforceBarrierPartitioning: + SAME TEXT AS ABOVE + + physical_plan after SanityCheckPlan: + SAME TEXT AS ABOVE + + physical_plan: + ProjectionExec: expr=[#9@0 as k, #10@1 as v] + SortPreservingMergeExec: [#9@0 ASC] + SortExec: expr=[#9@0 ASC], preserve_partitioning=[true] + ProjectionExec: expr=[#2@0 as #9, #3@1 as #10] + HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(#2@0, #6@1), (#3@1, #8@0)], projection=[#2@0, #3@1] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[max(inner_t.#7)@1 as #8, #6@0 as #6] + AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + RepartitionExec: partitioning=Hash([#6@0], 4), input_partitions=1 + AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + ProjectionExec: expr=[column1@0 as #6, column2@1 as #7] + DataSourceExec: partitions=1, partition_sizes=[1] + + == Physical Plan == ProjectionExec: expr=[#9@0 as k, #10@1 as v] SortPreservingMergeExec: [#9@0 ASC] @@ -58,81 +638,466 @@ data: name: "test_correlated_scalar_subquery_in_projection" data: |- - == Physical Plan == - ProjectionExec: expr=[#9@0 as k, #10@1 as v, #11@2 as mx] - SortPreservingMergeExec: [#9@0 ASC, #10@1 ASC] - SortExec: expr=[#9@0 ASC, #10@1 ASC], preserve_partitioning=[true] - ProjectionExec: expr=[#2@0 as #9, #3@1 as #10, #8@2 as #11] - HashJoinExec: mode=CollectLeft, join_type=Left, on=[(#2@0, #6@1)], projection=[#2@0, #3@1, #8@2] - ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] - DataSourceExec: partitions=1, partition_sizes=[1] - ProjectionExec: expr=[max(inner_t.#7)@1 as #8, #6@0 as #6] - AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] - RepartitionExec: partitioning=Hash([#6@0], 4), input_partitions=1 - AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] - ProjectionExec: expr=[column1@0 as #6, column2@1 as #7] - DataSourceExec: partitions=1, partition_sizes=[1] + == Codegen == + Whole-stage codegen is not supported; showing physical plan instead. + == Plan Steps == + initial_logical_plan: + Sort: #9 ASC NULLS FIRST, #10 ASC NULLS FIRST + Projection: outer_t.#2 AS #9, outer_t.#3 AS #10, () AS mx AS #11 + Subquery: + Projection: max(inner_t.#7) AS #8 + Aggregate: groupBy=[[]], aggr=[[max(inner_t.#7)]] + Filter: inner_t.#6 = outer_ref(outer_t.#2) + SubqueryAlias: inner_t + Projection: #4 AS #6, #5 AS #7 + Projection: column1 AS #4, column2 AS #5 + Values: (Int32(1), Int32(4)), (Int32(1), Int32(2)), (Int32(2), Int32(3)), (Int32(2), Int32(1)) + SubqueryAlias: outer_t + Projection: #0 AS #2, #1 AS #3 + Projection: column1 AS #0, column2 AS #1 + Values: (Int32(1), Int32(2)), (Int32(1), Int32(4)), (Int32(2), Int32(1)), (Int32(2), Int32(3)) - == Distributed Plan == - === stage 0 === - inputs=[] - mode=Pipelined - partitions=1 - distribution=Hash(keys=[#2@0], channels=4) - placement=Worker - ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] - DataSourceExec: partitions=1, partition_sizes=[1] + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE - === stage 1 === - inputs=[] - mode=Pipelined - partitions=1 - distribution=Hash(keys=[#6@0], channels=4) - placement=Worker - AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] - ProjectionExec: expr=[column1@0 as #6, column2@1 as #7] - DataSourceExec: partitions=1, partition_sizes=[1] + logical_plan after resolve_grouping_function: + SAME TEXT AS ABOVE - === stage 2 === - inputs=[StageInput(stage=1, mode=Shuffle)] - mode=Pipelined - partitions=4 - distribution=Hash(keys=[#6@1], channels=4) - placement=Worker - ProjectionExec: expr=[max(inner_t.#7)@1 as #8, #6@0 as #6] - AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] - StageInputExec: input=0, partitioning=Hash([#6@0], 4) + logical_plan after type_coercion: + SAME TEXT AS ABOVE - === stage 3 === - inputs=[StageInput(stage=0, mode=Shuffle), StageInput(stage=2, mode=Shuffle)] - mode=Pipelined - partitions=4 - distribution=RoundRobin(channels=1) - placement=Worker - SortExec: expr=[#9@0 ASC, #10@1 ASC], preserve_partitioning=[true] - ProjectionExec: expr=[#2@0 as #9, #3@1 as #10, #8@2 as #11] - HashJoinExec: mode=Partitioned, join_type=Left, on=[(#2@0, #6@1)], projection=[#2@0, #3@1, #8@2] - StageInputExec: input=0, partitioning=Hash([#2@0], 4) - StageInputExec: input=1, partitioning=Hash([#6@1], 4) + analyzed_logical_plan: + SAME TEXT AS ABOVE - === stage 4 === - inputs=[StageInput(stage=3, mode=Merge)] - mode=Pipelined - partitions=1 - distribution=RoundRobin(channels=1) - placement=Worker - ProjectionExec: expr=[#9@0 as k, #10@1 as v, #11@2 as mx] - SortPreservingMergeExec: [#9@0 ASC, #10@1 ASC] - StageInputExec: input=0, partitioning=Hash([#9@0], 4) ---- -name: "test_correlated_scalar_subquery_nomatch_result" -data: - |- - == Physical Plan == - ProjectionExec: expr=[#9@0 as k, #10@1 as v, #11@2 as mx] - SortPreservingMergeExec: [#9@0 ASC] - SortExec: expr=[#9@0 ASC], preserve_partitioning=[true] + logical_plan after decorrelate_lateral_projection: + SAME TEXT AS ABOVE + + logical_plan after rewrite_set_comparison: + SAME TEXT AS ABOVE + + logical_plan after optimize_unions: + SAME TEXT AS ABOVE + + logical_plan after unions_to_filter: + SAME TEXT AS ABOVE + + logical_plan after simplify_expressions: + SAME TEXT AS ABOVE + + logical_plan after replace_distinct_aggregate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_predicate_subquery: + SAME TEXT AS ABOVE + + logical_plan after scalar_subquery_to_join: + Sort: #9 ASC NULLS FIRST, #10 ASC NULLS FIRST + Projection: outer_t.#2 AS #9, outer_t.#3 AS #10, __scalar_sq_1.#8 AS mx AS #11 + Left Join: Filter: __scalar_sq_1.#6 = outer_t.#2 + SubqueryAlias: outer_t + Projection: #0 AS #2, #1 AS #3 + Projection: column1 AS #0, column2 AS #1 + Values: (Int32(1), Int32(2)), (Int32(1), Int32(4)), (Int32(2), Int32(1)), (Int32(2), Int32(3)) + SubqueryAlias: __scalar_sq_1 + Projection: max(inner_t.#7) AS #8, inner_t.#6, __always_true + Aggregate: groupBy=[[inner_t.#6, Boolean(true) AS __always_true]], aggr=[[max(inner_t.#7)]] + SubqueryAlias: inner_t + Projection: #4 AS #6, #5 AS #7 + Projection: column1 AS #4, column2 AS #5 + Values: (Int32(1), Int32(4)), (Int32(1), Int32(2)), (Int32(2), Int32(3)), (Int32(2), Int32(1)) + + logical_plan after decorrelate_lateral_join: + SAME TEXT AS ABOVE + + logical_plan after extract_equijoin_predicate: + Sort: #9 ASC NULLS FIRST, #10 ASC NULLS FIRST + Projection: outer_t.#2 AS #9, outer_t.#3 AS #10, __scalar_sq_1.#8 AS mx AS #11 + Left Join: outer_t.#2 = __scalar_sq_1.#6 + SubqueryAlias: outer_t + Projection: #0 AS #2, #1 AS #3 + Projection: column1 AS #0, column2 AS #1 + Values: (Int32(1), Int32(2)), (Int32(1), Int32(4)), (Int32(2), Int32(1)), (Int32(2), Int32(3)) + SubqueryAlias: __scalar_sq_1 + Projection: max(inner_t.#7) AS #8, inner_t.#6, __always_true + Aggregate: groupBy=[[inner_t.#6, Boolean(true) AS __always_true]], aggr=[[max(inner_t.#7)]] + SubqueryAlias: inner_t + Projection: #4 AS #6, #5 AS #7 + Projection: column1 AS #4, column2 AS #5 + Values: (Int32(1), Int32(4)), (Int32(1), Int32(2)), (Int32(2), Int32(3)), (Int32(2), Int32(1)) + + logical_plan after eliminate_duplicated_expr: + SAME TEXT AS ABOVE + + logical_plan after eliminate_filter: + SAME TEXT AS ABOVE + + logical_plan after eliminate_cross_join: + SAME TEXT AS ABOVE + + logical_plan after eliminate_limit: + SAME TEXT AS ABOVE + + logical_plan after propagate_empty_relation: + SAME TEXT AS ABOVE + + logical_plan after filter_null_join_keys: + SAME TEXT AS ABOVE + + logical_plan after eliminate_outer_join: + SAME TEXT AS ABOVE + + logical_plan after push_down_limit: + SAME TEXT AS ABOVE + + logical_plan after push_down_filter: + SAME TEXT AS ABOVE + + logical_plan after single_distinct_aggregation_to_group_by: + SAME TEXT AS ABOVE + + logical_plan after eliminate_group_by_constant: + Sort: #9 ASC NULLS FIRST, #10 ASC NULLS FIRST + Projection: outer_t.#2 AS #9, outer_t.#3 AS #10, __scalar_sq_1.#8 AS mx AS #11 + Left Join: outer_t.#2 = __scalar_sq_1.#6 + SubqueryAlias: outer_t + Projection: #0 AS #2, #1 AS #3 + Projection: column1 AS #0, column2 AS #1 + Values: (Int32(1), Int32(2)), (Int32(1), Int32(4)), (Int32(2), Int32(1)), (Int32(2), Int32(3)) + SubqueryAlias: __scalar_sq_1 + Projection: max(inner_t.#7) AS #8, inner_t.#6, __always_true + Projection: inner_t.#6, Boolean(true) AS __always_true, max(inner_t.#7) + Aggregate: groupBy=[[inner_t.#6]], aggr=[[max(inner_t.#7)]] + SubqueryAlias: inner_t + Projection: #4 AS #6, #5 AS #7 + Projection: column1 AS #4, column2 AS #5 + Values: (Int32(1), Int32(4)), (Int32(1), Int32(2)), (Int32(2), Int32(3)), (Int32(2), Int32(1)) + + logical_plan after common_sub_expression_eliminate: + SAME TEXT AS ABOVE + + logical_plan after extract_leaf_expressions: + SAME TEXT AS ABOVE + + logical_plan after optimize_projections: + Sort: #9 ASC NULLS FIRST, #10 ASC NULLS FIRST + Projection: outer_t.#2 AS #9, outer_t.#3 AS #10, __scalar_sq_1.#8 AS mx AS #11 + Left Join: outer_t.#2 = __scalar_sq_1.#6 + SubqueryAlias: outer_t + Projection: column1 AS #2, column2 AS #3 + Values: (Int32(1), Int32(2)), (Int32(1), Int32(4)), (Int32(2), Int32(1)), (Int32(2), Int32(3)) + SubqueryAlias: __scalar_sq_1 + Projection: max(inner_t.#7) AS #8, inner_t.#6 + Aggregate: groupBy=[[inner_t.#6]], aggr=[[max(inner_t.#7)]] + SubqueryAlias: inner_t + Projection: column1 AS #6, column2 AS #7 + Values: (Int32(1), Int32(4)), (Int32(1), Int32(2)), (Int32(2), Int32(3)), (Int32(2), Int32(1)) + + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_projection: + SAME TEXT AS ABOVE + + logical_plan after rewrite_set_comparison: + SAME TEXT AS ABOVE + + logical_plan after optimize_unions: + SAME TEXT AS ABOVE + + logical_plan after unions_to_filter: + SAME TEXT AS ABOVE + + logical_plan after simplify_expressions: + SAME TEXT AS ABOVE + + logical_plan after replace_distinct_aggregate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_predicate_subquery: + SAME TEXT AS ABOVE + + logical_plan after scalar_subquery_to_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_join: + SAME TEXT AS ABOVE + + logical_plan after extract_equijoin_predicate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_duplicated_expr: + SAME TEXT AS ABOVE + + logical_plan after eliminate_filter: + SAME TEXT AS ABOVE + + logical_plan after eliminate_cross_join: + SAME TEXT AS ABOVE + + logical_plan after eliminate_limit: + SAME TEXT AS ABOVE + + logical_plan after propagate_empty_relation: + SAME TEXT AS ABOVE + + logical_plan after filter_null_join_keys: + SAME TEXT AS ABOVE + + logical_plan after eliminate_outer_join: + SAME TEXT AS ABOVE + + logical_plan after push_down_limit: + SAME TEXT AS ABOVE + + logical_plan after push_down_filter: + SAME TEXT AS ABOVE + + logical_plan after single_distinct_aggregation_to_group_by: + SAME TEXT AS ABOVE + + logical_plan after eliminate_group_by_constant: + SAME TEXT AS ABOVE + + logical_plan after common_sub_expression_eliminate: + SAME TEXT AS ABOVE + + logical_plan after extract_leaf_expressions: + SAME TEXT AS ABOVE + + logical_plan after optimize_projections: + SAME TEXT AS ABOVE + + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE + + logical_plan: + Sort: #9 ASC NULLS FIRST, #10 ASC NULLS FIRST + Projection: outer_t.#2 AS #9, outer_t.#3 AS #10, __scalar_sq_1.#8 AS mx AS #11 + Left Join: outer_t.#2 = __scalar_sq_1.#6 + SubqueryAlias: outer_t + Projection: column1 AS #2, column2 AS #3 + Values: (Int32(1), Int32(2)), (Int32(1), Int32(4)), (Int32(2), Int32(1)), (Int32(2), Int32(3)) + SubqueryAlias: __scalar_sq_1 + Projection: max(inner_t.#7) AS #8, inner_t.#6 + Aggregate: groupBy=[[inner_t.#6]], aggr=[[max(inner_t.#7)]] + SubqueryAlias: inner_t + Projection: column1 AS #6, column2 AS #7 + Values: (Int32(1), Int32(4)), (Int32(1), Int32(2)), (Int32(2), Int32(3)), (Int32(2), Int32(1)) + + initial_physical_plan: + SortExec: expr=[#9@0 ASC, #10@1 ASC], preserve_partitioning=[false] + ProjectionExec: expr=[#2@0 as #9, #3@1 as #10, #8@2 as #11] + HashJoinExec: mode=Auto, join_type=Left, on=[(#2@0, #6@1)] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[max(inner_t.#7)@1 as #8, #6@0 as #6] + AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + ProjectionExec: expr=[column1@0 as #6, column2@1 as #7] + DataSourceExec: partitions=1, partition_sizes=[1] + + + initial_physical_plan_with_stats: + SortExec: expr=[#9@0 ASC, #10@1 ASC], preserve_partitioning=[false], statistics=[Rows=Inexact(4), Bytes=Inexact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0)),(Col[2]:)]] + ProjectionExec: expr=[#2@0 as #9, #3@1 as #10, #8@2 as #11], statistics=[Rows=Inexact(4), Bytes=Inexact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0)),(Col[2]:)]] + HashJoinExec: mode=Auto, join_type=Left, on=[(#2@0, #6@1)], statistics=[Rows=Inexact(4), Bytes=Absent, [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0)),(Col[2]:),(Col[3]:)]] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3], statistics=[Rows=Exact(4), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]] + DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(4), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]] + ProjectionExec: expr=[max(inner_t.#7)@1 as #8, #6@0 as #6], statistics=[Rows=Inexact(4), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]] + AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)], statistics=[Rows=Inexact(4), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]] + AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)], statistics=[Rows=Inexact(4), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]] + ProjectionExec: expr=[column1@0 as #6, column2@1 as #7], statistics=[Rows=Exact(4), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]] + DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(4), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]] + + + initial_physical_plan_with_schema: + SortExec: expr=[#9@0 ASC, #10@1 ASC], preserve_partitioning=[false], schema=[#9:Int32;N, #10:Int32;N, #11:Int32;N] + ProjectionExec: expr=[#2@0 as #9, #3@1 as #10, #8@2 as #11], schema=[#9:Int32;N, #10:Int32;N, #11:Int32;N] + HashJoinExec: mode=Auto, join_type=Left, on=[(#2@0, #6@1)], schema=[#2:Int32;N, #3:Int32;N, #8:Int32;N, #6:Int32;N] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3], schema=[#2:Int32;N, #3:Int32;N] + DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N, column2:Int32;N] + ProjectionExec: expr=[max(inner_t.#7)@1 as #8, #6@0 as #6], schema=[#8:Int32;N, #6:Int32;N] + AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)], schema=[#6:Int32;N, max(inner_t.#7):Int32;N] + AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)], schema=[#6:Int32;N, max(inner_t.#7)[value]:Int32;N] + ProjectionExec: expr=[column1@0 as #6, column2@1 as #7], schema=[#6:Int32;N, #7:Int32;N] + DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N, column2:Int32;N] + + + physical_plan after OutputRequirements: + OutputRequirementExec: order_by=[(#9@0, asc), (#10@1, asc)], dist_by=SinglePartition + SortExec: expr=[#9@0 ASC, #10@1 ASC], preserve_partitioning=[false] + ProjectionExec: expr=[#2@0 as #9, #3@1 as #10, #8@2 as #11] + HashJoinExec: mode=Auto, join_type=Left, on=[(#2@0, #6@1)] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[max(inner_t.#7)@1 as #8, #6@0 as #6] + AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + ProjectionExec: expr=[column1@0 as #6, column2@1 as #7] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after aggregate_statistics: + SAME TEXT AS ABOVE + + physical_plan after join_selection: + OutputRequirementExec: order_by=[(#9@0, asc), (#10@1, asc)], dist_by=SinglePartition + SortExec: expr=[#9@0 ASC, #10@1 ASC], preserve_partitioning=[false] + ProjectionExec: expr=[#2@0 as #9, #3@1 as #10, #8@2 as #11] + HashJoinExec: mode=CollectLeft, join_type=Left, on=[(#2@0, #6@1)] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[max(inner_t.#7)@1 as #8, #6@0 as #6] + AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + ProjectionExec: expr=[column1@0 as #6, column2@1 as #7] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after LimitedDistinctAggregation: + SAME TEXT AS ABOVE + + physical_plan after FilterPushdown: + SAME TEXT AS ABOVE + + physical_plan after EnforceDistribution: + OutputRequirementExec: order_by=[(#9@0, asc), (#10@1, asc)], dist_by=SinglePartition + SortExec: expr=[#9@0 ASC, #10@1 ASC], preserve_partitioning=[false] + CoalescePartitionsExec + ProjectionExec: expr=[#2@0 as #9, #3@1 as #10, #8@2 as #11] + HashJoinExec: mode=CollectLeft, join_type=Left, on=[(#2@0, #6@1)] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[max(inner_t.#7)@1 as #8, #6@0 as #6] + AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + RepartitionExec: partitioning=Hash([#6@0], 4), input_partitions=1 + AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + ProjectionExec: expr=[column1@0 as #6, column2@1 as #7] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after CombinePartialFinalAggregate: + SAME TEXT AS ABOVE + + physical_plan after EnforceSorting: + OutputRequirementExec: order_by=[(#9@0, asc), (#10@1, asc)], dist_by=SinglePartition + SortPreservingMergeExec: [#9@0 ASC, #10@1 ASC] + SortExec: expr=[#9@0 ASC, #10@1 ASC], preserve_partitioning=[true] + ProjectionExec: expr=[#2@0 as #9, #3@1 as #10, #8@2 as #11] + HashJoinExec: mode=CollectLeft, join_type=Left, on=[(#2@0, #6@1)] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[max(inner_t.#7)@1 as #8, #6@0 as #6] + AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + RepartitionExec: partitioning=Hash([#6@0], 4), input_partitions=1 + AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + ProjectionExec: expr=[column1@0 as #6, column2@1 as #7] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after OptimizeAggregateOrder: + SAME TEXT AS ABOVE + + physical_plan after WindowTopN: + SAME TEXT AS ABOVE + + physical_plan after ProjectionPushdown: + OutputRequirementExec: order_by=[(#9@0, asc), (#10@1, asc)], dist_by=SinglePartition + SortPreservingMergeExec: [#9@0 ASC, #10@1 ASC] + SortExec: expr=[#9@0 ASC, #10@1 ASC], preserve_partitioning=[true] + ProjectionExec: expr=[#2@0 as #9, #3@1 as #10, #8@2 as #11] + HashJoinExec: mode=CollectLeft, join_type=Left, on=[(#2@0, #6@1)], projection=[#2@0, #3@1, #8@2] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[max(inner_t.#7)@1 as #8, #6@0 as #6] + AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + RepartitionExec: partitioning=Hash([#6@0], 4), input_partitions=1 + AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + ProjectionExec: expr=[column1@0 as #6, column2@1 as #7] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after OutputRequirements: + SortPreservingMergeExec: [#9@0 ASC, #10@1 ASC] + SortExec: expr=[#9@0 ASC, #10@1 ASC], preserve_partitioning=[true] + ProjectionExec: expr=[#2@0 as #9, #3@1 as #10, #8@2 as #11] + HashJoinExec: mode=CollectLeft, join_type=Left, on=[(#2@0, #6@1)], projection=[#2@0, #3@1, #8@2] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[max(inner_t.#7)@1 as #8, #6@0 as #6] + AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + RepartitionExec: partitioning=Hash([#6@0], 4), input_partitions=1 + AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + ProjectionExec: expr=[column1@0 as #6, column2@1 as #7] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after LimitAggregation: + SAME TEXT AS ABOVE + + physical_plan after LimitPushPastWindows: + SAME TEXT AS ABOVE + + physical_plan after HashJoinBuffering: + SAME TEXT AS ABOVE + + physical_plan after LimitPushdown: + SAME TEXT AS ABOVE + + physical_plan after TopKRepartition: + SAME TEXT AS ABOVE + + physical_plan after ProjectionPushdown: + SAME TEXT AS ABOVE + + physical_plan after PushdownSort: + SAME TEXT AS ABOVE + + physical_plan after EnsureCooperative: + SAME TEXT AS ABOVE + + physical_plan after FilterPushdown(Post): + SAME TEXT AS ABOVE + + physical_plan after RewriteExplicitRepartition: + SAME TEXT AS ABOVE + + physical_plan after RewriteCollectLeftHashJoin: + SAME TEXT AS ABOVE + + physical_plan after EnforceBarrierPartitioning: + SAME TEXT AS ABOVE + + physical_plan after SanityCheckPlan: + SAME TEXT AS ABOVE + + physical_plan: + ProjectionExec: expr=[#9@0 as k, #10@1 as v, #11@2 as mx] + SortPreservingMergeExec: [#9@0 ASC, #10@1 ASC] + SortExec: expr=[#9@0 ASC, #10@1 ASC], preserve_partitioning=[true] + ProjectionExec: expr=[#2@0 as #9, #3@1 as #10, #8@2 as #11] + HashJoinExec: mode=CollectLeft, join_type=Left, on=[(#2@0, #6@1)], projection=[#2@0, #3@1, #8@2] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[max(inner_t.#7)@1 as #8, #6@0 as #6] + AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + RepartitionExec: partitioning=Hash([#6@0], 4), input_partitions=1 + AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + ProjectionExec: expr=[column1@0 as #6, column2@1 as #7] + DataSourceExec: partitions=1, partition_sizes=[1] + + + == Physical Plan == + ProjectionExec: expr=[#9@0 as k, #10@1 as v, #11@2 as mx] + SortPreservingMergeExec: [#9@0 ASC, #10@1 ASC] + SortExec: expr=[#9@0 ASC, #10@1 ASC], preserve_partitioning=[true] ProjectionExec: expr=[#2@0 as #9, #3@1 as #10, #8@2 as #11] HashJoinExec: mode=CollectLeft, join_type=Left, on=[(#2@0, #6@1)], projection=[#2@0, #3@1, #8@2] ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] @@ -181,7 +1146,7 @@ data: partitions=4 distribution=RoundRobin(channels=1) placement=Worker - SortExec: expr=[#9@0 ASC], preserve_partitioning=[true] + SortExec: expr=[#9@0 ASC, #10@1 ASC], preserve_partitioning=[true] ProjectionExec: expr=[#2@0 as #9, #3@1 as #10, #8@2 as #11] HashJoinExec: mode=Partitioned, join_type=Left, on=[(#2@0, #6@1)], projection=[#2@0, #3@1, #8@2] StageInputExec: input=0, partitioning=Hash([#2@0], 4) @@ -194,119 +1159,486 @@ data: distribution=RoundRobin(channels=1) placement=Worker ProjectionExec: expr=[#9@0 as k, #10@1 as v, #11@2 as mx] - SortPreservingMergeExec: [#9@0 ASC] + SortPreservingMergeExec: [#9@0 ASC, #10@1 ASC] StageInputExec: input=0, partitioning=Hash([#9@0], 4) --- -name: "test_correlated_scalar_subquery_with_nested_scalar_in_filter" +name: "test_correlated_scalar_subquery_nomatch_result" data: |- - == Physical Plan == - ProjectionExec: expr=[#12@0 as k, #13@1 as v] - ScalarSubqueryExec: subqueries=1 - SortPreservingMergeExec: [#12@0 ASC] - SortExec: expr=[#12@0 ASC], preserve_partitioning=[true] - ProjectionExec: expr=[#2@0 as #12, #3@1 as #13] - HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(#2@0, #6@1), (#3@1, #11@0)], projection=[#2@0, #3@1] - ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] - DataSourceExec: partitions=1, partition_sizes=[1] - ProjectionExec: expr=[max(inner_t.#7)@1 + scalar_subquery() as #11, #6@0 as #6] - AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] - RepartitionExec: partitioning=Hash([#6@0], 4), input_partitions=1 - AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] - ProjectionExec: expr=[column1@0 as #6, column2@1 as #7] - DataSourceExec: partitions=1, partition_sizes=[1] - ProjectionExec: expr=[min(u.#9)@0 as #10] - AggregateExec: mode=Single, gby=[], aggr=[min(u.#9)] - ProjectionExec: expr=[column1@0 as #9] - DataSourceExec: partitions=1, partition_sizes=[1] + == Codegen == + Whole-stage codegen is not supported; showing physical plan instead. + == Plan Steps == + initial_logical_plan: + Sort: #9 ASC NULLS FIRST + Projection: outer_t.#2 AS #9, outer_t.#3 AS #10, () AS mx AS #11 + Subquery: + Projection: max(inner_t.#7) AS #8 + Aggregate: groupBy=[[]], aggr=[[max(inner_t.#7)]] + Filter: inner_t.#6 = outer_ref(outer_t.#2) + SubqueryAlias: inner_t + Projection: #4 AS #6, #5 AS #7 + Projection: column1 AS #4, column2 AS #5 + Values: (Int32(1), Int32(4)), (Int32(1), Int32(2)) + SubqueryAlias: outer_t + Projection: #0 AS #2, #1 AS #3 + Projection: column1 AS #0, column2 AS #1 + Values: (Int32(1), Int32(2)), (Int32(2), Int32(3)) - == Distributed Plan == - === stage 0 === - inputs=[] - mode=Pipelined - partitions=1 - distribution=RoundRobin(channels=1) - placement=Worker - ProjectionExec: expr=[min(u.#9)@0 as #10] - AggregateExec: mode=Single, gby=[], aggr=[min(u.#9)] - ProjectionExec: expr=[column1@0 as #9] - DataSourceExec: partitions=1, partition_sizes=[1] + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE - === stage 1 === - inputs=[] - mode=Pipelined - partitions=1 - distribution=Hash(keys=[#6@0], channels=4) - placement=Worker - AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] - ProjectionExec: expr=[column1@0 as #6, column2@1 as #7] - DataSourceExec: partitions=1, partition_sizes=[1] + logical_plan after resolve_grouping_function: + SAME TEXT AS ABOVE - === stage 2 === - inputs=[StageInput(stage=1, mode=Shuffle), StageInput(stage=0, mode=Broadcast)] - mode=Pipelined - partitions=4 - distribution=RoundRobin(channels=1) - placement=Worker - ScalarSubqueryExec: subqueries=1 - SortExec: expr=[#12@0 ASC], preserve_partitioning=[true] - ProjectionExec: expr=[#2@0 as #12, #3@1 as #13] - HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(#2@0, #6@1), (#3@1, #11@0)], projection=[#2@0, #3@1] - ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] - DataSourceExec: partitions=1, partition_sizes=[1] - ProjectionExec: expr=[max(inner_t.#7)@1 + scalar_subquery() as #11, #6@0 as #6] - AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] - StageInputExec: input=0, partitioning=Hash([#6@0], 4) - StageInputExec: input=1, partitioning=UnknownPartitioning(1) + logical_plan after type_coercion: + SAME TEXT AS ABOVE - === stage 3 === - inputs=[StageInput(stage=2, mode=Merge)] - mode=Pipelined - partitions=1 - distribution=RoundRobin(channels=1) - placement=Worker - ProjectionExec: expr=[#12@0 as k, #13@1 as v] - SortPreservingMergeExec: [#12@0 ASC] - StageInputExec: input=0, partitioning=Hash([#12@0], 4) ---- -name: "test_correlated_scalar_subquery_with_nested_scalar_in_projection" -data: - |- - == Physical Plan == - ProjectionExec: expr=[#12@0 as k, #13@1 as v, #14@2 as shifted] - ScalarSubqueryExec: subqueries=1 - SortPreservingMergeExec: [#12@0 ASC, #13@1 ASC] - SortExec: expr=[#12@0 ASC, #13@1 ASC], preserve_partitioning=[true] - ProjectionExec: expr=[#2@0 as #12, #3@1 as #13, #11@2 as #14] - HashJoinExec: mode=CollectLeft, join_type=Left, on=[(#2@0, #6@1)], projection=[#2@0, #3@1, #11@2] - ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] - DataSourceExec: partitions=1, partition_sizes=[1] - ProjectionExec: expr=[max(inner_t.#7)@1 + scalar_subquery() as #11, #6@0 as #6] - AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] - RepartitionExec: partitioning=Hash([#6@0], 4), input_partitions=1 - AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] - ProjectionExec: expr=[column1@0 as #6, column2@1 as #7] - DataSourceExec: partitions=1, partition_sizes=[1] - ProjectionExec: expr=[min(u.#9)@0 as #10] - AggregateExec: mode=Single, gby=[], aggr=[min(u.#9)] - ProjectionExec: expr=[column1@0 as #9] - DataSourceExec: partitions=1, partition_sizes=[1] + analyzed_logical_plan: + SAME TEXT AS ABOVE + logical_plan after decorrelate_lateral_projection: + SAME TEXT AS ABOVE - == Distributed Plan == - === stage 0 === - inputs=[] - mode=Pipelined - partitions=1 - distribution=RoundRobin(channels=1) - placement=Worker - ProjectionExec: expr=[min(u.#9)@0 as #10] - AggregateExec: mode=Single, gby=[], aggr=[min(u.#9)] - ProjectionExec: expr=[column1@0 as #9] - DataSourceExec: partitions=1, partition_sizes=[1] + logical_plan after rewrite_set_comparison: + SAME TEXT AS ABOVE - === stage 1 === + logical_plan after optimize_unions: + SAME TEXT AS ABOVE + + logical_plan after unions_to_filter: + SAME TEXT AS ABOVE + + logical_plan after simplify_expressions: + SAME TEXT AS ABOVE + + logical_plan after replace_distinct_aggregate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_predicate_subquery: + SAME TEXT AS ABOVE + + logical_plan after scalar_subquery_to_join: + Sort: #9 ASC NULLS FIRST + Projection: outer_t.#2 AS #9, outer_t.#3 AS #10, __scalar_sq_1.#8 AS mx AS #11 + Left Join: Filter: __scalar_sq_1.#6 = outer_t.#2 + SubqueryAlias: outer_t + Projection: #0 AS #2, #1 AS #3 + Projection: column1 AS #0, column2 AS #1 + Values: (Int32(1), Int32(2)), (Int32(2), Int32(3)) + SubqueryAlias: __scalar_sq_1 + Projection: max(inner_t.#7) AS #8, inner_t.#6, __always_true + Aggregate: groupBy=[[inner_t.#6, Boolean(true) AS __always_true]], aggr=[[max(inner_t.#7)]] + SubqueryAlias: inner_t + Projection: #4 AS #6, #5 AS #7 + Projection: column1 AS #4, column2 AS #5 + Values: (Int32(1), Int32(4)), (Int32(1), Int32(2)) + + logical_plan after decorrelate_lateral_join: + SAME TEXT AS ABOVE + + logical_plan after extract_equijoin_predicate: + Sort: #9 ASC NULLS FIRST + Projection: outer_t.#2 AS #9, outer_t.#3 AS #10, __scalar_sq_1.#8 AS mx AS #11 + Left Join: outer_t.#2 = __scalar_sq_1.#6 + SubqueryAlias: outer_t + Projection: #0 AS #2, #1 AS #3 + Projection: column1 AS #0, column2 AS #1 + Values: (Int32(1), Int32(2)), (Int32(2), Int32(3)) + SubqueryAlias: __scalar_sq_1 + Projection: max(inner_t.#7) AS #8, inner_t.#6, __always_true + Aggregate: groupBy=[[inner_t.#6, Boolean(true) AS __always_true]], aggr=[[max(inner_t.#7)]] + SubqueryAlias: inner_t + Projection: #4 AS #6, #5 AS #7 + Projection: column1 AS #4, column2 AS #5 + Values: (Int32(1), Int32(4)), (Int32(1), Int32(2)) + + logical_plan after eliminate_duplicated_expr: + SAME TEXT AS ABOVE + + logical_plan after eliminate_filter: + SAME TEXT AS ABOVE + + logical_plan after eliminate_cross_join: + SAME TEXT AS ABOVE + + logical_plan after eliminate_limit: + SAME TEXT AS ABOVE + + logical_plan after propagate_empty_relation: + SAME TEXT AS ABOVE + + logical_plan after filter_null_join_keys: + SAME TEXT AS ABOVE + + logical_plan after eliminate_outer_join: + SAME TEXT AS ABOVE + + logical_plan after push_down_limit: + SAME TEXT AS ABOVE + + logical_plan after push_down_filter: + SAME TEXT AS ABOVE + + logical_plan after single_distinct_aggregation_to_group_by: + SAME TEXT AS ABOVE + + logical_plan after eliminate_group_by_constant: + Sort: #9 ASC NULLS FIRST + Projection: outer_t.#2 AS #9, outer_t.#3 AS #10, __scalar_sq_1.#8 AS mx AS #11 + Left Join: outer_t.#2 = __scalar_sq_1.#6 + SubqueryAlias: outer_t + Projection: #0 AS #2, #1 AS #3 + Projection: column1 AS #0, column2 AS #1 + Values: (Int32(1), Int32(2)), (Int32(2), Int32(3)) + SubqueryAlias: __scalar_sq_1 + Projection: max(inner_t.#7) AS #8, inner_t.#6, __always_true + Projection: inner_t.#6, Boolean(true) AS __always_true, max(inner_t.#7) + Aggregate: groupBy=[[inner_t.#6]], aggr=[[max(inner_t.#7)]] + SubqueryAlias: inner_t + Projection: #4 AS #6, #5 AS #7 + Projection: column1 AS #4, column2 AS #5 + Values: (Int32(1), Int32(4)), (Int32(1), Int32(2)) + + logical_plan after common_sub_expression_eliminate: + SAME TEXT AS ABOVE + + logical_plan after extract_leaf_expressions: + SAME TEXT AS ABOVE + + logical_plan after optimize_projections: + Sort: #9 ASC NULLS FIRST + Projection: outer_t.#2 AS #9, outer_t.#3 AS #10, __scalar_sq_1.#8 AS mx AS #11 + Left Join: outer_t.#2 = __scalar_sq_1.#6 + SubqueryAlias: outer_t + Projection: column1 AS #2, column2 AS #3 + Values: (Int32(1), Int32(2)), (Int32(2), Int32(3)) + SubqueryAlias: __scalar_sq_1 + Projection: max(inner_t.#7) AS #8, inner_t.#6 + Aggregate: groupBy=[[inner_t.#6]], aggr=[[max(inner_t.#7)]] + SubqueryAlias: inner_t + Projection: column1 AS #6, column2 AS #7 + Values: (Int32(1), Int32(4)), (Int32(1), Int32(2)) + + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_projection: + SAME TEXT AS ABOVE + + logical_plan after rewrite_set_comparison: + SAME TEXT AS ABOVE + + logical_plan after optimize_unions: + SAME TEXT AS ABOVE + + logical_plan after unions_to_filter: + SAME TEXT AS ABOVE + + logical_plan after simplify_expressions: + SAME TEXT AS ABOVE + + logical_plan after replace_distinct_aggregate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_predicate_subquery: + SAME TEXT AS ABOVE + + logical_plan after scalar_subquery_to_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_join: + SAME TEXT AS ABOVE + + logical_plan after extract_equijoin_predicate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_duplicated_expr: + SAME TEXT AS ABOVE + + logical_plan after eliminate_filter: + SAME TEXT AS ABOVE + + logical_plan after eliminate_cross_join: + SAME TEXT AS ABOVE + + logical_plan after eliminate_limit: + SAME TEXT AS ABOVE + + logical_plan after propagate_empty_relation: + SAME TEXT AS ABOVE + + logical_plan after filter_null_join_keys: + SAME TEXT AS ABOVE + + logical_plan after eliminate_outer_join: + SAME TEXT AS ABOVE + + logical_plan after push_down_limit: + SAME TEXT AS ABOVE + + logical_plan after push_down_filter: + SAME TEXT AS ABOVE + + logical_plan after single_distinct_aggregation_to_group_by: + SAME TEXT AS ABOVE + + logical_plan after eliminate_group_by_constant: + SAME TEXT AS ABOVE + + logical_plan after common_sub_expression_eliminate: + SAME TEXT AS ABOVE + + logical_plan after extract_leaf_expressions: + SAME TEXT AS ABOVE + + logical_plan after optimize_projections: + SAME TEXT AS ABOVE + + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE + + logical_plan: + Sort: #9 ASC NULLS FIRST + Projection: outer_t.#2 AS #9, outer_t.#3 AS #10, __scalar_sq_1.#8 AS mx AS #11 + Left Join: outer_t.#2 = __scalar_sq_1.#6 + SubqueryAlias: outer_t + Projection: column1 AS #2, column2 AS #3 + Values: (Int32(1), Int32(2)), (Int32(2), Int32(3)) + SubqueryAlias: __scalar_sq_1 + Projection: max(inner_t.#7) AS #8, inner_t.#6 + Aggregate: groupBy=[[inner_t.#6]], aggr=[[max(inner_t.#7)]] + SubqueryAlias: inner_t + Projection: column1 AS #6, column2 AS #7 + Values: (Int32(1), Int32(4)), (Int32(1), Int32(2)) + + initial_physical_plan: + SortExec: expr=[#9@0 ASC], preserve_partitioning=[false] + ProjectionExec: expr=[#2@0 as #9, #3@1 as #10, #8@2 as #11] + HashJoinExec: mode=Auto, join_type=Left, on=[(#2@0, #6@1)] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[max(inner_t.#7)@1 as #8, #6@0 as #6] + AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + ProjectionExec: expr=[column1@0 as #6, column2@1 as #7] + DataSourceExec: partitions=1, partition_sizes=[1] + + + initial_physical_plan_with_stats: + SortExec: expr=[#9@0 ASC], preserve_partitioning=[false], statistics=[Rows=Inexact(2), Bytes=Inexact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0)),(Col[2]:)]] + ProjectionExec: expr=[#2@0 as #9, #3@1 as #10, #8@2 as #11], statistics=[Rows=Inexact(2), Bytes=Inexact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0)),(Col[2]:)]] + HashJoinExec: mode=Auto, join_type=Left, on=[(#2@0, #6@1)], statistics=[Rows=Inexact(2), Bytes=Absent, [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0)),(Col[2]:),(Col[3]:)]] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3], statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]] + DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]] + ProjectionExec: expr=[max(inner_t.#7)@1 as #8, #6@0 as #6], statistics=[Rows=Inexact(2), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]] + AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)], statistics=[Rows=Inexact(2), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]] + AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)], statistics=[Rows=Inexact(2), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]] + ProjectionExec: expr=[column1@0 as #6, column2@1 as #7], statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]] + DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]] + + + initial_physical_plan_with_schema: + SortExec: expr=[#9@0 ASC], preserve_partitioning=[false], schema=[#9:Int32;N, #10:Int32;N, #11:Int32;N] + ProjectionExec: expr=[#2@0 as #9, #3@1 as #10, #8@2 as #11], schema=[#9:Int32;N, #10:Int32;N, #11:Int32;N] + HashJoinExec: mode=Auto, join_type=Left, on=[(#2@0, #6@1)], schema=[#2:Int32;N, #3:Int32;N, #8:Int32;N, #6:Int32;N] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3], schema=[#2:Int32;N, #3:Int32;N] + DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N, column2:Int32;N] + ProjectionExec: expr=[max(inner_t.#7)@1 as #8, #6@0 as #6], schema=[#8:Int32;N, #6:Int32;N] + AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)], schema=[#6:Int32;N, max(inner_t.#7):Int32;N] + AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)], schema=[#6:Int32;N, max(inner_t.#7)[value]:Int32;N] + ProjectionExec: expr=[column1@0 as #6, column2@1 as #7], schema=[#6:Int32;N, #7:Int32;N] + DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N, column2:Int32;N] + + + physical_plan after OutputRequirements: + OutputRequirementExec: order_by=[(#9@0, asc)], dist_by=SinglePartition + SortExec: expr=[#9@0 ASC], preserve_partitioning=[false] + ProjectionExec: expr=[#2@0 as #9, #3@1 as #10, #8@2 as #11] + HashJoinExec: mode=Auto, join_type=Left, on=[(#2@0, #6@1)] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[max(inner_t.#7)@1 as #8, #6@0 as #6] + AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + ProjectionExec: expr=[column1@0 as #6, column2@1 as #7] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after aggregate_statistics: + SAME TEXT AS ABOVE + + physical_plan after join_selection: + OutputRequirementExec: order_by=[(#9@0, asc)], dist_by=SinglePartition + SortExec: expr=[#9@0 ASC], preserve_partitioning=[false] + ProjectionExec: expr=[#2@0 as #9, #3@1 as #10, #8@2 as #11] + HashJoinExec: mode=CollectLeft, join_type=Left, on=[(#2@0, #6@1)] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[max(inner_t.#7)@1 as #8, #6@0 as #6] + AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + ProjectionExec: expr=[column1@0 as #6, column2@1 as #7] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after LimitedDistinctAggregation: + SAME TEXT AS ABOVE + + physical_plan after FilterPushdown: + SAME TEXT AS ABOVE + + physical_plan after EnforceDistribution: + OutputRequirementExec: order_by=[(#9@0, asc)], dist_by=SinglePartition + SortExec: expr=[#9@0 ASC], preserve_partitioning=[false] + CoalescePartitionsExec + ProjectionExec: expr=[#2@0 as #9, #3@1 as #10, #8@2 as #11] + HashJoinExec: mode=CollectLeft, join_type=Left, on=[(#2@0, #6@1)] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[max(inner_t.#7)@1 as #8, #6@0 as #6] + AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + RepartitionExec: partitioning=Hash([#6@0], 4), input_partitions=1 + AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + ProjectionExec: expr=[column1@0 as #6, column2@1 as #7] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after CombinePartialFinalAggregate: + SAME TEXT AS ABOVE + + physical_plan after EnforceSorting: + OutputRequirementExec: order_by=[(#9@0, asc)], dist_by=SinglePartition + SortPreservingMergeExec: [#9@0 ASC] + SortExec: expr=[#9@0 ASC], preserve_partitioning=[true] + ProjectionExec: expr=[#2@0 as #9, #3@1 as #10, #8@2 as #11] + HashJoinExec: mode=CollectLeft, join_type=Left, on=[(#2@0, #6@1)] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[max(inner_t.#7)@1 as #8, #6@0 as #6] + AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + RepartitionExec: partitioning=Hash([#6@0], 4), input_partitions=1 + AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + ProjectionExec: expr=[column1@0 as #6, column2@1 as #7] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after OptimizeAggregateOrder: + SAME TEXT AS ABOVE + + physical_plan after WindowTopN: + SAME TEXT AS ABOVE + + physical_plan after ProjectionPushdown: + OutputRequirementExec: order_by=[(#9@0, asc)], dist_by=SinglePartition + SortPreservingMergeExec: [#9@0 ASC] + SortExec: expr=[#9@0 ASC], preserve_partitioning=[true] + ProjectionExec: expr=[#2@0 as #9, #3@1 as #10, #8@2 as #11] + HashJoinExec: mode=CollectLeft, join_type=Left, on=[(#2@0, #6@1)], projection=[#2@0, #3@1, #8@2] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[max(inner_t.#7)@1 as #8, #6@0 as #6] + AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + RepartitionExec: partitioning=Hash([#6@0], 4), input_partitions=1 + AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + ProjectionExec: expr=[column1@0 as #6, column2@1 as #7] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after OutputRequirements: + SortPreservingMergeExec: [#9@0 ASC] + SortExec: expr=[#9@0 ASC], preserve_partitioning=[true] + ProjectionExec: expr=[#2@0 as #9, #3@1 as #10, #8@2 as #11] + HashJoinExec: mode=CollectLeft, join_type=Left, on=[(#2@0, #6@1)], projection=[#2@0, #3@1, #8@2] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[max(inner_t.#7)@1 as #8, #6@0 as #6] + AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + RepartitionExec: partitioning=Hash([#6@0], 4), input_partitions=1 + AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + ProjectionExec: expr=[column1@0 as #6, column2@1 as #7] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after LimitAggregation: + SAME TEXT AS ABOVE + + physical_plan after LimitPushPastWindows: + SAME TEXT AS ABOVE + + physical_plan after HashJoinBuffering: + SAME TEXT AS ABOVE + + physical_plan after LimitPushdown: + SAME TEXT AS ABOVE + + physical_plan after TopKRepartition: + SAME TEXT AS ABOVE + + physical_plan after ProjectionPushdown: + SAME TEXT AS ABOVE + + physical_plan after PushdownSort: + SAME TEXT AS ABOVE + + physical_plan after EnsureCooperative: + SAME TEXT AS ABOVE + + physical_plan after FilterPushdown(Post): + SAME TEXT AS ABOVE + + physical_plan after RewriteExplicitRepartition: + SAME TEXT AS ABOVE + + physical_plan after RewriteCollectLeftHashJoin: + SAME TEXT AS ABOVE + + physical_plan after EnforceBarrierPartitioning: + SAME TEXT AS ABOVE + + physical_plan after SanityCheckPlan: + SAME TEXT AS ABOVE + + physical_plan: + ProjectionExec: expr=[#9@0 as k, #10@1 as v, #11@2 as mx] + SortPreservingMergeExec: [#9@0 ASC] + SortExec: expr=[#9@0 ASC], preserve_partitioning=[true] + ProjectionExec: expr=[#2@0 as #9, #3@1 as #10, #8@2 as #11] + HashJoinExec: mode=CollectLeft, join_type=Left, on=[(#2@0, #6@1)], projection=[#2@0, #3@1, #8@2] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[max(inner_t.#7)@1 as #8, #6@0 as #6] + AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + RepartitionExec: partitioning=Hash([#6@0], 4), input_partitions=1 + AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + ProjectionExec: expr=[column1@0 as #6, column2@1 as #7] + DataSourceExec: partitions=1, partition_sizes=[1] + + + == Physical Plan == + ProjectionExec: expr=[#9@0 as k, #10@1 as v, #11@2 as mx] + SortPreservingMergeExec: [#9@0 ASC] + SortExec: expr=[#9@0 ASC], preserve_partitioning=[true] + ProjectionExec: expr=[#2@0 as #9, #3@1 as #10, #8@2 as #11] + HashJoinExec: mode=CollectLeft, join_type=Left, on=[(#2@0, #6@1)], projection=[#2@0, #3@1, #8@2] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[max(inner_t.#7)@1 as #8, #6@0 as #6] + AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + RepartitionExec: partitioning=Hash([#6@0], 4), input_partitions=1 + AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + ProjectionExec: expr=[column1@0 as #6, column2@1 as #7] + DataSourceExec: partitions=1, partition_sizes=[1] + + + == Distributed Plan == + === stage 0 === inputs=[] mode=Pipelined partitions=1 @@ -315,7 +1647,7 @@ data: ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] DataSourceExec: partitions=1, partition_sizes=[1] - === stage 2 === + === stage 1 === inputs=[] mode=Pipelined partitions=1 @@ -325,165 +1657,7532 @@ data: ProjectionExec: expr=[column1@0 as #6, column2@1 as #7] DataSourceExec: partitions=1, partition_sizes=[1] - === stage 3 === - inputs=[StageInput(stage=2, mode=Shuffle), StageInput(stage=0, mode=Broadcast)] + === stage 2 === + inputs=[StageInput(stage=1, mode=Shuffle)] mode=Pipelined partitions=4 distribution=Hash(keys=[#6@1], channels=4) placement=Worker - ScalarSubqueryExec: subqueries=1 - ProjectionExec: expr=[max(inner_t.#7)@1 + scalar_subquery() as #11, #6@0 as #6] - AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] - StageInputExec: input=0, partitioning=Hash([#6@0], 4) - StageInputExec: input=1, partitioning=UnknownPartitioning(1) + ProjectionExec: expr=[max(inner_t.#7)@1 as #8, #6@0 as #6] + AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + StageInputExec: input=0, partitioning=Hash([#6@0], 4) - === stage 4 === - inputs=[StageInput(stage=1, mode=Shuffle), StageInput(stage=3, mode=Shuffle)] + === stage 3 === + inputs=[StageInput(stage=0, mode=Shuffle), StageInput(stage=2, mode=Shuffle)] mode=Pipelined partitions=4 distribution=RoundRobin(channels=1) placement=Worker - SortExec: expr=[#12@0 ASC, #13@1 ASC], preserve_partitioning=[true] - ProjectionExec: expr=[#2@0 as #12, #3@1 as #13, #11@2 as #14] - HashJoinExec: mode=Partitioned, join_type=Left, on=[(#2@0, #6@1)], projection=[#2@0, #3@1, #11@2] + SortExec: expr=[#9@0 ASC], preserve_partitioning=[true] + ProjectionExec: expr=[#2@0 as #9, #3@1 as #10, #8@2 as #11] + HashJoinExec: mode=Partitioned, join_type=Left, on=[(#2@0, #6@1)], projection=[#2@0, #3@1, #8@2] StageInputExec: input=0, partitioning=Hash([#2@0], 4) StageInputExec: input=1, partitioning=Hash([#6@1], 4) - === stage 5 === - inputs=[StageInput(stage=4, mode=Merge)] + === stage 4 === + inputs=[StageInput(stage=3, mode=Merge)] mode=Pipelined partitions=1 distribution=RoundRobin(channels=1) placement=Worker - ProjectionExec: expr=[#12@0 as k, #13@1 as v, #14@2 as shifted] - SortPreservingMergeExec: [#12@0 ASC, #13@1 ASC] - StageInputExec: input=0, partitioning=Hash([#12@0], 4) + ProjectionExec: expr=[#9@0 as k, #10@1 as v, #11@2 as mx] + SortPreservingMergeExec: [#9@0 ASC] + StageInputExec: input=0, partitioning=Hash([#9@0], 4) --- -name: "test_empty_scalar_subquery_result" +name: "test_correlated_scalar_subquery_with_nested_scalar_in_filter" data: |- - == Physical Plan == - ProjectionExec: expr=[#5@0 as v, #6@1 as shifted] - ScalarSubqueryExec: subqueries=1 - SortExec: expr=[#5@0 ASC], preserve_partitioning=[false] - ProjectionExec: expr=[column1@0 as #5, CAST(column1@0 AS Int64) + scalar_subquery() as #6] - DataSourceExec: partitions=1, partition_sizes=[1] - ProjectionExec: expr=[CAST(#3@0 AS Int64) as #4] - RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 - ProjectionExec: expr=[column1@0 as #3] - FilterExec: column1@0 > 100 - DataSourceExec: partitions=1, partition_sizes=[1] + == Codegen == + Whole-stage codegen is not supported; showing physical plan instead. + == Plan Steps == + initial_logical_plan: + Sort: #12 ASC NULLS FIRST + Projection: outer_t.#2 AS #12, outer_t.#3 AS #13 + Filter: outer_t.#3 = () + Subquery: + Projection: max(inner_t.#7) + () AS #11 + Subquery: + Projection: min(u.#9) AS #10 + Aggregate: groupBy=[[]], aggr=[[min(u.#9)]] + SubqueryAlias: u + Projection: #8 AS #9 + Projection: column1 AS #8 + Values: (Int32(1)) + Aggregate: groupBy=[[]], aggr=[[max(inner_t.#7)]] + Filter: inner_t.#6 = outer_ref(outer_t.#2) + SubqueryAlias: inner_t + Projection: #4 AS #6, #5 AS #7 + Projection: column1 AS #4, column2 AS #5 + Values: (Int32(1), Int32(4)), (Int32(1), Int32(2)), (Int32(2), Int32(3)), (Int32(2), Int32(1)) + SubqueryAlias: outer_t + Projection: #0 AS #2, #1 AS #3 + Projection: column1 AS #0, column2 AS #1 + Values: (Int32(1), Int32(2)), (Int32(1), Int32(5)), (Int32(2), Int32(1)), (Int32(2), Int32(4)) - == Distributed Plan == - === stage 0 === - inputs=[] - mode=Pipelined - partitions=1 - distribution=RoundRobin(channels=4) - placement=Worker - ProjectionExec: expr=[column1@0 as #3] - FilterExec: column1@0 > 100 - DataSourceExec: partitions=1, partition_sizes=[1] + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE - === stage 1 === - inputs=[StageInput(stage=0, mode=Shuffle)] - mode=Pipelined - partitions=4 - distribution=RoundRobin(channels=1) - placement=Worker - ProjectionExec: expr=[CAST(#3@0 AS Int64) as #4] - StageInputExec: input=0, partitioning=RoundRobinBatch(4) + logical_plan after resolve_grouping_function: + SAME TEXT AS ABOVE - === stage 2 === - inputs=[StageInput(stage=1, mode=Broadcast)] - mode=Pipelined - partitions=1 - distribution=RoundRobin(channels=1) - placement=Worker - ProjectionExec: expr=[#5@0 as v, #6@1 as shifted] - ScalarSubqueryExec: subqueries=1 - SortExec: expr=[#5@0 ASC], preserve_partitioning=[false] - ProjectionExec: expr=[column1@0 as #5, CAST(column1@0 AS Int64) + scalar_subquery() as #6] - DataSourceExec: partitions=1, partition_sizes=[1] - StageInputExec: input=0, partitioning=UnknownPartitioning(1) + logical_plan after type_coercion: + SAME TEXT AS ABOVE + + analyzed_logical_plan: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_projection: + SAME TEXT AS ABOVE + + logical_plan after rewrite_set_comparison: + SAME TEXT AS ABOVE + + logical_plan after optimize_unions: + SAME TEXT AS ABOVE + + logical_plan after unions_to_filter: + SAME TEXT AS ABOVE + + logical_plan after simplify_expressions: + SAME TEXT AS ABOVE + + logical_plan after replace_distinct_aggregate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_predicate_subquery: + SAME TEXT AS ABOVE + + logical_plan after scalar_subquery_to_join: + Sort: #12 ASC NULLS FIRST + Projection: outer_t.#2 AS #12, outer_t.#3 AS #13 + Projection: outer_t.#2, outer_t.#3 + Filter: outer_t.#3 = __scalar_sq_1.#11 + Left Join: Filter: __scalar_sq_1.#6 = outer_t.#2 + SubqueryAlias: outer_t + Projection: #0 AS #2, #1 AS #3 + Projection: column1 AS #0, column2 AS #1 + Values: (Int32(1), Int32(2)), (Int32(1), Int32(5)), (Int32(2), Int32(1)), (Int32(2), Int32(4)) + SubqueryAlias: __scalar_sq_1 + Projection: max(inner_t.#7) + () AS #11, inner_t.#6, __always_true + Subquery: + Projection: min(u.#9) AS #10 + Aggregate: groupBy=[[]], aggr=[[min(u.#9)]] + SubqueryAlias: u + Projection: #8 AS #9 + Projection: column1 AS #8 + Values: (Int32(1)) + Aggregate: groupBy=[[inner_t.#6, Boolean(true) AS __always_true]], aggr=[[max(inner_t.#7)]] + SubqueryAlias: inner_t + Projection: #4 AS #6, #5 AS #7 + Projection: column1 AS #4, column2 AS #5 + Values: (Int32(1), Int32(4)), (Int32(1), Int32(2)), (Int32(2), Int32(3)), (Int32(2), Int32(1)) + + logical_plan after decorrelate_lateral_join: + SAME TEXT AS ABOVE + + logical_plan after extract_equijoin_predicate: + Sort: #12 ASC NULLS FIRST + Projection: outer_t.#2 AS #12, outer_t.#3 AS #13 + Projection: outer_t.#2, outer_t.#3 + Filter: outer_t.#3 = __scalar_sq_1.#11 + Left Join: outer_t.#2 = __scalar_sq_1.#6 + SubqueryAlias: outer_t + Projection: #0 AS #2, #1 AS #3 + Projection: column1 AS #0, column2 AS #1 + Values: (Int32(1), Int32(2)), (Int32(1), Int32(5)), (Int32(2), Int32(1)), (Int32(2), Int32(4)) + SubqueryAlias: __scalar_sq_1 + Projection: max(inner_t.#7) + () AS #11, inner_t.#6, __always_true + Subquery: + Projection: min(u.#9) AS #10 + Aggregate: groupBy=[[]], aggr=[[min(u.#9)]] + SubqueryAlias: u + Projection: #8 AS #9 + Projection: column1 AS #8 + Values: (Int32(1)) + Aggregate: groupBy=[[inner_t.#6, Boolean(true) AS __always_true]], aggr=[[max(inner_t.#7)]] + SubqueryAlias: inner_t + Projection: #4 AS #6, #5 AS #7 + Projection: column1 AS #4, column2 AS #5 + Values: (Int32(1), Int32(4)), (Int32(1), Int32(2)), (Int32(2), Int32(3)), (Int32(2), Int32(1)) + + logical_plan after eliminate_duplicated_expr: + SAME TEXT AS ABOVE + + logical_plan after eliminate_filter: + SAME TEXT AS ABOVE + + logical_plan after eliminate_cross_join: + SAME TEXT AS ABOVE + + logical_plan after eliminate_limit: + SAME TEXT AS ABOVE + + logical_plan after propagate_empty_relation: + SAME TEXT AS ABOVE + + logical_plan after filter_null_join_keys: + SAME TEXT AS ABOVE + + logical_plan after eliminate_outer_join: + Sort: #12 ASC NULLS FIRST + Projection: outer_t.#2 AS #12, outer_t.#3 AS #13 + Projection: outer_t.#2, outer_t.#3 + Filter: outer_t.#3 = __scalar_sq_1.#11 + Inner Join: outer_t.#2 = __scalar_sq_1.#6 + SubqueryAlias: outer_t + Projection: #0 AS #2, #1 AS #3 + Projection: column1 AS #0, column2 AS #1 + Values: (Int32(1), Int32(2)), (Int32(1), Int32(5)), (Int32(2), Int32(1)), (Int32(2), Int32(4)) + SubqueryAlias: __scalar_sq_1 + Projection: max(inner_t.#7) + () AS #11, inner_t.#6, __always_true + Subquery: + Projection: min(u.#9) AS #10 + Aggregate: groupBy=[[]], aggr=[[min(u.#9)]] + SubqueryAlias: u + Projection: #8 AS #9 + Projection: column1 AS #8 + Values: (Int32(1)) + Aggregate: groupBy=[[inner_t.#6, Boolean(true) AS __always_true]], aggr=[[max(inner_t.#7)]] + SubqueryAlias: inner_t + Projection: #4 AS #6, #5 AS #7 + Projection: column1 AS #4, column2 AS #5 + Values: (Int32(1), Int32(4)), (Int32(1), Int32(2)), (Int32(2), Int32(3)), (Int32(2), Int32(1)) + + logical_plan after push_down_limit: + SAME TEXT AS ABOVE + + logical_plan after push_down_filter: + Sort: #12 ASC NULLS FIRST + Projection: outer_t.#2 AS #12, outer_t.#3 AS #13 + Projection: outer_t.#2, outer_t.#3 + Inner Join: outer_t.#2 = __scalar_sq_1.#6 Filter: outer_t.#3 = __scalar_sq_1.#11 + SubqueryAlias: outer_t + Projection: #0 AS #2, #1 AS #3 + Projection: column1 AS #0, column2 AS #1 + Values: (Int32(1), Int32(2)), (Int32(1), Int32(5)), (Int32(2), Int32(1)), (Int32(2), Int32(4)) + SubqueryAlias: __scalar_sq_1 + Projection: max(inner_t.#7) + () AS #11, inner_t.#6, __always_true + Subquery: + Projection: min(u.#9) AS #10 + Aggregate: groupBy=[[]], aggr=[[min(u.#9)]] + SubqueryAlias: u + Projection: #8 AS #9 + Projection: column1 AS #8 + Values: (Int32(1)) + Aggregate: groupBy=[[inner_t.#6, Boolean(true) AS __always_true]], aggr=[[max(inner_t.#7)]] + SubqueryAlias: inner_t + Projection: #4 AS #6, #5 AS #7 + Projection: column1 AS #4, column2 AS #5 + Values: (Int32(1), Int32(4)), (Int32(1), Int32(2)), (Int32(2), Int32(3)), (Int32(2), Int32(1)) + + logical_plan after single_distinct_aggregation_to_group_by: + SAME TEXT AS ABOVE + + logical_plan after eliminate_group_by_constant: + Sort: #12 ASC NULLS FIRST + Projection: outer_t.#2 AS #12, outer_t.#3 AS #13 + Projection: outer_t.#2, outer_t.#3 + Inner Join: outer_t.#2 = __scalar_sq_1.#6 Filter: outer_t.#3 = __scalar_sq_1.#11 + SubqueryAlias: outer_t + Projection: #0 AS #2, #1 AS #3 + Projection: column1 AS #0, column2 AS #1 + Values: (Int32(1), Int32(2)), (Int32(1), Int32(5)), (Int32(2), Int32(1)), (Int32(2), Int32(4)) + SubqueryAlias: __scalar_sq_1 + Projection: max(inner_t.#7) + () AS #11, inner_t.#6, __always_true + Subquery: + Projection: min(u.#9) AS #10 + Aggregate: groupBy=[[]], aggr=[[min(u.#9)]] + SubqueryAlias: u + Projection: #8 AS #9 + Projection: column1 AS #8 + Values: (Int32(1)) + Projection: inner_t.#6, Boolean(true) AS __always_true, max(inner_t.#7) + Aggregate: groupBy=[[inner_t.#6]], aggr=[[max(inner_t.#7)]] + SubqueryAlias: inner_t + Projection: #4 AS #6, #5 AS #7 + Projection: column1 AS #4, column2 AS #5 + Values: (Int32(1), Int32(4)), (Int32(1), Int32(2)), (Int32(2), Int32(3)), (Int32(2), Int32(1)) + + logical_plan after common_sub_expression_eliminate: + SAME TEXT AS ABOVE + + logical_plan after extract_leaf_expressions: + SAME TEXT AS ABOVE + + logical_plan after optimize_projections: + Sort: #12 ASC NULLS FIRST + Projection: outer_t.#2 AS #12, outer_t.#3 AS #13 + Inner Join: outer_t.#2 = __scalar_sq_1.#6 Filter: outer_t.#3 = __scalar_sq_1.#11 + SubqueryAlias: outer_t + Projection: column1 AS #2, column2 AS #3 + Values: (Int32(1), Int32(2)), (Int32(1), Int32(5)), (Int32(2), Int32(1)), (Int32(2), Int32(4)) + SubqueryAlias: __scalar_sq_1 + Projection: max(inner_t.#7) + () AS #11, inner_t.#6 + Subquery: + Projection: min(u.#9) AS #10 + Aggregate: groupBy=[[]], aggr=[[min(u.#9)]] + SubqueryAlias: u + Projection: column1 AS #9 + Values: (Int32(1)) + Aggregate: groupBy=[[inner_t.#6]], aggr=[[max(inner_t.#7)]] + SubqueryAlias: inner_t + Projection: column1 AS #6, column2 AS #7 + Values: (Int32(1), Int32(4)), (Int32(1), Int32(2)), (Int32(2), Int32(3)), (Int32(2), Int32(1)) + + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_projection: + SAME TEXT AS ABOVE + + logical_plan after rewrite_set_comparison: + SAME TEXT AS ABOVE + + logical_plan after optimize_unions: + SAME TEXT AS ABOVE + + logical_plan after unions_to_filter: + SAME TEXT AS ABOVE + + logical_plan after simplify_expressions: + SAME TEXT AS ABOVE + + logical_plan after replace_distinct_aggregate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_predicate_subquery: + SAME TEXT AS ABOVE + + logical_plan after scalar_subquery_to_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_join: + SAME TEXT AS ABOVE + + logical_plan after extract_equijoin_predicate: + Sort: #12 ASC NULLS FIRST + Projection: outer_t.#2 AS #12, outer_t.#3 AS #13 + Inner Join: outer_t.#2 = __scalar_sq_1.#6, outer_t.#3 = __scalar_sq_1.#11 + SubqueryAlias: outer_t + Projection: column1 AS #2, column2 AS #3 + Values: (Int32(1), Int32(2)), (Int32(1), Int32(5)), (Int32(2), Int32(1)), (Int32(2), Int32(4)) + SubqueryAlias: __scalar_sq_1 + Projection: max(inner_t.#7) + () AS #11, inner_t.#6 + Subquery: + Projection: min(u.#9) AS #10 + Aggregate: groupBy=[[]], aggr=[[min(u.#9)]] + SubqueryAlias: u + Projection: column1 AS #9 + Values: (Int32(1)) + Aggregate: groupBy=[[inner_t.#6]], aggr=[[max(inner_t.#7)]] + SubqueryAlias: inner_t + Projection: column1 AS #6, column2 AS #7 + Values: (Int32(1), Int32(4)), (Int32(1), Int32(2)), (Int32(2), Int32(3)), (Int32(2), Int32(1)) + + logical_plan after eliminate_duplicated_expr: + SAME TEXT AS ABOVE + + logical_plan after eliminate_filter: + SAME TEXT AS ABOVE + + logical_plan after eliminate_cross_join: + SAME TEXT AS ABOVE + + logical_plan after eliminate_limit: + SAME TEXT AS ABOVE + + logical_plan after propagate_empty_relation: + SAME TEXT AS ABOVE + + logical_plan after filter_null_join_keys: + SAME TEXT AS ABOVE + + logical_plan after eliminate_outer_join: + SAME TEXT AS ABOVE + + logical_plan after push_down_limit: + SAME TEXT AS ABOVE + + logical_plan after push_down_filter: + SAME TEXT AS ABOVE + + logical_plan after single_distinct_aggregation_to_group_by: + SAME TEXT AS ABOVE + + logical_plan after eliminate_group_by_constant: + SAME TEXT AS ABOVE + + logical_plan after common_sub_expression_eliminate: + SAME TEXT AS ABOVE + + logical_plan after extract_leaf_expressions: + SAME TEXT AS ABOVE + + logical_plan after optimize_projections: + SAME TEXT AS ABOVE + + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_projection: + SAME TEXT AS ABOVE + + logical_plan after rewrite_set_comparison: + SAME TEXT AS ABOVE + + logical_plan after optimize_unions: + SAME TEXT AS ABOVE + + logical_plan after unions_to_filter: + SAME TEXT AS ABOVE + + logical_plan after simplify_expressions: + SAME TEXT AS ABOVE + + logical_plan after replace_distinct_aggregate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_predicate_subquery: + SAME TEXT AS ABOVE + + logical_plan after scalar_subquery_to_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_join: + SAME TEXT AS ABOVE + + logical_plan after extract_equijoin_predicate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_duplicated_expr: + SAME TEXT AS ABOVE + + logical_plan after eliminate_filter: + SAME TEXT AS ABOVE + + logical_plan after eliminate_cross_join: + SAME TEXT AS ABOVE + + logical_plan after eliminate_limit: + SAME TEXT AS ABOVE + + logical_plan after propagate_empty_relation: + SAME TEXT AS ABOVE + + logical_plan after filter_null_join_keys: + SAME TEXT AS ABOVE + + logical_plan after eliminate_outer_join: + SAME TEXT AS ABOVE + + logical_plan after push_down_limit: + SAME TEXT AS ABOVE + + logical_plan after push_down_filter: + SAME TEXT AS ABOVE + + logical_plan after single_distinct_aggregation_to_group_by: + SAME TEXT AS ABOVE + + logical_plan after eliminate_group_by_constant: + SAME TEXT AS ABOVE + + logical_plan after common_sub_expression_eliminate: + SAME TEXT AS ABOVE + + logical_plan after extract_leaf_expressions: + SAME TEXT AS ABOVE + + logical_plan after optimize_projections: + SAME TEXT AS ABOVE + + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE + + logical_plan: + Sort: #12 ASC NULLS FIRST + Projection: outer_t.#2 AS #12, outer_t.#3 AS #13 + Inner Join: outer_t.#2 = __scalar_sq_1.#6, outer_t.#3 = __scalar_sq_1.#11 + SubqueryAlias: outer_t + Projection: column1 AS #2, column2 AS #3 + Values: (Int32(1), Int32(2)), (Int32(1), Int32(5)), (Int32(2), Int32(1)), (Int32(2), Int32(4)) + SubqueryAlias: __scalar_sq_1 + Projection: max(inner_t.#7) + () AS #11, inner_t.#6 + Subquery: + Projection: min(u.#9) AS #10 + Aggregate: groupBy=[[]], aggr=[[min(u.#9)]] + SubqueryAlias: u + Projection: column1 AS #9 + Values: (Int32(1)) + Aggregate: groupBy=[[inner_t.#6]], aggr=[[max(inner_t.#7)]] + SubqueryAlias: inner_t + Projection: column1 AS #6, column2 AS #7 + Values: (Int32(1), Int32(4)), (Int32(1), Int32(2)), (Int32(2), Int32(3)), (Int32(2), Int32(1)) + + initial_physical_plan: + ScalarSubqueryExec: subqueries=1 + SortExec: expr=[#12@0 ASC], preserve_partitioning=[false] + ProjectionExec: expr=[#2@0 as #12, #3@1 as #13] + HashJoinExec: mode=Auto, join_type=Inner, on=[(#2@0, #6@1), (#3@1, #11@0)] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[max(inner_t.#7)@1 + scalar_subquery() as #11, #6@0 as #6] + AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + ProjectionExec: expr=[column1@0 as #6, column2@1 as #7] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(u.#9)@0 as #10] + AggregateExec: mode=Final, gby=[], aggr=[min(u.#9)] + AggregateExec: mode=Partial, gby=[], aggr=[min(u.#9)] + ProjectionExec: expr=[column1@0 as #9] + DataSourceExec: partitions=1, partition_sizes=[1] + + + initial_physical_plan_with_stats: + ScalarSubqueryExec: subqueries=1, statistics=[Rows=Inexact(4), Bytes=Inexact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]] + SortExec: expr=[#12@0 ASC], preserve_partitioning=[false], statistics=[Rows=Inexact(4), Bytes=Inexact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]] + ProjectionExec: expr=[#2@0 as #12, #3@1 as #13], statistics=[Rows=Inexact(4), Bytes=Inexact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]] + HashJoinExec: mode=Auto, join_type=Inner, on=[(#2@0, #6@1), (#3@1, #11@0)], statistics=[Rows=Inexact(4), Bytes=Absent, [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0)),(Col[2]:),(Col[3]:)]] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3], statistics=[Rows=Exact(4), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]] + DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(4), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]] + ProjectionExec: expr=[max(inner_t.#7)@1 + scalar_subquery() as #11, #6@0 as #6], statistics=[Rows=Inexact(4), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]] + AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)], statistics=[Rows=Inexact(4), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]] + AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)], statistics=[Rows=Inexact(4), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]] + ProjectionExec: expr=[column1@0 as #6, column2@1 as #7], statistics=[Rows=Exact(4), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]] + DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(4), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]] + ProjectionExec: expr=[min(u.#9)@0 as #10], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]:)]] + AggregateExec: mode=Final, gby=[], aggr=[min(u.#9)], statistics=[Rows=Exact(1), Bytes=Absent, [(Col[0]:)]] + AggregateExec: mode=Partial, gby=[], aggr=[min(u.#9)], statistics=[Rows=Exact(0), Bytes=Inexact(), [(Col[0]:)]] + ProjectionExec: expr=[column1@0 as #9], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]: Null=Exact(0))]] + DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]: Null=Exact(0))]] + + + initial_physical_plan_with_schema: + ScalarSubqueryExec: subqueries=1, schema=[#12:Int32;N, #13:Int32;N] + SortExec: expr=[#12@0 ASC], preserve_partitioning=[false], schema=[#12:Int32;N, #13:Int32;N] + ProjectionExec: expr=[#2@0 as #12, #3@1 as #13], schema=[#12:Int32;N, #13:Int32;N] + HashJoinExec: mode=Auto, join_type=Inner, on=[(#2@0, #6@1), (#3@1, #11@0)], schema=[#2:Int32;N, #3:Int32;N, #11:Int32;N, #6:Int32;N] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3], schema=[#2:Int32;N, #3:Int32;N] + DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N, column2:Int32;N] + ProjectionExec: expr=[max(inner_t.#7)@1 + scalar_subquery() as #11, #6@0 as #6], schema=[#11:Int32;N, #6:Int32;N] + AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)], schema=[#6:Int32;N, max(inner_t.#7):Int32;N] + AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)], schema=[#6:Int32;N, max(inner_t.#7)[value]:Int32;N] + ProjectionExec: expr=[column1@0 as #6, column2@1 as #7], schema=[#6:Int32;N, #7:Int32;N] + DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N, column2:Int32;N] + ProjectionExec: expr=[min(u.#9)@0 as #10], schema=[#10:Int32;N] + AggregateExec: mode=Final, gby=[], aggr=[min(u.#9)], schema=[min(u.#9):Int32;N] + AggregateExec: mode=Partial, gby=[], aggr=[min(u.#9)], schema=[min(u.#9)[value]:Int32;N] + ProjectionExec: expr=[column1@0 as #9], schema=[#9:Int32;N] + DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N] + + + physical_plan after OutputRequirements: + OutputRequirementExec: order_by=[], dist_by=Unspecified + ScalarSubqueryExec: subqueries=1 + SortExec: expr=[#12@0 ASC], preserve_partitioning=[false] + ProjectionExec: expr=[#2@0 as #12, #3@1 as #13] + HashJoinExec: mode=Auto, join_type=Inner, on=[(#2@0, #6@1), (#3@1, #11@0)] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[max(inner_t.#7)@1 + scalar_subquery() as #11, #6@0 as #6] + AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + ProjectionExec: expr=[column1@0 as #6, column2@1 as #7] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(u.#9)@0 as #10] + AggregateExec: mode=Final, gby=[], aggr=[min(u.#9)] + AggregateExec: mode=Partial, gby=[], aggr=[min(u.#9)] + ProjectionExec: expr=[column1@0 as #9] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after aggregate_statistics: + SAME TEXT AS ABOVE + + physical_plan after join_selection: + OutputRequirementExec: order_by=[], dist_by=Unspecified + ScalarSubqueryExec: subqueries=1 + SortExec: expr=[#12@0 ASC], preserve_partitioning=[false] + ProjectionExec: expr=[#2@0 as #12, #3@1 as #13] + HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(#2@0, #6@1), (#3@1, #11@0)] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[max(inner_t.#7)@1 + scalar_subquery() as #11, #6@0 as #6] + AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + ProjectionExec: expr=[column1@0 as #6, column2@1 as #7] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(u.#9)@0 as #10] + AggregateExec: mode=Final, gby=[], aggr=[min(u.#9)] + AggregateExec: mode=Partial, gby=[], aggr=[min(u.#9)] + ProjectionExec: expr=[column1@0 as #9] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after LimitedDistinctAggregation: + SAME TEXT AS ABOVE + + physical_plan after FilterPushdown: + SAME TEXT AS ABOVE + + physical_plan after EnforceDistribution: + OutputRequirementExec: order_by=[], dist_by=Unspecified + ScalarSubqueryExec: subqueries=1 + SortExec: expr=[#12@0 ASC], preserve_partitioning=[false] + CoalescePartitionsExec + ProjectionExec: expr=[#2@0 as #12, #3@1 as #13] + HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(#2@0, #6@1), (#3@1, #11@0)] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[max(inner_t.#7)@1 + scalar_subquery() as #11, #6@0 as #6] + AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + RepartitionExec: partitioning=Hash([#6@0], 4), input_partitions=1 + AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + ProjectionExec: expr=[column1@0 as #6, column2@1 as #7] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(u.#9)@0 as #10] + AggregateExec: mode=Final, gby=[], aggr=[min(u.#9)] + AggregateExec: mode=Partial, gby=[], aggr=[min(u.#9)] + ProjectionExec: expr=[column1@0 as #9] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after CombinePartialFinalAggregate: + OutputRequirementExec: order_by=[], dist_by=Unspecified + ScalarSubqueryExec: subqueries=1 + SortExec: expr=[#12@0 ASC], preserve_partitioning=[false] + CoalescePartitionsExec + ProjectionExec: expr=[#2@0 as #12, #3@1 as #13] + HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(#2@0, #6@1), (#3@1, #11@0)] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[max(inner_t.#7)@1 + scalar_subquery() as #11, #6@0 as #6] + AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + RepartitionExec: partitioning=Hash([#6@0], 4), input_partitions=1 + AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + ProjectionExec: expr=[column1@0 as #6, column2@1 as #7] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(u.#9)@0 as #10] + AggregateExec: mode=Single, gby=[], aggr=[min(u.#9)] + ProjectionExec: expr=[column1@0 as #9] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after EnforceSorting: + OutputRequirementExec: order_by=[], dist_by=Unspecified + ScalarSubqueryExec: subqueries=1 + SortPreservingMergeExec: [#12@0 ASC] + SortExec: expr=[#12@0 ASC], preserve_partitioning=[true] + ProjectionExec: expr=[#2@0 as #12, #3@1 as #13] + HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(#2@0, #6@1), (#3@1, #11@0)] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[max(inner_t.#7)@1 + scalar_subquery() as #11, #6@0 as #6] + AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + RepartitionExec: partitioning=Hash([#6@0], 4), input_partitions=1 + AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + ProjectionExec: expr=[column1@0 as #6, column2@1 as #7] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(u.#9)@0 as #10] + AggregateExec: mode=Single, gby=[], aggr=[min(u.#9)] + ProjectionExec: expr=[column1@0 as #9] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after OptimizeAggregateOrder: + SAME TEXT AS ABOVE + + physical_plan after WindowTopN: + SAME TEXT AS ABOVE + + physical_plan after ProjectionPushdown: + OutputRequirementExec: order_by=[], dist_by=Unspecified + ScalarSubqueryExec: subqueries=1 + SortPreservingMergeExec: [#12@0 ASC] + SortExec: expr=[#12@0 ASC], preserve_partitioning=[true] + ProjectionExec: expr=[#2@0 as #12, #3@1 as #13] + HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(#2@0, #6@1), (#3@1, #11@0)], projection=[#2@0, #3@1] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[max(inner_t.#7)@1 + scalar_subquery() as #11, #6@0 as #6] + AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + RepartitionExec: partitioning=Hash([#6@0], 4), input_partitions=1 + AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + ProjectionExec: expr=[column1@0 as #6, column2@1 as #7] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(u.#9)@0 as #10] + AggregateExec: mode=Single, gby=[], aggr=[min(u.#9)] + ProjectionExec: expr=[column1@0 as #9] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after OutputRequirements: + ScalarSubqueryExec: subqueries=1 + SortPreservingMergeExec: [#12@0 ASC] + SortExec: expr=[#12@0 ASC], preserve_partitioning=[true] + ProjectionExec: expr=[#2@0 as #12, #3@1 as #13] + HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(#2@0, #6@1), (#3@1, #11@0)], projection=[#2@0, #3@1] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[max(inner_t.#7)@1 + scalar_subquery() as #11, #6@0 as #6] + AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + RepartitionExec: partitioning=Hash([#6@0], 4), input_partitions=1 + AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + ProjectionExec: expr=[column1@0 as #6, column2@1 as #7] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(u.#9)@0 as #10] + AggregateExec: mode=Single, gby=[], aggr=[min(u.#9)] + ProjectionExec: expr=[column1@0 as #9] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after LimitAggregation: + SAME TEXT AS ABOVE + + physical_plan after LimitPushPastWindows: + SAME TEXT AS ABOVE + + physical_plan after HashJoinBuffering: + SAME TEXT AS ABOVE + + physical_plan after LimitPushdown: + SAME TEXT AS ABOVE + + physical_plan after TopKRepartition: + SAME TEXT AS ABOVE + + physical_plan after ProjectionPushdown: + SAME TEXT AS ABOVE + + physical_plan after PushdownSort: + SAME TEXT AS ABOVE + + physical_plan after EnsureCooperative: + SAME TEXT AS ABOVE + + physical_plan after FilterPushdown(Post): + SAME TEXT AS ABOVE + + physical_plan after RewriteExplicitRepartition: + SAME TEXT AS ABOVE + + physical_plan after RewriteCollectLeftHashJoin: + SAME TEXT AS ABOVE + + physical_plan after EnforceBarrierPartitioning: + SAME TEXT AS ABOVE + + physical_plan after SanityCheckPlan: + SAME TEXT AS ABOVE + + physical_plan: + ProjectionExec: expr=[#12@0 as k, #13@1 as v] + ScalarSubqueryExec: subqueries=1 + SortPreservingMergeExec: [#12@0 ASC] + SortExec: expr=[#12@0 ASC], preserve_partitioning=[true] + ProjectionExec: expr=[#2@0 as #12, #3@1 as #13] + HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(#2@0, #6@1), (#3@1, #11@0)], projection=[#2@0, #3@1] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[max(inner_t.#7)@1 + scalar_subquery() as #11, #6@0 as #6] + AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + RepartitionExec: partitioning=Hash([#6@0], 4), input_partitions=1 + AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + ProjectionExec: expr=[column1@0 as #6, column2@1 as #7] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(u.#9)@0 as #10] + AggregateExec: mode=Single, gby=[], aggr=[min(u.#9)] + ProjectionExec: expr=[column1@0 as #9] + DataSourceExec: partitions=1, partition_sizes=[1] + + + == Physical Plan == + ProjectionExec: expr=[#12@0 as k, #13@1 as v] + ScalarSubqueryExec: subqueries=1 + SortPreservingMergeExec: [#12@0 ASC] + SortExec: expr=[#12@0 ASC], preserve_partitioning=[true] + ProjectionExec: expr=[#2@0 as #12, #3@1 as #13] + HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(#2@0, #6@1), (#3@1, #11@0)], projection=[#2@0, #3@1] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[max(inner_t.#7)@1 + scalar_subquery() as #11, #6@0 as #6] + AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + RepartitionExec: partitioning=Hash([#6@0], 4), input_partitions=1 + AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + ProjectionExec: expr=[column1@0 as #6, column2@1 as #7] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(u.#9)@0 as #10] + AggregateExec: mode=Single, gby=[], aggr=[min(u.#9)] + ProjectionExec: expr=[column1@0 as #9] + DataSourceExec: partitions=1, partition_sizes=[1] + + + == Distributed Plan == + === stage 0 === + inputs=[] + mode=Pipelined + partitions=1 + distribution=RoundRobin(channels=1) + placement=Worker + ProjectionExec: expr=[min(u.#9)@0 as #10] + AggregateExec: mode=Single, gby=[], aggr=[min(u.#9)] + ProjectionExec: expr=[column1@0 as #9] + DataSourceExec: partitions=1, partition_sizes=[1] + + === stage 1 === + inputs=[] + mode=Pipelined + partitions=1 + distribution=Hash(keys=[#6@0], channels=4) + placement=Worker + AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + ProjectionExec: expr=[column1@0 as #6, column2@1 as #7] + DataSourceExec: partitions=1, partition_sizes=[1] + + === stage 2 === + inputs=[StageInput(stage=1, mode=Shuffle), StageInput(stage=0, mode=Broadcast)] + mode=Pipelined + partitions=4 + distribution=RoundRobin(channels=1) + placement=Worker + ScalarSubqueryExec: subqueries=1 + SortExec: expr=[#12@0 ASC], preserve_partitioning=[true] + ProjectionExec: expr=[#2@0 as #12, #3@1 as #13] + HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(#2@0, #6@1), (#3@1, #11@0)], projection=[#2@0, #3@1] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[max(inner_t.#7)@1 + scalar_subquery() as #11, #6@0 as #6] + AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + StageInputExec: input=0, partitioning=Hash([#6@0], 4) + StageInputExec: input=1, partitioning=UnknownPartitioning(1) + + === stage 3 === + inputs=[StageInput(stage=2, mode=Merge)] + mode=Pipelined + partitions=1 + distribution=RoundRobin(channels=1) + placement=Worker + ProjectionExec: expr=[#12@0 as k, #13@1 as v] + SortPreservingMergeExec: [#12@0 ASC] + StageInputExec: input=0, partitioning=Hash([#12@0], 4) +--- +name: "test_correlated_scalar_subquery_with_nested_scalar_in_projection" +data: + |- + == Codegen == + Whole-stage codegen is not supported; showing physical plan instead. + + == Plan Steps == + initial_logical_plan: + Sort: #12 ASC NULLS FIRST, #13 ASC NULLS FIRST + Projection: outer_t.#2 AS #12, outer_t.#3 AS #13, () AS shifted AS #14 + Subquery: + Projection: max(inner_t.#7) + () AS #11 + Subquery: + Projection: min(u.#9) AS #10 + Aggregate: groupBy=[[]], aggr=[[min(u.#9)]] + SubqueryAlias: u + Projection: #8 AS #9 + Projection: column1 AS #8 + Values: (Int32(1)) + Aggregate: groupBy=[[]], aggr=[[max(inner_t.#7)]] + Filter: inner_t.#6 = outer_ref(outer_t.#2) + SubqueryAlias: inner_t + Projection: #4 AS #6, #5 AS #7 + Projection: column1 AS #4, column2 AS #5 + Values: (Int32(1), Int32(4)), (Int32(1), Int32(2)), (Int32(2), Int32(3)), (Int32(2), Int32(1)) + SubqueryAlias: outer_t + Projection: #0 AS #2, #1 AS #3 + Projection: column1 AS #0, column2 AS #1 + Values: (Int32(1), Int32(2)), (Int32(1), Int32(4)), (Int32(2), Int32(1)), (Int32(2), Int32(3)) + + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE + + logical_plan after resolve_grouping_function: + SAME TEXT AS ABOVE + + logical_plan after type_coercion: + SAME TEXT AS ABOVE + + analyzed_logical_plan: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_projection: + SAME TEXT AS ABOVE + + logical_plan after rewrite_set_comparison: + SAME TEXT AS ABOVE + + logical_plan after optimize_unions: + SAME TEXT AS ABOVE + + logical_plan after unions_to_filter: + SAME TEXT AS ABOVE + + logical_plan after simplify_expressions: + SAME TEXT AS ABOVE + + logical_plan after replace_distinct_aggregate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_predicate_subquery: + SAME TEXT AS ABOVE + + logical_plan after scalar_subquery_to_join: + Sort: #12 ASC NULLS FIRST, #13 ASC NULLS FIRST + Projection: outer_t.#2 AS #12, outer_t.#3 AS #13, __scalar_sq_1.#11 AS shifted AS #14 + Left Join: Filter: __scalar_sq_1.#6 = outer_t.#2 + SubqueryAlias: outer_t + Projection: #0 AS #2, #1 AS #3 + Projection: column1 AS #0, column2 AS #1 + Values: (Int32(1), Int32(2)), (Int32(1), Int32(4)), (Int32(2), Int32(1)), (Int32(2), Int32(3)) + SubqueryAlias: __scalar_sq_1 + Projection: max(inner_t.#7) + () AS #11, inner_t.#6, __always_true + Subquery: + Projection: min(u.#9) AS #10 + Aggregate: groupBy=[[]], aggr=[[min(u.#9)]] + SubqueryAlias: u + Projection: #8 AS #9 + Projection: column1 AS #8 + Values: (Int32(1)) + Aggregate: groupBy=[[inner_t.#6, Boolean(true) AS __always_true]], aggr=[[max(inner_t.#7)]] + SubqueryAlias: inner_t + Projection: #4 AS #6, #5 AS #7 + Projection: column1 AS #4, column2 AS #5 + Values: (Int32(1), Int32(4)), (Int32(1), Int32(2)), (Int32(2), Int32(3)), (Int32(2), Int32(1)) + + logical_plan after decorrelate_lateral_join: + SAME TEXT AS ABOVE + + logical_plan after extract_equijoin_predicate: + Sort: #12 ASC NULLS FIRST, #13 ASC NULLS FIRST + Projection: outer_t.#2 AS #12, outer_t.#3 AS #13, __scalar_sq_1.#11 AS shifted AS #14 + Left Join: outer_t.#2 = __scalar_sq_1.#6 + SubqueryAlias: outer_t + Projection: #0 AS #2, #1 AS #3 + Projection: column1 AS #0, column2 AS #1 + Values: (Int32(1), Int32(2)), (Int32(1), Int32(4)), (Int32(2), Int32(1)), (Int32(2), Int32(3)) + SubqueryAlias: __scalar_sq_1 + Projection: max(inner_t.#7) + () AS #11, inner_t.#6, __always_true + Subquery: + Projection: min(u.#9) AS #10 + Aggregate: groupBy=[[]], aggr=[[min(u.#9)]] + SubqueryAlias: u + Projection: #8 AS #9 + Projection: column1 AS #8 + Values: (Int32(1)) + Aggregate: groupBy=[[inner_t.#6, Boolean(true) AS __always_true]], aggr=[[max(inner_t.#7)]] + SubqueryAlias: inner_t + Projection: #4 AS #6, #5 AS #7 + Projection: column1 AS #4, column2 AS #5 + Values: (Int32(1), Int32(4)), (Int32(1), Int32(2)), (Int32(2), Int32(3)), (Int32(2), Int32(1)) + + logical_plan after eliminate_duplicated_expr: + SAME TEXT AS ABOVE + + logical_plan after eliminate_filter: + SAME TEXT AS ABOVE + + logical_plan after eliminate_cross_join: + SAME TEXT AS ABOVE + + logical_plan after eliminate_limit: + SAME TEXT AS ABOVE + + logical_plan after propagate_empty_relation: + SAME TEXT AS ABOVE + + logical_plan after filter_null_join_keys: + SAME TEXT AS ABOVE + + logical_plan after eliminate_outer_join: + SAME TEXT AS ABOVE + + logical_plan after push_down_limit: + SAME TEXT AS ABOVE + + logical_plan after push_down_filter: + SAME TEXT AS ABOVE + + logical_plan after single_distinct_aggregation_to_group_by: + SAME TEXT AS ABOVE + + logical_plan after eliminate_group_by_constant: + Sort: #12 ASC NULLS FIRST, #13 ASC NULLS FIRST + Projection: outer_t.#2 AS #12, outer_t.#3 AS #13, __scalar_sq_1.#11 AS shifted AS #14 + Left Join: outer_t.#2 = __scalar_sq_1.#6 + SubqueryAlias: outer_t + Projection: #0 AS #2, #1 AS #3 + Projection: column1 AS #0, column2 AS #1 + Values: (Int32(1), Int32(2)), (Int32(1), Int32(4)), (Int32(2), Int32(1)), (Int32(2), Int32(3)) + SubqueryAlias: __scalar_sq_1 + Projection: max(inner_t.#7) + () AS #11, inner_t.#6, __always_true + Subquery: + Projection: min(u.#9) AS #10 + Aggregate: groupBy=[[]], aggr=[[min(u.#9)]] + SubqueryAlias: u + Projection: #8 AS #9 + Projection: column1 AS #8 + Values: (Int32(1)) + Projection: inner_t.#6, Boolean(true) AS __always_true, max(inner_t.#7) + Aggregate: groupBy=[[inner_t.#6]], aggr=[[max(inner_t.#7)]] + SubqueryAlias: inner_t + Projection: #4 AS #6, #5 AS #7 + Projection: column1 AS #4, column2 AS #5 + Values: (Int32(1), Int32(4)), (Int32(1), Int32(2)), (Int32(2), Int32(3)), (Int32(2), Int32(1)) + + logical_plan after common_sub_expression_eliminate: + SAME TEXT AS ABOVE + + logical_plan after extract_leaf_expressions: + SAME TEXT AS ABOVE + + logical_plan after optimize_projections: + Sort: #12 ASC NULLS FIRST, #13 ASC NULLS FIRST + Projection: outer_t.#2 AS #12, outer_t.#3 AS #13, __scalar_sq_1.#11 AS shifted AS #14 + Left Join: outer_t.#2 = __scalar_sq_1.#6 + SubqueryAlias: outer_t + Projection: column1 AS #2, column2 AS #3 + Values: (Int32(1), Int32(2)), (Int32(1), Int32(4)), (Int32(2), Int32(1)), (Int32(2), Int32(3)) + SubqueryAlias: __scalar_sq_1 + Projection: max(inner_t.#7) + () AS #11, inner_t.#6 + Subquery: + Projection: min(u.#9) AS #10 + Aggregate: groupBy=[[]], aggr=[[min(u.#9)]] + SubqueryAlias: u + Projection: column1 AS #9 + Values: (Int32(1)) + Aggregate: groupBy=[[inner_t.#6]], aggr=[[max(inner_t.#7)]] + SubqueryAlias: inner_t + Projection: column1 AS #6, column2 AS #7 + Values: (Int32(1), Int32(4)), (Int32(1), Int32(2)), (Int32(2), Int32(3)), (Int32(2), Int32(1)) + + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_projection: + SAME TEXT AS ABOVE + + logical_plan after rewrite_set_comparison: + SAME TEXT AS ABOVE + + logical_plan after optimize_unions: + SAME TEXT AS ABOVE + + logical_plan after unions_to_filter: + SAME TEXT AS ABOVE + + logical_plan after simplify_expressions: + SAME TEXT AS ABOVE + + logical_plan after replace_distinct_aggregate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_predicate_subquery: + SAME TEXT AS ABOVE + + logical_plan after scalar_subquery_to_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_join: + SAME TEXT AS ABOVE + + logical_plan after extract_equijoin_predicate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_duplicated_expr: + SAME TEXT AS ABOVE + + logical_plan after eliminate_filter: + SAME TEXT AS ABOVE + + logical_plan after eliminate_cross_join: + SAME TEXT AS ABOVE + + logical_plan after eliminate_limit: + SAME TEXT AS ABOVE + + logical_plan after propagate_empty_relation: + SAME TEXT AS ABOVE + + logical_plan after filter_null_join_keys: + SAME TEXT AS ABOVE + + logical_plan after eliminate_outer_join: + SAME TEXT AS ABOVE + + logical_plan after push_down_limit: + SAME TEXT AS ABOVE + + logical_plan after push_down_filter: + SAME TEXT AS ABOVE + + logical_plan after single_distinct_aggregation_to_group_by: + SAME TEXT AS ABOVE + + logical_plan after eliminate_group_by_constant: + SAME TEXT AS ABOVE + + logical_plan after common_sub_expression_eliminate: + SAME TEXT AS ABOVE + + logical_plan after extract_leaf_expressions: + SAME TEXT AS ABOVE + + logical_plan after optimize_projections: + SAME TEXT AS ABOVE + + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE + + logical_plan: + Sort: #12 ASC NULLS FIRST, #13 ASC NULLS FIRST + Projection: outer_t.#2 AS #12, outer_t.#3 AS #13, __scalar_sq_1.#11 AS shifted AS #14 + Left Join: outer_t.#2 = __scalar_sq_1.#6 + SubqueryAlias: outer_t + Projection: column1 AS #2, column2 AS #3 + Values: (Int32(1), Int32(2)), (Int32(1), Int32(4)), (Int32(2), Int32(1)), (Int32(2), Int32(3)) + SubqueryAlias: __scalar_sq_1 + Projection: max(inner_t.#7) + () AS #11, inner_t.#6 + Subquery: + Projection: min(u.#9) AS #10 + Aggregate: groupBy=[[]], aggr=[[min(u.#9)]] + SubqueryAlias: u + Projection: column1 AS #9 + Values: (Int32(1)) + Aggregate: groupBy=[[inner_t.#6]], aggr=[[max(inner_t.#7)]] + SubqueryAlias: inner_t + Projection: column1 AS #6, column2 AS #7 + Values: (Int32(1), Int32(4)), (Int32(1), Int32(2)), (Int32(2), Int32(3)), (Int32(2), Int32(1)) + + initial_physical_plan: + ScalarSubqueryExec: subqueries=1 + SortExec: expr=[#12@0 ASC, #13@1 ASC], preserve_partitioning=[false] + ProjectionExec: expr=[#2@0 as #12, #3@1 as #13, #11@2 as #14] + HashJoinExec: mode=Auto, join_type=Left, on=[(#2@0, #6@1)] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[max(inner_t.#7)@1 + scalar_subquery() as #11, #6@0 as #6] + AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + ProjectionExec: expr=[column1@0 as #6, column2@1 as #7] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(u.#9)@0 as #10] + AggregateExec: mode=Final, gby=[], aggr=[min(u.#9)] + AggregateExec: mode=Partial, gby=[], aggr=[min(u.#9)] + ProjectionExec: expr=[column1@0 as #9] + DataSourceExec: partitions=1, partition_sizes=[1] + + + initial_physical_plan_with_stats: + ScalarSubqueryExec: subqueries=1, statistics=[Rows=Inexact(4), Bytes=Inexact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0)),(Col[2]:)]] + SortExec: expr=[#12@0 ASC, #13@1 ASC], preserve_partitioning=[false], statistics=[Rows=Inexact(4), Bytes=Inexact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0)),(Col[2]:)]] + ProjectionExec: expr=[#2@0 as #12, #3@1 as #13, #11@2 as #14], statistics=[Rows=Inexact(4), Bytes=Inexact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0)),(Col[2]:)]] + HashJoinExec: mode=Auto, join_type=Left, on=[(#2@0, #6@1)], statistics=[Rows=Inexact(4), Bytes=Absent, [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0)),(Col[2]:),(Col[3]:)]] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3], statistics=[Rows=Exact(4), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]] + DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(4), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]] + ProjectionExec: expr=[max(inner_t.#7)@1 + scalar_subquery() as #11, #6@0 as #6], statistics=[Rows=Inexact(4), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]] + AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)], statistics=[Rows=Inexact(4), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]] + AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)], statistics=[Rows=Inexact(4), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]] + ProjectionExec: expr=[column1@0 as #6, column2@1 as #7], statistics=[Rows=Exact(4), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]] + DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(4), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]] + ProjectionExec: expr=[min(u.#9)@0 as #10], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]:)]] + AggregateExec: mode=Final, gby=[], aggr=[min(u.#9)], statistics=[Rows=Exact(1), Bytes=Absent, [(Col[0]:)]] + AggregateExec: mode=Partial, gby=[], aggr=[min(u.#9)], statistics=[Rows=Exact(0), Bytes=Inexact(), [(Col[0]:)]] + ProjectionExec: expr=[column1@0 as #9], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]: Null=Exact(0))]] + DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]: Null=Exact(0))]] + + + initial_physical_plan_with_schema: + ScalarSubqueryExec: subqueries=1, schema=[#12:Int32;N, #13:Int32;N, #14:Int32;N] + SortExec: expr=[#12@0 ASC, #13@1 ASC], preserve_partitioning=[false], schema=[#12:Int32;N, #13:Int32;N, #14:Int32;N] + ProjectionExec: expr=[#2@0 as #12, #3@1 as #13, #11@2 as #14], schema=[#12:Int32;N, #13:Int32;N, #14:Int32;N] + HashJoinExec: mode=Auto, join_type=Left, on=[(#2@0, #6@1)], schema=[#2:Int32;N, #3:Int32;N, #11:Int32;N, #6:Int32;N] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3], schema=[#2:Int32;N, #3:Int32;N] + DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N, column2:Int32;N] + ProjectionExec: expr=[max(inner_t.#7)@1 + scalar_subquery() as #11, #6@0 as #6], schema=[#11:Int32;N, #6:Int32;N] + AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)], schema=[#6:Int32;N, max(inner_t.#7):Int32;N] + AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)], schema=[#6:Int32;N, max(inner_t.#7)[value]:Int32;N] + ProjectionExec: expr=[column1@0 as #6, column2@1 as #7], schema=[#6:Int32;N, #7:Int32;N] + DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N, column2:Int32;N] + ProjectionExec: expr=[min(u.#9)@0 as #10], schema=[#10:Int32;N] + AggregateExec: mode=Final, gby=[], aggr=[min(u.#9)], schema=[min(u.#9):Int32;N] + AggregateExec: mode=Partial, gby=[], aggr=[min(u.#9)], schema=[min(u.#9)[value]:Int32;N] + ProjectionExec: expr=[column1@0 as #9], schema=[#9:Int32;N] + DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N] + + + physical_plan after OutputRequirements: + OutputRequirementExec: order_by=[], dist_by=Unspecified + ScalarSubqueryExec: subqueries=1 + SortExec: expr=[#12@0 ASC, #13@1 ASC], preserve_partitioning=[false] + ProjectionExec: expr=[#2@0 as #12, #3@1 as #13, #11@2 as #14] + HashJoinExec: mode=Auto, join_type=Left, on=[(#2@0, #6@1)] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[max(inner_t.#7)@1 + scalar_subquery() as #11, #6@0 as #6] + AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + ProjectionExec: expr=[column1@0 as #6, column2@1 as #7] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(u.#9)@0 as #10] + AggregateExec: mode=Final, gby=[], aggr=[min(u.#9)] + AggregateExec: mode=Partial, gby=[], aggr=[min(u.#9)] + ProjectionExec: expr=[column1@0 as #9] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after aggregate_statistics: + SAME TEXT AS ABOVE + + physical_plan after join_selection: + OutputRequirementExec: order_by=[], dist_by=Unspecified + ScalarSubqueryExec: subqueries=1 + SortExec: expr=[#12@0 ASC, #13@1 ASC], preserve_partitioning=[false] + ProjectionExec: expr=[#2@0 as #12, #3@1 as #13, #11@2 as #14] + HashJoinExec: mode=CollectLeft, join_type=Left, on=[(#2@0, #6@1)] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[max(inner_t.#7)@1 + scalar_subquery() as #11, #6@0 as #6] + AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + ProjectionExec: expr=[column1@0 as #6, column2@1 as #7] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(u.#9)@0 as #10] + AggregateExec: mode=Final, gby=[], aggr=[min(u.#9)] + AggregateExec: mode=Partial, gby=[], aggr=[min(u.#9)] + ProjectionExec: expr=[column1@0 as #9] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after LimitedDistinctAggregation: + SAME TEXT AS ABOVE + + physical_plan after FilterPushdown: + SAME TEXT AS ABOVE + + physical_plan after EnforceDistribution: + OutputRequirementExec: order_by=[], dist_by=Unspecified + ScalarSubqueryExec: subqueries=1 + SortExec: expr=[#12@0 ASC, #13@1 ASC], preserve_partitioning=[false] + CoalescePartitionsExec + ProjectionExec: expr=[#2@0 as #12, #3@1 as #13, #11@2 as #14] + HashJoinExec: mode=CollectLeft, join_type=Left, on=[(#2@0, #6@1)] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[max(inner_t.#7)@1 + scalar_subquery() as #11, #6@0 as #6] + AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + RepartitionExec: partitioning=Hash([#6@0], 4), input_partitions=1 + AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + ProjectionExec: expr=[column1@0 as #6, column2@1 as #7] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(u.#9)@0 as #10] + AggregateExec: mode=Final, gby=[], aggr=[min(u.#9)] + AggregateExec: mode=Partial, gby=[], aggr=[min(u.#9)] + ProjectionExec: expr=[column1@0 as #9] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after CombinePartialFinalAggregate: + OutputRequirementExec: order_by=[], dist_by=Unspecified + ScalarSubqueryExec: subqueries=1 + SortExec: expr=[#12@0 ASC, #13@1 ASC], preserve_partitioning=[false] + CoalescePartitionsExec + ProjectionExec: expr=[#2@0 as #12, #3@1 as #13, #11@2 as #14] + HashJoinExec: mode=CollectLeft, join_type=Left, on=[(#2@0, #6@1)] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[max(inner_t.#7)@1 + scalar_subquery() as #11, #6@0 as #6] + AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + RepartitionExec: partitioning=Hash([#6@0], 4), input_partitions=1 + AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + ProjectionExec: expr=[column1@0 as #6, column2@1 as #7] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(u.#9)@0 as #10] + AggregateExec: mode=Single, gby=[], aggr=[min(u.#9)] + ProjectionExec: expr=[column1@0 as #9] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after EnforceSorting: + OutputRequirementExec: order_by=[], dist_by=Unspecified + ScalarSubqueryExec: subqueries=1 + SortPreservingMergeExec: [#12@0 ASC, #13@1 ASC] + SortExec: expr=[#12@0 ASC, #13@1 ASC], preserve_partitioning=[true] + ProjectionExec: expr=[#2@0 as #12, #3@1 as #13, #11@2 as #14] + HashJoinExec: mode=CollectLeft, join_type=Left, on=[(#2@0, #6@1)] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[max(inner_t.#7)@1 + scalar_subquery() as #11, #6@0 as #6] + AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + RepartitionExec: partitioning=Hash([#6@0], 4), input_partitions=1 + AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + ProjectionExec: expr=[column1@0 as #6, column2@1 as #7] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(u.#9)@0 as #10] + AggregateExec: mode=Single, gby=[], aggr=[min(u.#9)] + ProjectionExec: expr=[column1@0 as #9] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after OptimizeAggregateOrder: + SAME TEXT AS ABOVE + + physical_plan after WindowTopN: + SAME TEXT AS ABOVE + + physical_plan after ProjectionPushdown: + OutputRequirementExec: order_by=[], dist_by=Unspecified + ScalarSubqueryExec: subqueries=1 + SortPreservingMergeExec: [#12@0 ASC, #13@1 ASC] + SortExec: expr=[#12@0 ASC, #13@1 ASC], preserve_partitioning=[true] + ProjectionExec: expr=[#2@0 as #12, #3@1 as #13, #11@2 as #14] + HashJoinExec: mode=CollectLeft, join_type=Left, on=[(#2@0, #6@1)], projection=[#2@0, #3@1, #11@2] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[max(inner_t.#7)@1 + scalar_subquery() as #11, #6@0 as #6] + AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + RepartitionExec: partitioning=Hash([#6@0], 4), input_partitions=1 + AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + ProjectionExec: expr=[column1@0 as #6, column2@1 as #7] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(u.#9)@0 as #10] + AggregateExec: mode=Single, gby=[], aggr=[min(u.#9)] + ProjectionExec: expr=[column1@0 as #9] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after OutputRequirements: + ScalarSubqueryExec: subqueries=1 + SortPreservingMergeExec: [#12@0 ASC, #13@1 ASC] + SortExec: expr=[#12@0 ASC, #13@1 ASC], preserve_partitioning=[true] + ProjectionExec: expr=[#2@0 as #12, #3@1 as #13, #11@2 as #14] + HashJoinExec: mode=CollectLeft, join_type=Left, on=[(#2@0, #6@1)], projection=[#2@0, #3@1, #11@2] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[max(inner_t.#7)@1 + scalar_subquery() as #11, #6@0 as #6] + AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + RepartitionExec: partitioning=Hash([#6@0], 4), input_partitions=1 + AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + ProjectionExec: expr=[column1@0 as #6, column2@1 as #7] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(u.#9)@0 as #10] + AggregateExec: mode=Single, gby=[], aggr=[min(u.#9)] + ProjectionExec: expr=[column1@0 as #9] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after LimitAggregation: + SAME TEXT AS ABOVE + + physical_plan after LimitPushPastWindows: + SAME TEXT AS ABOVE + + physical_plan after HashJoinBuffering: + SAME TEXT AS ABOVE + + physical_plan after LimitPushdown: + SAME TEXT AS ABOVE + + physical_plan after TopKRepartition: + SAME TEXT AS ABOVE + + physical_plan after ProjectionPushdown: + SAME TEXT AS ABOVE + + physical_plan after PushdownSort: + SAME TEXT AS ABOVE + + physical_plan after EnsureCooperative: + SAME TEXT AS ABOVE + + physical_plan after FilterPushdown(Post): + SAME TEXT AS ABOVE + + physical_plan after RewriteExplicitRepartition: + SAME TEXT AS ABOVE + + physical_plan after RewriteCollectLeftHashJoin: + SAME TEXT AS ABOVE + + physical_plan after EnforceBarrierPartitioning: + SAME TEXT AS ABOVE + + physical_plan after SanityCheckPlan: + SAME TEXT AS ABOVE + + physical_plan: + ProjectionExec: expr=[#12@0 as k, #13@1 as v, #14@2 as shifted] + ScalarSubqueryExec: subqueries=1 + SortPreservingMergeExec: [#12@0 ASC, #13@1 ASC] + SortExec: expr=[#12@0 ASC, #13@1 ASC], preserve_partitioning=[true] + ProjectionExec: expr=[#2@0 as #12, #3@1 as #13, #11@2 as #14] + HashJoinExec: mode=CollectLeft, join_type=Left, on=[(#2@0, #6@1)], projection=[#2@0, #3@1, #11@2] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[max(inner_t.#7)@1 + scalar_subquery() as #11, #6@0 as #6] + AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + RepartitionExec: partitioning=Hash([#6@0], 4), input_partitions=1 + AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + ProjectionExec: expr=[column1@0 as #6, column2@1 as #7] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(u.#9)@0 as #10] + AggregateExec: mode=Single, gby=[], aggr=[min(u.#9)] + ProjectionExec: expr=[column1@0 as #9] + DataSourceExec: partitions=1, partition_sizes=[1] + + + == Physical Plan == + ProjectionExec: expr=[#12@0 as k, #13@1 as v, #14@2 as shifted] + ScalarSubqueryExec: subqueries=1 + SortPreservingMergeExec: [#12@0 ASC, #13@1 ASC] + SortExec: expr=[#12@0 ASC, #13@1 ASC], preserve_partitioning=[true] + ProjectionExec: expr=[#2@0 as #12, #3@1 as #13, #11@2 as #14] + HashJoinExec: mode=CollectLeft, join_type=Left, on=[(#2@0, #6@1)], projection=[#2@0, #3@1, #11@2] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[max(inner_t.#7)@1 + scalar_subquery() as #11, #6@0 as #6] + AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + RepartitionExec: partitioning=Hash([#6@0], 4), input_partitions=1 + AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + ProjectionExec: expr=[column1@0 as #6, column2@1 as #7] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(u.#9)@0 as #10] + AggregateExec: mode=Single, gby=[], aggr=[min(u.#9)] + ProjectionExec: expr=[column1@0 as #9] + DataSourceExec: partitions=1, partition_sizes=[1] + + + == Distributed Plan == + === stage 0 === + inputs=[] + mode=Pipelined + partitions=1 + distribution=RoundRobin(channels=1) + placement=Worker + ProjectionExec: expr=[min(u.#9)@0 as #10] + AggregateExec: mode=Single, gby=[], aggr=[min(u.#9)] + ProjectionExec: expr=[column1@0 as #9] + DataSourceExec: partitions=1, partition_sizes=[1] + + === stage 1 === + inputs=[] + mode=Pipelined + partitions=1 + distribution=Hash(keys=[#2@0], channels=4) + placement=Worker + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + + === stage 2 === + inputs=[] + mode=Pipelined + partitions=1 + distribution=Hash(keys=[#6@0], channels=4) + placement=Worker + AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + ProjectionExec: expr=[column1@0 as #6, column2@1 as #7] + DataSourceExec: partitions=1, partition_sizes=[1] + + === stage 3 === + inputs=[StageInput(stage=2, mode=Shuffle), StageInput(stage=0, mode=Broadcast)] + mode=Pipelined + partitions=4 + distribution=Hash(keys=[#6@1], channels=4) + placement=Worker + ScalarSubqueryExec: subqueries=1 + ProjectionExec: expr=[max(inner_t.#7)@1 + scalar_subquery() as #11, #6@0 as #6] + AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)] + StageInputExec: input=0, partitioning=Hash([#6@0], 4) + StageInputExec: input=1, partitioning=UnknownPartitioning(1) + + === stage 4 === + inputs=[StageInput(stage=1, mode=Shuffle), StageInput(stage=3, mode=Shuffle)] + mode=Pipelined + partitions=4 + distribution=RoundRobin(channels=1) + placement=Worker + SortExec: expr=[#12@0 ASC, #13@1 ASC], preserve_partitioning=[true] + ProjectionExec: expr=[#2@0 as #12, #3@1 as #13, #11@2 as #14] + HashJoinExec: mode=Partitioned, join_type=Left, on=[(#2@0, #6@1)], projection=[#2@0, #3@1, #11@2] + StageInputExec: input=0, partitioning=Hash([#2@0], 4) + StageInputExec: input=1, partitioning=Hash([#6@1], 4) + + === stage 5 === + inputs=[StageInput(stage=4, mode=Merge)] + mode=Pipelined + partitions=1 + distribution=RoundRobin(channels=1) + placement=Worker + ProjectionExec: expr=[#12@0 as k, #13@1 as v, #14@2 as shifted] + SortPreservingMergeExec: [#12@0 ASC, #13@1 ASC] + StageInputExec: input=0, partitioning=Hash([#12@0], 4) +--- +name: "test_empty_scalar_subquery_result" +data: + |- + == Codegen == + Whole-stage codegen is not supported; showing physical plan instead. + + == Plan Steps == + initial_logical_plan: + Sort: #5 ASC NULLS FIRST + Projection: t.#1 AS #5, CAST(t.#1 AS Int64) + () AS shifted AS #6 + Subquery: + Projection: CAST(s.#3 AS Int64) AS #4 + Filter: s.#3 > Int32(100) + SubqueryAlias: s + Projection: #2 AS #3 + Projection: column1 AS #2 + Values: (Int32(10)) + SubqueryAlias: t + Projection: #0 AS #1 + Projection: column1 AS #0 + Values: (Int32(1)), (Int32(2)) + + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE + + logical_plan after resolve_grouping_function: + SAME TEXT AS ABOVE + + logical_plan after type_coercion: + SAME TEXT AS ABOVE + + analyzed_logical_plan: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_projection: + SAME TEXT AS ABOVE + + logical_plan after rewrite_set_comparison: + SAME TEXT AS ABOVE + + logical_plan after optimize_unions: + SAME TEXT AS ABOVE + + logical_plan after unions_to_filter: + SAME TEXT AS ABOVE + + logical_plan after simplify_expressions: + SAME TEXT AS ABOVE + + logical_plan after replace_distinct_aggregate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_predicate_subquery: + SAME TEXT AS ABOVE + + logical_plan after scalar_subquery_to_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_join: + SAME TEXT AS ABOVE + + logical_plan after extract_equijoin_predicate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_duplicated_expr: + SAME TEXT AS ABOVE + + logical_plan after eliminate_filter: + SAME TEXT AS ABOVE + + logical_plan after eliminate_cross_join: + SAME TEXT AS ABOVE + + logical_plan after eliminate_limit: + SAME TEXT AS ABOVE + + logical_plan after propagate_empty_relation: + SAME TEXT AS ABOVE + + logical_plan after filter_null_join_keys: + SAME TEXT AS ABOVE + + logical_plan after eliminate_outer_join: + SAME TEXT AS ABOVE + + logical_plan after push_down_limit: + SAME TEXT AS ABOVE + + logical_plan after push_down_filter: + Sort: #5 ASC NULLS FIRST + Projection: t.#1 AS #5, CAST(t.#1 AS Int64) + () AS shifted AS #6 + Subquery: + Projection: CAST(s.#3 AS Int64) AS #4 + SubqueryAlias: s + Projection: #2 AS #3 + Projection: column1 AS #2 + Filter: column1 > Int32(100) + Values: (Int32(10)) + SubqueryAlias: t + Projection: #0 AS #1 + Projection: column1 AS #0 + Values: (Int32(1)), (Int32(2)) + + logical_plan after single_distinct_aggregation_to_group_by: + SAME TEXT AS ABOVE + + logical_plan after eliminate_group_by_constant: + SAME TEXT AS ABOVE + + logical_plan after common_sub_expression_eliminate: + SAME TEXT AS ABOVE + + logical_plan after extract_leaf_expressions: + SAME TEXT AS ABOVE + + logical_plan after optimize_projections: + Sort: #5 ASC NULLS FIRST + Projection: t.#1 AS #5, CAST(t.#1 AS Int64) + () AS shifted AS #6 + Subquery: + Projection: CAST(s.#3 AS Int64) AS #4 + SubqueryAlias: s + Projection: column1 AS #3 + Filter: column1 > Int32(100) + Values: (Int32(10)) + SubqueryAlias: t + Projection: column1 AS #1 + Values: (Int32(1)), (Int32(2)) + + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_projection: + SAME TEXT AS ABOVE + + logical_plan after rewrite_set_comparison: + SAME TEXT AS ABOVE + + logical_plan after optimize_unions: + SAME TEXT AS ABOVE + + logical_plan after unions_to_filter: + SAME TEXT AS ABOVE + + logical_plan after simplify_expressions: + SAME TEXT AS ABOVE + + logical_plan after replace_distinct_aggregate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_predicate_subquery: + SAME TEXT AS ABOVE + + logical_plan after scalar_subquery_to_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_join: + SAME TEXT AS ABOVE + + logical_plan after extract_equijoin_predicate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_duplicated_expr: + SAME TEXT AS ABOVE + + logical_plan after eliminate_filter: + SAME TEXT AS ABOVE + + logical_plan after eliminate_cross_join: + SAME TEXT AS ABOVE + + logical_plan after eliminate_limit: + SAME TEXT AS ABOVE + + logical_plan after propagate_empty_relation: + SAME TEXT AS ABOVE + + logical_plan after filter_null_join_keys: + SAME TEXT AS ABOVE + + logical_plan after eliminate_outer_join: + SAME TEXT AS ABOVE + + logical_plan after push_down_limit: + SAME TEXT AS ABOVE + + logical_plan after push_down_filter: + SAME TEXT AS ABOVE + + logical_plan after single_distinct_aggregation_to_group_by: + SAME TEXT AS ABOVE + + logical_plan after eliminate_group_by_constant: + SAME TEXT AS ABOVE + + logical_plan after common_sub_expression_eliminate: + SAME TEXT AS ABOVE + + logical_plan after extract_leaf_expressions: + SAME TEXT AS ABOVE + + logical_plan after optimize_projections: + SAME TEXT AS ABOVE + + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE + + logical_plan: + Sort: #5 ASC NULLS FIRST + Projection: t.#1 AS #5, CAST(t.#1 AS Int64) + () AS shifted AS #6 + Subquery: + Projection: CAST(s.#3 AS Int64) AS #4 + SubqueryAlias: s + Projection: column1 AS #3 + Filter: column1 > Int32(100) + Values: (Int32(10)) + SubqueryAlias: t + Projection: column1 AS #1 + Values: (Int32(1)), (Int32(2)) + + initial_physical_plan: + ScalarSubqueryExec: subqueries=1 + SortExec: expr=[#5@0 ASC], preserve_partitioning=[false] + ProjectionExec: expr=[#1@0 as #5, CAST(#1@0 AS Int64) + scalar_subquery() as #6] + ProjectionExec: expr=[column1@0 as #1] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[CAST(#3@0 AS Int64) as #4] + ProjectionExec: expr=[column1@0 as #3] + FilterExec: column1@0 > 100 + DataSourceExec: partitions=1, partition_sizes=[1] + + + initial_physical_plan_with_stats: + ScalarSubqueryExec: subqueries=1, statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]:)]] + SortExec: expr=[#5@0 ASC], preserve_partitioning=[false], statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]:)]] + ProjectionExec: expr=[#1@0 as #5, CAST(#1@0 AS Int64) + scalar_subquery() as #6], statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]:)]] + ProjectionExec: expr=[column1@0 as #1], statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0))]] + DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0))]] + ProjectionExec: expr=[CAST(#3@0 AS Int64) as #4], statistics=[Rows=Inexact(1), Bytes=Inexact(), [(Col[0]: Min=Inexact(Int64(101)) Null=Inexact(0))]] + ProjectionExec: expr=[column1@0 as #3], statistics=[Rows=Inexact(1), Bytes=Inexact(), [(Col[0]: Min=Inexact(Int32(101)) Null=Inexact(0))]] + FilterExec: column1@0 > 100, statistics=[Rows=Inexact(1), Bytes=Inexact(), [(Col[0]: Min=Inexact(Int32(101)) Null=Inexact(0))]] + DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]: Null=Exact(0))]] + + + initial_physical_plan_with_schema: + ScalarSubqueryExec: subqueries=1, schema=[#5:Int32;N, #6:Int64;N] + SortExec: expr=[#5@0 ASC], preserve_partitioning=[false], schema=[#5:Int32;N, #6:Int64;N] + ProjectionExec: expr=[#1@0 as #5, CAST(#1@0 AS Int64) + scalar_subquery() as #6], schema=[#5:Int32;N, #6:Int64;N] + ProjectionExec: expr=[column1@0 as #1], schema=[#1:Int32;N] + DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N] + ProjectionExec: expr=[CAST(#3@0 AS Int64) as #4], schema=[#4:Int64;N] + ProjectionExec: expr=[column1@0 as #3], schema=[#3:Int32;N] + FilterExec: column1@0 > 100, schema=[column1:Int32;N] + DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N] + + + physical_plan after OutputRequirements: + OutputRequirementExec: order_by=[], dist_by=Unspecified + ScalarSubqueryExec: subqueries=1 + SortExec: expr=[#5@0 ASC], preserve_partitioning=[false] + ProjectionExec: expr=[#1@0 as #5, CAST(#1@0 AS Int64) + scalar_subquery() as #6] + ProjectionExec: expr=[column1@0 as #1] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[CAST(#3@0 AS Int64) as #4] + ProjectionExec: expr=[column1@0 as #3] + FilterExec: column1@0 > 100 + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after aggregate_statistics: + SAME TEXT AS ABOVE + + physical_plan after join_selection: + SAME TEXT AS ABOVE + + physical_plan after LimitedDistinctAggregation: + SAME TEXT AS ABOVE + + physical_plan after FilterPushdown: + SAME TEXT AS ABOVE + + physical_plan after EnforceDistribution: + OutputRequirementExec: order_by=[], dist_by=Unspecified + ScalarSubqueryExec: subqueries=1 + SortExec: expr=[#5@0 ASC], preserve_partitioning=[false] + ProjectionExec: expr=[#1@0 as #5, CAST(#1@0 AS Int64) + scalar_subquery() as #6] + ProjectionExec: expr=[column1@0 as #1] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[CAST(#3@0 AS Int64) as #4] + RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 + ProjectionExec: expr=[column1@0 as #3] + FilterExec: column1@0 > 100 + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after CombinePartialFinalAggregate: + SAME TEXT AS ABOVE + + physical_plan after EnforceSorting: + SAME TEXT AS ABOVE + + physical_plan after OptimizeAggregateOrder: + SAME TEXT AS ABOVE + + physical_plan after WindowTopN: + SAME TEXT AS ABOVE + + physical_plan after ProjectionPushdown: + OutputRequirementExec: order_by=[], dist_by=Unspecified + ScalarSubqueryExec: subqueries=1 + SortExec: expr=[#5@0 ASC], preserve_partitioning=[false] + ProjectionExec: expr=[column1@0 as #5, CAST(column1@0 AS Int64) + scalar_subquery() as #6] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[CAST(#3@0 AS Int64) as #4] + RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 + ProjectionExec: expr=[column1@0 as #3] + FilterExec: column1@0 > 100 + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after OutputRequirements: + ScalarSubqueryExec: subqueries=1 + SortExec: expr=[#5@0 ASC], preserve_partitioning=[false] + ProjectionExec: expr=[column1@0 as #5, CAST(column1@0 AS Int64) + scalar_subquery() as #6] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[CAST(#3@0 AS Int64) as #4] + RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 + ProjectionExec: expr=[column1@0 as #3] + FilterExec: column1@0 > 100 + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after LimitAggregation: + SAME TEXT AS ABOVE + + physical_plan after LimitPushPastWindows: + SAME TEXT AS ABOVE + + physical_plan after HashJoinBuffering: + SAME TEXT AS ABOVE + + physical_plan after LimitPushdown: + SAME TEXT AS ABOVE + + physical_plan after TopKRepartition: + SAME TEXT AS ABOVE + + physical_plan after ProjectionPushdown: + SAME TEXT AS ABOVE + + physical_plan after PushdownSort: + SAME TEXT AS ABOVE + + physical_plan after EnsureCooperative: + SAME TEXT AS ABOVE + + physical_plan after FilterPushdown(Post): + SAME TEXT AS ABOVE + + physical_plan after RewriteExplicitRepartition: + SAME TEXT AS ABOVE + + physical_plan after RewriteCollectLeftHashJoin: + SAME TEXT AS ABOVE + + physical_plan after EnforceBarrierPartitioning: + SAME TEXT AS ABOVE + + physical_plan after SanityCheckPlan: + SAME TEXT AS ABOVE + + physical_plan: + ProjectionExec: expr=[#5@0 as v, #6@1 as shifted] + ScalarSubqueryExec: subqueries=1 + SortExec: expr=[#5@0 ASC], preserve_partitioning=[false] + ProjectionExec: expr=[column1@0 as #5, CAST(column1@0 AS Int64) + scalar_subquery() as #6] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[CAST(#3@0 AS Int64) as #4] + RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 + ProjectionExec: expr=[column1@0 as #3] + FilterExec: column1@0 > 100 + DataSourceExec: partitions=1, partition_sizes=[1] + + + == Physical Plan == + ProjectionExec: expr=[#5@0 as v, #6@1 as shifted] + ScalarSubqueryExec: subqueries=1 + SortExec: expr=[#5@0 ASC], preserve_partitioning=[false] + ProjectionExec: expr=[column1@0 as #5, CAST(column1@0 AS Int64) + scalar_subquery() as #6] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[CAST(#3@0 AS Int64) as #4] + RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 + ProjectionExec: expr=[column1@0 as #3] + FilterExec: column1@0 > 100 + DataSourceExec: partitions=1, partition_sizes=[1] + + + == Distributed Plan == + === stage 0 === + inputs=[] + mode=Pipelined + partitions=1 + distribution=RoundRobin(channels=4) + placement=Worker + ProjectionExec: expr=[column1@0 as #3] + FilterExec: column1@0 > 100 + DataSourceExec: partitions=1, partition_sizes=[1] + + === stage 1 === + inputs=[StageInput(stage=0, mode=Shuffle)] + mode=Pipelined + partitions=4 + distribution=RoundRobin(channels=1) + placement=Worker + ProjectionExec: expr=[CAST(#3@0 AS Int64) as #4] + StageInputExec: input=0, partitioning=RoundRobinBatch(4) + + === stage 2 === + inputs=[StageInput(stage=1, mode=Broadcast)] + mode=Pipelined + partitions=1 + distribution=RoundRobin(channels=1) + placement=Worker + ProjectionExec: expr=[#5@0 as v, #6@1 as shifted] + ScalarSubqueryExec: subqueries=1 + SortExec: expr=[#5@0 ASC], preserve_partitioning=[false] + ProjectionExec: expr=[column1@0 as #5, CAST(column1@0 AS Int64) + scalar_subquery() as #6] + DataSourceExec: partitions=1, partition_sizes=[1] + StageInputExec: input=0, partitioning=UnknownPartitioning(1) +--- +name: "test_multiple_scalar_subqueries_in_one_expression" +data: + |- + == Codegen == + Whole-stage codegen is not supported; showing physical plan instead. + + == Plan Steps == + initial_logical_plan: + Projection: sum(t.#1 + #4 + #7) AS total AS #8 + Aggregate: groupBy=[[]], aggr=[[sum(CAST(t.#1 AS Int64) + () + ())]] + Subquery: + Projection: min(s.#3) AS #4 + Aggregate: groupBy=[[]], aggr=[[min(CAST(s.#3 AS Int64))]] + SubqueryAlias: s + Projection: #2 AS #3 + Projection: column1 AS #2 + Values: (Int32(10)), (Int32(20)) + Subquery: + Projection: max(u.#6) AS #7 + Aggregate: groupBy=[[]], aggr=[[max(CAST(u.#6 AS Int64))]] + SubqueryAlias: u + Projection: #5 AS #6 + Projection: column1 AS #5 + Values: (Int32(1)), (Int32(2)) + SubqueryAlias: t + Projection: #0 AS #1 + Projection: column1 AS #0 + Values: (Int32(1)), (Int32(2)), (Int32(3)) + + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE + + logical_plan after resolve_grouping_function: + SAME TEXT AS ABOVE + + logical_plan after type_coercion: + SAME TEXT AS ABOVE + + analyzed_logical_plan: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_projection: + SAME TEXT AS ABOVE + + logical_plan after rewrite_set_comparison: + SAME TEXT AS ABOVE + + logical_plan after optimize_unions: + SAME TEXT AS ABOVE + + logical_plan after unions_to_filter: + SAME TEXT AS ABOVE + + logical_plan after simplify_expressions: + SAME TEXT AS ABOVE + + logical_plan after replace_distinct_aggregate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_predicate_subquery: + SAME TEXT AS ABOVE + + logical_plan after scalar_subquery_to_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_join: + SAME TEXT AS ABOVE + + logical_plan after extract_equijoin_predicate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_duplicated_expr: + SAME TEXT AS ABOVE + + logical_plan after eliminate_filter: + SAME TEXT AS ABOVE + + logical_plan after eliminate_cross_join: + SAME TEXT AS ABOVE + + logical_plan after eliminate_limit: + SAME TEXT AS ABOVE + + logical_plan after propagate_empty_relation: + SAME TEXT AS ABOVE + + logical_plan after filter_null_join_keys: + SAME TEXT AS ABOVE + + logical_plan after eliminate_outer_join: + SAME TEXT AS ABOVE + + logical_plan after push_down_limit: + SAME TEXT AS ABOVE + + logical_plan after push_down_filter: + SAME TEXT AS ABOVE + + logical_plan after single_distinct_aggregation_to_group_by: + SAME TEXT AS ABOVE + + logical_plan after eliminate_group_by_constant: + SAME TEXT AS ABOVE + + logical_plan after common_sub_expression_eliminate: + SAME TEXT AS ABOVE + + logical_plan after extract_leaf_expressions: + SAME TEXT AS ABOVE + + logical_plan after optimize_projections: + Projection: sum(t.#1 + #4 + #7) AS total AS #8 + Aggregate: groupBy=[[]], aggr=[[sum(CAST(t.#1 AS Int64) + () + ())]] + Subquery: + Projection: min(s.#3) AS #4 + Aggregate: groupBy=[[]], aggr=[[min(CAST(s.#3 AS Int64))]] + SubqueryAlias: s + Projection: column1 AS #3 + Values: (Int32(10)), (Int32(20)) + Subquery: + Projection: max(u.#6) AS #7 + Aggregate: groupBy=[[]], aggr=[[max(CAST(u.#6 AS Int64))]] + SubqueryAlias: u + Projection: column1 AS #6 + Values: (Int32(1)), (Int32(2)) + SubqueryAlias: t + Projection: column1 AS #1 + Values: (Int32(1)), (Int32(2)), (Int32(3)) + + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_projection: + SAME TEXT AS ABOVE + + logical_plan after rewrite_set_comparison: + SAME TEXT AS ABOVE + + logical_plan after optimize_unions: + SAME TEXT AS ABOVE + + logical_plan after unions_to_filter: + SAME TEXT AS ABOVE + + logical_plan after simplify_expressions: + SAME TEXT AS ABOVE + + logical_plan after replace_distinct_aggregate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_predicate_subquery: + SAME TEXT AS ABOVE + + logical_plan after scalar_subquery_to_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_join: + SAME TEXT AS ABOVE + + logical_plan after extract_equijoin_predicate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_duplicated_expr: + SAME TEXT AS ABOVE + + logical_plan after eliminate_filter: + SAME TEXT AS ABOVE + + logical_plan after eliminate_cross_join: + SAME TEXT AS ABOVE + + logical_plan after eliminate_limit: + SAME TEXT AS ABOVE + + logical_plan after propagate_empty_relation: + SAME TEXT AS ABOVE + + logical_plan after filter_null_join_keys: + SAME TEXT AS ABOVE + + logical_plan after eliminate_outer_join: + SAME TEXT AS ABOVE + + logical_plan after push_down_limit: + SAME TEXT AS ABOVE + + logical_plan after push_down_filter: + SAME TEXT AS ABOVE + + logical_plan after single_distinct_aggregation_to_group_by: + SAME TEXT AS ABOVE + + logical_plan after eliminate_group_by_constant: + SAME TEXT AS ABOVE + + logical_plan after common_sub_expression_eliminate: + SAME TEXT AS ABOVE + + logical_plan after extract_leaf_expressions: + SAME TEXT AS ABOVE + + logical_plan after optimize_projections: + SAME TEXT AS ABOVE + + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE + + logical_plan: + Projection: sum(t.#1 + #4 + #7) AS total AS #8 + Aggregate: groupBy=[[]], aggr=[[sum(CAST(t.#1 AS Int64) + () + ())]] + Subquery: + Projection: min(s.#3) AS #4 + Aggregate: groupBy=[[]], aggr=[[min(CAST(s.#3 AS Int64))]] + SubqueryAlias: s + Projection: column1 AS #3 + Values: (Int32(10)), (Int32(20)) + Subquery: + Projection: max(u.#6) AS #7 + Aggregate: groupBy=[[]], aggr=[[max(CAST(u.#6 AS Int64))]] + SubqueryAlias: u + Projection: column1 AS #6 + Values: (Int32(1)), (Int32(2)) + SubqueryAlias: t + Projection: column1 AS #1 + Values: (Int32(1)), (Int32(2)), (Int32(3)) + + initial_physical_plan: + ScalarSubqueryExec: subqueries=2 + ProjectionExec: expr=[sum(t.#1 + #4 + #7)@0 as #8] + AggregateExec: mode=Final, gby=[], aggr=[sum(t.#1 + #4 + #7)] + AggregateExec: mode=Partial, gby=[], aggr=[sum(t.#1 + #4 + #7)] + ProjectionExec: expr=[column1@0 as #1] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#3)@0 as #4] + AggregateExec: mode=Final, gby=[], aggr=[min(s.#3)] + AggregateExec: mode=Partial, gby=[], aggr=[min(s.#3)] + ProjectionExec: expr=[column1@0 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[max(u.#6)@0 as #7] + AggregateExec: mode=Final, gby=[], aggr=[max(u.#6)] + AggregateExec: mode=Partial, gby=[], aggr=[max(u.#6)] + ProjectionExec: expr=[column1@0 as #6] + DataSourceExec: partitions=1, partition_sizes=[1] + + + initial_physical_plan_with_stats: + ScalarSubqueryExec: subqueries=2, statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]:)]] + ProjectionExec: expr=[sum(t.#1 + #4 + #7)@0 as #8], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]:)]] + AggregateExec: mode=Final, gby=[], aggr=[sum(t.#1 + #4 + #7)], statistics=[Rows=Exact(1), Bytes=Inexact(), [(Col[0]:)]] + AggregateExec: mode=Partial, gby=[], aggr=[sum(t.#1 + #4 + #7)], statistics=[Rows=Inexact(3), Bytes=Inexact(), [(Col[0]:)]] + ProjectionExec: expr=[column1@0 as #1], statistics=[Rows=Exact(3), Bytes=Exact(), [(Col[0]: Null=Exact(0))]] + DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(3), Bytes=Exact(), [(Col[0]: Null=Exact(0))]] + ProjectionExec: expr=[min(s.#3)@0 as #4], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]:)]] + AggregateExec: mode=Final, gby=[], aggr=[min(s.#3)], statistics=[Rows=Exact(1), Bytes=Inexact(), [(Col[0]:)]] + AggregateExec: mode=Partial, gby=[], aggr=[min(s.#3)], statistics=[Rows=Inexact(2), Bytes=Inexact(), [(Col[0]:)]] + ProjectionExec: expr=[column1@0 as #3], statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0))]] + DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0))]] + ProjectionExec: expr=[max(u.#6)@0 as #7], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]:)]] + AggregateExec: mode=Final, gby=[], aggr=[max(u.#6)], statistics=[Rows=Exact(1), Bytes=Inexact(), [(Col[0]:)]] + AggregateExec: mode=Partial, gby=[], aggr=[max(u.#6)], statistics=[Rows=Inexact(2), Bytes=Inexact(), [(Col[0]:)]] + ProjectionExec: expr=[column1@0 as #6], statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0))]] + DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0))]] + + + initial_physical_plan_with_schema: + ScalarSubqueryExec: subqueries=2, schema=[#8:Int64;N] + ProjectionExec: expr=[sum(t.#1 + #4 + #7)@0 as #8], schema=[#8:Int64;N] + AggregateExec: mode=Final, gby=[], aggr=[sum(t.#1 + #4 + #7)], schema=[sum(t.#1 + #4 + #7):Int64;N] + AggregateExec: mode=Partial, gby=[], aggr=[sum(t.#1 + #4 + #7)], schema=[sum(t.#1 + #4 + #7)[sum]:Int64;N] + ProjectionExec: expr=[column1@0 as #1], schema=[#1:Int32;N] + DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N] + ProjectionExec: expr=[min(s.#3)@0 as #4], schema=[#4:Int64;N] + AggregateExec: mode=Final, gby=[], aggr=[min(s.#3)], schema=[min(s.#3):Int64;N] + AggregateExec: mode=Partial, gby=[], aggr=[min(s.#3)], schema=[min(s.#3)[value]:Int64;N] + ProjectionExec: expr=[column1@0 as #3], schema=[#3:Int32;N] + DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N] + ProjectionExec: expr=[max(u.#6)@0 as #7], schema=[#7:Int64;N] + AggregateExec: mode=Final, gby=[], aggr=[max(u.#6)], schema=[max(u.#6):Int64;N] + AggregateExec: mode=Partial, gby=[], aggr=[max(u.#6)], schema=[max(u.#6)[value]:Int64;N] + ProjectionExec: expr=[column1@0 as #6], schema=[#6:Int32;N] + DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N] + + + physical_plan after OutputRequirements: + OutputRequirementExec: order_by=[], dist_by=Unspecified + ScalarSubqueryExec: subqueries=2 + ProjectionExec: expr=[sum(t.#1 + #4 + #7)@0 as #8] + AggregateExec: mode=Final, gby=[], aggr=[sum(t.#1 + #4 + #7)] + AggregateExec: mode=Partial, gby=[], aggr=[sum(t.#1 + #4 + #7)] + ProjectionExec: expr=[column1@0 as #1] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#3)@0 as #4] + AggregateExec: mode=Final, gby=[], aggr=[min(s.#3)] + AggregateExec: mode=Partial, gby=[], aggr=[min(s.#3)] + ProjectionExec: expr=[column1@0 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[max(u.#6)@0 as #7] + AggregateExec: mode=Final, gby=[], aggr=[max(u.#6)] + AggregateExec: mode=Partial, gby=[], aggr=[max(u.#6)] + ProjectionExec: expr=[column1@0 as #6] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after aggregate_statistics: + SAME TEXT AS ABOVE + + physical_plan after join_selection: + SAME TEXT AS ABOVE + + physical_plan after LimitedDistinctAggregation: + SAME TEXT AS ABOVE + + physical_plan after FilterPushdown: + SAME TEXT AS ABOVE + + physical_plan after EnforceDistribution: + SAME TEXT AS ABOVE + + physical_plan after CombinePartialFinalAggregate: + OutputRequirementExec: order_by=[], dist_by=Unspecified + ScalarSubqueryExec: subqueries=2 + ProjectionExec: expr=[sum(t.#1 + #4 + #7)@0 as #8] + AggregateExec: mode=Single, gby=[], aggr=[sum(t.#1 + #4 + #7)] + ProjectionExec: expr=[column1@0 as #1] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#3)@0 as #4] + AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)] + ProjectionExec: expr=[column1@0 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[max(u.#6)@0 as #7] + AggregateExec: mode=Single, gby=[], aggr=[max(u.#6)] + ProjectionExec: expr=[column1@0 as #6] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after EnforceSorting: + SAME TEXT AS ABOVE + + physical_plan after OptimizeAggregateOrder: + SAME TEXT AS ABOVE + + physical_plan after WindowTopN: + SAME TEXT AS ABOVE + + physical_plan after ProjectionPushdown: + SAME TEXT AS ABOVE + + physical_plan after OutputRequirements: + ScalarSubqueryExec: subqueries=2 + ProjectionExec: expr=[sum(t.#1 + #4 + #7)@0 as #8] + AggregateExec: mode=Single, gby=[], aggr=[sum(t.#1 + #4 + #7)] + ProjectionExec: expr=[column1@0 as #1] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#3)@0 as #4] + AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)] + ProjectionExec: expr=[column1@0 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[max(u.#6)@0 as #7] + AggregateExec: mode=Single, gby=[], aggr=[max(u.#6)] + ProjectionExec: expr=[column1@0 as #6] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after LimitAggregation: + SAME TEXT AS ABOVE + + physical_plan after LimitPushPastWindows: + SAME TEXT AS ABOVE + + physical_plan after HashJoinBuffering: + SAME TEXT AS ABOVE + + physical_plan after LimitPushdown: + SAME TEXT AS ABOVE + + physical_plan after TopKRepartition: + SAME TEXT AS ABOVE + + physical_plan after ProjectionPushdown: + SAME TEXT AS ABOVE + + physical_plan after PushdownSort: + SAME TEXT AS ABOVE + + physical_plan after EnsureCooperative: + SAME TEXT AS ABOVE + + physical_plan after FilterPushdown(Post): + SAME TEXT AS ABOVE + + physical_plan after RewriteExplicitRepartition: + SAME TEXT AS ABOVE + + physical_plan after RewriteCollectLeftHashJoin: + SAME TEXT AS ABOVE + + physical_plan after EnforceBarrierPartitioning: + SAME TEXT AS ABOVE + + physical_plan after SanityCheckPlan: + SAME TEXT AS ABOVE + + physical_plan: + ProjectionExec: expr=[#8@0 as total] + ScalarSubqueryExec: subqueries=2 + ProjectionExec: expr=[sum(t.#1 + #4 + #7)@0 as #8] + AggregateExec: mode=Single, gby=[], aggr=[sum(t.#1 + #4 + #7)] + ProjectionExec: expr=[column1@0 as #1] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#3)@0 as #4] + AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)] + ProjectionExec: expr=[column1@0 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[max(u.#6)@0 as #7] + AggregateExec: mode=Single, gby=[], aggr=[max(u.#6)] + ProjectionExec: expr=[column1@0 as #6] + DataSourceExec: partitions=1, partition_sizes=[1] + + + == Physical Plan == + ProjectionExec: expr=[#8@0 as total] + ScalarSubqueryExec: subqueries=2 + ProjectionExec: expr=[sum(t.#1 + #4 + #7)@0 as #8] + AggregateExec: mode=Single, gby=[], aggr=[sum(t.#1 + #4 + #7)] + ProjectionExec: expr=[column1@0 as #1] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#3)@0 as #4] + AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)] + ProjectionExec: expr=[column1@0 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[max(u.#6)@0 as #7] + AggregateExec: mode=Single, gby=[], aggr=[max(u.#6)] + ProjectionExec: expr=[column1@0 as #6] + DataSourceExec: partitions=1, partition_sizes=[1] + + + == Distributed Plan == + === stage 0 === + inputs=[] + mode=Pipelined + partitions=1 + distribution=RoundRobin(channels=1) + placement=Worker + ProjectionExec: expr=[min(s.#3)@0 as #4] + AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)] + ProjectionExec: expr=[column1@0 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + + === stage 1 === + inputs=[] + mode=Pipelined + partitions=1 + distribution=RoundRobin(channels=1) + placement=Worker + ProjectionExec: expr=[max(u.#6)@0 as #7] + AggregateExec: mode=Single, gby=[], aggr=[max(u.#6)] + ProjectionExec: expr=[column1@0 as #6] + DataSourceExec: partitions=1, partition_sizes=[1] + + === stage 2 === + inputs=[StageInput(stage=0, mode=Broadcast), StageInput(stage=1, mode=Broadcast)] + mode=Pipelined + partitions=1 + distribution=RoundRobin(channels=1) + placement=Worker + ProjectionExec: expr=[#8@0 as total] + ScalarSubqueryExec: subqueries=2 + ProjectionExec: expr=[sum(t.#1 + #4 + #7)@0 as #8] + AggregateExec: mode=Single, gby=[], aggr=[sum(t.#1 + #4 + #7)] + ProjectionExec: expr=[column1@0 as #1] + DataSourceExec: partitions=1, partition_sizes=[1] + StageInputExec: input=0, partitioning=UnknownPartitioning(1) + StageInputExec: input=1, partitioning=UnknownPartitioning(1) +--- +name: "test_nested_scalar_subquery" +data: + |- + == Codegen == + Whole-stage codegen is not supported; showing physical plan instead. + + == Plan Steps == + initial_logical_plan: + Sort: #8 ASC NULLS FIRST + Projection: t.#1 AS #8, CAST(t.#1 AS Int64) + () AS shifted AS #9 + Subquery: + Projection: min(s.#3) + () AS #7 + Subquery: + Projection: min(u.#5) AS #6 + Aggregate: groupBy=[[]], aggr=[[min(u.#5)]] + SubqueryAlias: u + Projection: #4 AS #5 + Projection: column1 AS #4 + Values: (Int32(1)) + Aggregate: groupBy=[[]], aggr=[[min(s.#3)]] + SubqueryAlias: s + Projection: #2 AS #3 + Projection: column1 AS #2 + Values: (Int32(10)), (Int32(20)) + SubqueryAlias: t + Projection: #0 AS #1 + Projection: column1 AS #0 + Values: (Int32(1)), (Int32(2)) + + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE + + logical_plan after resolve_grouping_function: + SAME TEXT AS ABOVE + + logical_plan after type_coercion: + Sort: #8 ASC NULLS FIRST + Projection: t.#1 AS #8, CAST(t.#1 AS Int64) + () AS shifted AS #9 + Subquery: + Projection: CAST(min(s.#3) + () AS #7 AS Int64) + Subquery: + Projection: min(u.#5) AS #6 + Aggregate: groupBy=[[]], aggr=[[min(u.#5)]] + SubqueryAlias: u + Projection: #4 AS #5 + Projection: column1 AS #4 + Values: (Int32(1)) + Aggregate: groupBy=[[]], aggr=[[min(s.#3)]] + SubqueryAlias: s + Projection: #2 AS #3 + Projection: column1 AS #2 + Values: (Int32(10)), (Int32(20)) + SubqueryAlias: t + Projection: #0 AS #1 + Projection: column1 AS #0 + Values: (Int32(1)), (Int32(2)) + + analyzed_logical_plan: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_projection: + SAME TEXT AS ABOVE + + logical_plan after rewrite_set_comparison: + SAME TEXT AS ABOVE + + logical_plan after optimize_unions: + SAME TEXT AS ABOVE + + logical_plan after unions_to_filter: + SAME TEXT AS ABOVE + + logical_plan after simplify_expressions: + SAME TEXT AS ABOVE + + logical_plan after replace_distinct_aggregate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_predicate_subquery: + SAME TEXT AS ABOVE + + logical_plan after scalar_subquery_to_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_join: + SAME TEXT AS ABOVE + + logical_plan after extract_equijoin_predicate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_duplicated_expr: + SAME TEXT AS ABOVE + + logical_plan after eliminate_filter: + SAME TEXT AS ABOVE + + logical_plan after eliminate_cross_join: + SAME TEXT AS ABOVE + + logical_plan after eliminate_limit: + SAME TEXT AS ABOVE + + logical_plan after propagate_empty_relation: + SAME TEXT AS ABOVE + + logical_plan after filter_null_join_keys: + SAME TEXT AS ABOVE + + logical_plan after eliminate_outer_join: + SAME TEXT AS ABOVE + + logical_plan after push_down_limit: + SAME TEXT AS ABOVE + + logical_plan after push_down_filter: + SAME TEXT AS ABOVE + + logical_plan after single_distinct_aggregation_to_group_by: + SAME TEXT AS ABOVE + + logical_plan after eliminate_group_by_constant: + SAME TEXT AS ABOVE + + logical_plan after common_sub_expression_eliminate: + SAME TEXT AS ABOVE + + logical_plan after extract_leaf_expressions: + SAME TEXT AS ABOVE + + logical_plan after optimize_projections: + Sort: #8 ASC NULLS FIRST + Projection: t.#1 AS #8, CAST(t.#1 AS Int64) + () AS shifted AS #9 + Subquery: + Projection: CAST(min(s.#3) + () AS #7 AS Int64) + Subquery: + Projection: min(u.#5) AS #6 + Aggregate: groupBy=[[]], aggr=[[min(u.#5)]] + SubqueryAlias: u + Projection: column1 AS #5 + Values: (Int32(1)) + Aggregate: groupBy=[[]], aggr=[[min(s.#3)]] + SubqueryAlias: s + Projection: column1 AS #3 + Values: (Int32(10)), (Int32(20)) + SubqueryAlias: t + Projection: column1 AS #1 + Values: (Int32(1)), (Int32(2)) + + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_projection: + SAME TEXT AS ABOVE + + logical_plan after rewrite_set_comparison: + SAME TEXT AS ABOVE + + logical_plan after optimize_unions: + SAME TEXT AS ABOVE + + logical_plan after unions_to_filter: + SAME TEXT AS ABOVE + + logical_plan after simplify_expressions: + SAME TEXT AS ABOVE + + logical_plan after replace_distinct_aggregate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_predicate_subquery: + SAME TEXT AS ABOVE + + logical_plan after scalar_subquery_to_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_join: + SAME TEXT AS ABOVE + + logical_plan after extract_equijoin_predicate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_duplicated_expr: + SAME TEXT AS ABOVE + + logical_plan after eliminate_filter: + SAME TEXT AS ABOVE + + logical_plan after eliminate_cross_join: + SAME TEXT AS ABOVE + + logical_plan after eliminate_limit: + SAME TEXT AS ABOVE + + logical_plan after propagate_empty_relation: + SAME TEXT AS ABOVE + + logical_plan after filter_null_join_keys: + SAME TEXT AS ABOVE + + logical_plan after eliminate_outer_join: + SAME TEXT AS ABOVE + + logical_plan after push_down_limit: + SAME TEXT AS ABOVE + + logical_plan after push_down_filter: + SAME TEXT AS ABOVE + + logical_plan after single_distinct_aggregation_to_group_by: + SAME TEXT AS ABOVE + + logical_plan after eliminate_group_by_constant: + SAME TEXT AS ABOVE + + logical_plan after common_sub_expression_eliminate: + SAME TEXT AS ABOVE + + logical_plan after extract_leaf_expressions: + SAME TEXT AS ABOVE + + logical_plan after optimize_projections: + SAME TEXT AS ABOVE + + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE + + logical_plan: + Sort: #8 ASC NULLS FIRST + Projection: t.#1 AS #8, CAST(t.#1 AS Int64) + () AS shifted AS #9 + Subquery: + Projection: CAST(min(s.#3) + () AS #7 AS Int64) + Subquery: + Projection: min(u.#5) AS #6 + Aggregate: groupBy=[[]], aggr=[[min(u.#5)]] + SubqueryAlias: u + Projection: column1 AS #5 + Values: (Int32(1)) + Aggregate: groupBy=[[]], aggr=[[min(s.#3)]] + SubqueryAlias: s + Projection: column1 AS #3 + Values: (Int32(10)), (Int32(20)) + SubqueryAlias: t + Projection: column1 AS #1 + Values: (Int32(1)), (Int32(2)) + + initial_physical_plan: + ScalarSubqueryExec: subqueries=1 + SortExec: expr=[#8@0 ASC], preserve_partitioning=[false] + ProjectionExec: expr=[#1@0 as #8, CAST(#1@0 AS Int64) + scalar_subquery() as #9] + ProjectionExec: expr=[column1@0 as #1] + DataSourceExec: partitions=1, partition_sizes=[1] + ScalarSubqueryExec: subqueries=1 + ProjectionExec: expr=[CAST(min(s.#3)@0 + scalar_subquery() AS Int64) as #7] + AggregateExec: mode=Final, gby=[], aggr=[min(s.#3)] + AggregateExec: mode=Partial, gby=[], aggr=[min(s.#3)] + ProjectionExec: expr=[column1@0 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(u.#5)@0 as #6] + AggregateExec: mode=Final, gby=[], aggr=[min(u.#5)] + AggregateExec: mode=Partial, gby=[], aggr=[min(u.#5)] + ProjectionExec: expr=[column1@0 as #5] + DataSourceExec: partitions=1, partition_sizes=[1] + + + initial_physical_plan_with_stats: + ScalarSubqueryExec: subqueries=1, statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]:)]] + SortExec: expr=[#8@0 ASC], preserve_partitioning=[false], statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]:)]] + ProjectionExec: expr=[#1@0 as #8, CAST(#1@0 AS Int64) + scalar_subquery() as #9], statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]:)]] + ProjectionExec: expr=[column1@0 as #1], statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0))]] + DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0))]] + ScalarSubqueryExec: subqueries=1, statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]:)]] + ProjectionExec: expr=[CAST(min(s.#3)@0 + scalar_subquery() AS Int64) as #7], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]:)]] + AggregateExec: mode=Final, gby=[], aggr=[min(s.#3)], statistics=[Rows=Exact(1), Bytes=Inexact(), [(Col[0]:)]] + AggregateExec: mode=Partial, gby=[], aggr=[min(s.#3)], statistics=[Rows=Inexact(2), Bytes=Inexact(), [(Col[0]:)]] + ProjectionExec: expr=[column1@0 as #3], statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0))]] + DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0))]] + ProjectionExec: expr=[min(u.#5)@0 as #6], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]:)]] + AggregateExec: mode=Final, gby=[], aggr=[min(u.#5)], statistics=[Rows=Exact(1), Bytes=Absent, [(Col[0]:)]] + AggregateExec: mode=Partial, gby=[], aggr=[min(u.#5)], statistics=[Rows=Exact(0), Bytes=Inexact(), [(Col[0]:)]] + ProjectionExec: expr=[column1@0 as #5], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]: Null=Exact(0))]] + DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]: Null=Exact(0))]] + + + initial_physical_plan_with_schema: + ScalarSubqueryExec: subqueries=1, schema=[#8:Int32;N, #9:Int64;N] + SortExec: expr=[#8@0 ASC], preserve_partitioning=[false], schema=[#8:Int32;N, #9:Int64;N] + ProjectionExec: expr=[#1@0 as #8, CAST(#1@0 AS Int64) + scalar_subquery() as #9], schema=[#8:Int32;N, #9:Int64;N] + ProjectionExec: expr=[column1@0 as #1], schema=[#1:Int32;N] + DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N] + ScalarSubqueryExec: subqueries=1, schema=[#7:Int64;N] + ProjectionExec: expr=[CAST(min(s.#3)@0 + scalar_subquery() AS Int64) as #7], schema=[#7:Int64;N] + AggregateExec: mode=Final, gby=[], aggr=[min(s.#3)], schema=[min(s.#3):Int32;N] + AggregateExec: mode=Partial, gby=[], aggr=[min(s.#3)], schema=[min(s.#3)[value]:Int32;N] + ProjectionExec: expr=[column1@0 as #3], schema=[#3:Int32;N] + DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N] + ProjectionExec: expr=[min(u.#5)@0 as #6], schema=[#6:Int32;N] + AggregateExec: mode=Final, gby=[], aggr=[min(u.#5)], schema=[min(u.#5):Int32;N] + AggregateExec: mode=Partial, gby=[], aggr=[min(u.#5)], schema=[min(u.#5)[value]:Int32;N] + ProjectionExec: expr=[column1@0 as #5], schema=[#5:Int32;N] + DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N] + + + physical_plan after OutputRequirements: + OutputRequirementExec: order_by=[], dist_by=Unspecified + ScalarSubqueryExec: subqueries=1 + SortExec: expr=[#8@0 ASC], preserve_partitioning=[false] + ProjectionExec: expr=[#1@0 as #8, CAST(#1@0 AS Int64) + scalar_subquery() as #9] + ProjectionExec: expr=[column1@0 as #1] + DataSourceExec: partitions=1, partition_sizes=[1] + ScalarSubqueryExec: subqueries=1 + ProjectionExec: expr=[CAST(min(s.#3)@0 + scalar_subquery() AS Int64) as #7] + AggregateExec: mode=Final, gby=[], aggr=[min(s.#3)] + AggregateExec: mode=Partial, gby=[], aggr=[min(s.#3)] + ProjectionExec: expr=[column1@0 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(u.#5)@0 as #6] + AggregateExec: mode=Final, gby=[], aggr=[min(u.#5)] + AggregateExec: mode=Partial, gby=[], aggr=[min(u.#5)] + ProjectionExec: expr=[column1@0 as #5] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after aggregate_statistics: + SAME TEXT AS ABOVE + + physical_plan after join_selection: + SAME TEXT AS ABOVE + + physical_plan after LimitedDistinctAggregation: + SAME TEXT AS ABOVE + + physical_plan after FilterPushdown: + SAME TEXT AS ABOVE + + physical_plan after EnforceDistribution: + SAME TEXT AS ABOVE + + physical_plan after CombinePartialFinalAggregate: + OutputRequirementExec: order_by=[], dist_by=Unspecified + ScalarSubqueryExec: subqueries=1 + SortExec: expr=[#8@0 ASC], preserve_partitioning=[false] + ProjectionExec: expr=[#1@0 as #8, CAST(#1@0 AS Int64) + scalar_subquery() as #9] + ProjectionExec: expr=[column1@0 as #1] + DataSourceExec: partitions=1, partition_sizes=[1] + ScalarSubqueryExec: subqueries=1 + ProjectionExec: expr=[CAST(min(s.#3)@0 + scalar_subquery() AS Int64) as #7] + AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)] + ProjectionExec: expr=[column1@0 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(u.#5)@0 as #6] + AggregateExec: mode=Single, gby=[], aggr=[min(u.#5)] + ProjectionExec: expr=[column1@0 as #5] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after EnforceSorting: + SAME TEXT AS ABOVE + + physical_plan after OptimizeAggregateOrder: + SAME TEXT AS ABOVE + + physical_plan after WindowTopN: + SAME TEXT AS ABOVE + + physical_plan after ProjectionPushdown: + OutputRequirementExec: order_by=[], dist_by=Unspecified + ScalarSubqueryExec: subqueries=1 + SortExec: expr=[#8@0 ASC], preserve_partitioning=[false] + ProjectionExec: expr=[column1@0 as #8, CAST(column1@0 AS Int64) + scalar_subquery() as #9] + DataSourceExec: partitions=1, partition_sizes=[1] + ScalarSubqueryExec: subqueries=1 + ProjectionExec: expr=[CAST(min(s.#3)@0 + scalar_subquery() AS Int64) as #7] + AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)] + ProjectionExec: expr=[column1@0 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(u.#5)@0 as #6] + AggregateExec: mode=Single, gby=[], aggr=[min(u.#5)] + ProjectionExec: expr=[column1@0 as #5] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after OutputRequirements: + ScalarSubqueryExec: subqueries=1 + SortExec: expr=[#8@0 ASC], preserve_partitioning=[false] + ProjectionExec: expr=[column1@0 as #8, CAST(column1@0 AS Int64) + scalar_subquery() as #9] + DataSourceExec: partitions=1, partition_sizes=[1] + ScalarSubqueryExec: subqueries=1 + ProjectionExec: expr=[CAST(min(s.#3)@0 + scalar_subquery() AS Int64) as #7] + AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)] + ProjectionExec: expr=[column1@0 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(u.#5)@0 as #6] + AggregateExec: mode=Single, gby=[], aggr=[min(u.#5)] + ProjectionExec: expr=[column1@0 as #5] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after LimitAggregation: + SAME TEXT AS ABOVE + + physical_plan after LimitPushPastWindows: + SAME TEXT AS ABOVE + + physical_plan after HashJoinBuffering: + SAME TEXT AS ABOVE + + physical_plan after LimitPushdown: + SAME TEXT AS ABOVE + + physical_plan after TopKRepartition: + SAME TEXT AS ABOVE + + physical_plan after ProjectionPushdown: + SAME TEXT AS ABOVE + + physical_plan after PushdownSort: + SAME TEXT AS ABOVE + + physical_plan after EnsureCooperative: + SAME TEXT AS ABOVE + + physical_plan after FilterPushdown(Post): + SAME TEXT AS ABOVE + + physical_plan after RewriteExplicitRepartition: + SAME TEXT AS ABOVE + + physical_plan after RewriteCollectLeftHashJoin: + SAME TEXT AS ABOVE + + physical_plan after EnforceBarrierPartitioning: + SAME TEXT AS ABOVE + + physical_plan after SanityCheckPlan: + SAME TEXT AS ABOVE + + physical_plan: + ProjectionExec: expr=[#8@0 as v, #9@1 as shifted] + ScalarSubqueryExec: subqueries=1 + SortExec: expr=[#8@0 ASC], preserve_partitioning=[false] + ProjectionExec: expr=[column1@0 as #8, CAST(column1@0 AS Int64) + scalar_subquery() as #9] + DataSourceExec: partitions=1, partition_sizes=[1] + ScalarSubqueryExec: subqueries=1 + ProjectionExec: expr=[CAST(min(s.#3)@0 + scalar_subquery() AS Int64) as #7] + AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)] + ProjectionExec: expr=[column1@0 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(u.#5)@0 as #6] + AggregateExec: mode=Single, gby=[], aggr=[min(u.#5)] + ProjectionExec: expr=[column1@0 as #5] + DataSourceExec: partitions=1, partition_sizes=[1] + + + == Physical Plan == + ProjectionExec: expr=[#8@0 as v, #9@1 as shifted] + ScalarSubqueryExec: subqueries=1 + SortExec: expr=[#8@0 ASC], preserve_partitioning=[false] + ProjectionExec: expr=[column1@0 as #8, CAST(column1@0 AS Int64) + scalar_subquery() as #9] + DataSourceExec: partitions=1, partition_sizes=[1] + ScalarSubqueryExec: subqueries=1 + ProjectionExec: expr=[CAST(min(s.#3)@0 + scalar_subquery() AS Int64) as #7] + AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)] + ProjectionExec: expr=[column1@0 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(u.#5)@0 as #6] + AggregateExec: mode=Single, gby=[], aggr=[min(u.#5)] + ProjectionExec: expr=[column1@0 as #5] + DataSourceExec: partitions=1, partition_sizes=[1] + + + == Distributed Plan == + === stage 0 === + inputs=[] + mode=Pipelined + partitions=1 + distribution=RoundRobin(channels=1) + placement=Worker + ProjectionExec: expr=[min(u.#5)@0 as #6] + AggregateExec: mode=Single, gby=[], aggr=[min(u.#5)] + ProjectionExec: expr=[column1@0 as #5] + DataSourceExec: partitions=1, partition_sizes=[1] + + === stage 1 === + inputs=[StageInput(stage=0, mode=Broadcast)] + mode=Pipelined + partitions=1 + distribution=RoundRobin(channels=1) + placement=Worker + ScalarSubqueryExec: subqueries=1 + ProjectionExec: expr=[CAST(min(s.#3)@0 + scalar_subquery() AS Int64) as #7] + AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)] + ProjectionExec: expr=[column1@0 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + StageInputExec: input=0, partitioning=UnknownPartitioning(1) + + === stage 2 === + inputs=[StageInput(stage=1, mode=Broadcast)] + mode=Pipelined + partitions=1 + distribution=RoundRobin(channels=1) + placement=Worker + ProjectionExec: expr=[#8@0 as v, #9@1 as shifted] + ScalarSubqueryExec: subqueries=1 + SortExec: expr=[#8@0 ASC], preserve_partitioning=[false] + ProjectionExec: expr=[column1@0 as #8, CAST(column1@0 AS Int64) + scalar_subquery() as #9] + DataSourceExec: partitions=1, partition_sizes=[1] + StageInputExec: input=0, partitioning=UnknownPartitioning(1) +--- +name: "test_null_scalar_subquery_result" +data: + |- + == Codegen == + Whole-stage codegen is not supported; showing physical plan instead. + + == Plan Steps == + initial_logical_plan: + Sort: #5 ASC NULLS FIRST + Projection: t.#1 AS #5, CAST(t.#1 AS Int64) + () AS maybe_shifted AS #6 + Subquery: + Projection: min(s.#3) AS #4 + Aggregate: groupBy=[[]], aggr=[[min(CAST(s.#3 AS Int64))]] + SubqueryAlias: s + Projection: #2 AS #3 + Projection: column1 AS #2 + Values: (CAST(NULL AS Int32)) + SubqueryAlias: t + Projection: #0 AS #1 + Projection: column1 AS #0 + Values: (Int32(1)), (Int32(2)) + + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE + + logical_plan after resolve_grouping_function: + SAME TEXT AS ABOVE + + logical_plan after type_coercion: + SAME TEXT AS ABOVE + + analyzed_logical_plan: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_projection: + SAME TEXT AS ABOVE + + logical_plan after rewrite_set_comparison: + SAME TEXT AS ABOVE + + logical_plan after optimize_unions: + SAME TEXT AS ABOVE + + logical_plan after unions_to_filter: + SAME TEXT AS ABOVE + + logical_plan after simplify_expressions: + Sort: #5 ASC NULLS FIRST + Projection: t.#1 AS #5, CAST(t.#1 AS Int64) + () AS maybe_shifted AS #6 + Subquery: + Projection: min(s.#3) AS #4 + Aggregate: groupBy=[[]], aggr=[[min(CAST(s.#3 AS Int64))]] + SubqueryAlias: s + Projection: #2 AS #3 + Projection: column1 AS #2 + Values: (Int32(NULL) AS NULL) + SubqueryAlias: t + Projection: #0 AS #1 + Projection: column1 AS #0 + Values: (Int32(1)), (Int32(2)) + + logical_plan after replace_distinct_aggregate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_predicate_subquery: + SAME TEXT AS ABOVE + + logical_plan after scalar_subquery_to_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_join: + SAME TEXT AS ABOVE + + logical_plan after extract_equijoin_predicate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_duplicated_expr: + SAME TEXT AS ABOVE + + logical_plan after eliminate_filter: + SAME TEXT AS ABOVE + + logical_plan after eliminate_cross_join: + SAME TEXT AS ABOVE + + logical_plan after eliminate_limit: + SAME TEXT AS ABOVE + + logical_plan after propagate_empty_relation: + SAME TEXT AS ABOVE + + logical_plan after filter_null_join_keys: + SAME TEXT AS ABOVE + + logical_plan after eliminate_outer_join: + SAME TEXT AS ABOVE + + logical_plan after push_down_limit: + SAME TEXT AS ABOVE + + logical_plan after push_down_filter: + SAME TEXT AS ABOVE + + logical_plan after single_distinct_aggregation_to_group_by: + SAME TEXT AS ABOVE + + logical_plan after eliminate_group_by_constant: + SAME TEXT AS ABOVE + + logical_plan after common_sub_expression_eliminate: + SAME TEXT AS ABOVE + + logical_plan after extract_leaf_expressions: + SAME TEXT AS ABOVE + + logical_plan after optimize_projections: + Sort: #5 ASC NULLS FIRST + Projection: t.#1 AS #5, CAST(t.#1 AS Int64) + () AS maybe_shifted AS #6 + Subquery: + Projection: min(s.#3) AS #4 + Aggregate: groupBy=[[]], aggr=[[min(CAST(s.#3 AS Int64))]] + SubqueryAlias: s + Projection: column1 AS #3 + Values: (Int32(NULL) AS NULL) + SubqueryAlias: t + Projection: column1 AS #1 + Values: (Int32(1)), (Int32(2)) + + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_projection: + SAME TEXT AS ABOVE + + logical_plan after rewrite_set_comparison: + SAME TEXT AS ABOVE + + logical_plan after optimize_unions: + SAME TEXT AS ABOVE + + logical_plan after unions_to_filter: + SAME TEXT AS ABOVE + + logical_plan after simplify_expressions: + SAME TEXT AS ABOVE + + logical_plan after replace_distinct_aggregate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_predicate_subquery: + SAME TEXT AS ABOVE + + logical_plan after scalar_subquery_to_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_join: + SAME TEXT AS ABOVE + + logical_plan after extract_equijoin_predicate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_duplicated_expr: + SAME TEXT AS ABOVE + + logical_plan after eliminate_filter: + SAME TEXT AS ABOVE + + logical_plan after eliminate_cross_join: + SAME TEXT AS ABOVE + + logical_plan after eliminate_limit: + SAME TEXT AS ABOVE + + logical_plan after propagate_empty_relation: + SAME TEXT AS ABOVE + + logical_plan after filter_null_join_keys: + SAME TEXT AS ABOVE + + logical_plan after eliminate_outer_join: + SAME TEXT AS ABOVE + + logical_plan after push_down_limit: + SAME TEXT AS ABOVE + + logical_plan after push_down_filter: + SAME TEXT AS ABOVE + + logical_plan after single_distinct_aggregation_to_group_by: + SAME TEXT AS ABOVE + + logical_plan after eliminate_group_by_constant: + SAME TEXT AS ABOVE + + logical_plan after common_sub_expression_eliminate: + SAME TEXT AS ABOVE + + logical_plan after extract_leaf_expressions: + SAME TEXT AS ABOVE + + logical_plan after optimize_projections: + SAME TEXT AS ABOVE + + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE + + logical_plan: + Sort: #5 ASC NULLS FIRST + Projection: t.#1 AS #5, CAST(t.#1 AS Int64) + () AS maybe_shifted AS #6 + Subquery: + Projection: min(s.#3) AS #4 + Aggregate: groupBy=[[]], aggr=[[min(CAST(s.#3 AS Int64))]] + SubqueryAlias: s + Projection: column1 AS #3 + Values: (Int32(NULL) AS NULL) + SubqueryAlias: t + Projection: column1 AS #1 + Values: (Int32(1)), (Int32(2)) + + initial_physical_plan: + ScalarSubqueryExec: subqueries=1 + SortExec: expr=[#5@0 ASC], preserve_partitioning=[false] + ProjectionExec: expr=[#1@0 as #5, CAST(#1@0 AS Int64) + scalar_subquery() as #6] + ProjectionExec: expr=[column1@0 as #1] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#3)@0 as #4] + AggregateExec: mode=Final, gby=[], aggr=[min(s.#3)] + AggregateExec: mode=Partial, gby=[], aggr=[min(s.#3)] + ProjectionExec: expr=[column1@0 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + + + initial_physical_plan_with_stats: + ScalarSubqueryExec: subqueries=1, statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]:)]] + SortExec: expr=[#5@0 ASC], preserve_partitioning=[false], statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]:)]] + ProjectionExec: expr=[#1@0 as #5, CAST(#1@0 AS Int64) + scalar_subquery() as #6], statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]:)]] + ProjectionExec: expr=[column1@0 as #1], statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0))]] + DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0))]] + ProjectionExec: expr=[min(s.#3)@0 as #4], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]:)]] + AggregateExec: mode=Final, gby=[], aggr=[min(s.#3)], statistics=[Rows=Exact(1), Bytes=Absent, [(Col[0]:)]] + AggregateExec: mode=Partial, gby=[], aggr=[min(s.#3)], statistics=[Rows=Exact(0), Bytes=Inexact(), [(Col[0]:)]] + ProjectionExec: expr=[column1@0 as #3], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]: Null=Exact(1))]] + DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]: Null=Exact(1))]] + + + initial_physical_plan_with_schema: + ScalarSubqueryExec: subqueries=1, schema=[#5:Int32;N, #6:Int64;N] + SortExec: expr=[#5@0 ASC], preserve_partitioning=[false], schema=[#5:Int32;N, #6:Int64;N] + ProjectionExec: expr=[#1@0 as #5, CAST(#1@0 AS Int64) + scalar_subquery() as #6], schema=[#5:Int32;N, #6:Int64;N] + ProjectionExec: expr=[column1@0 as #1], schema=[#1:Int32;N] + DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N] + ProjectionExec: expr=[min(s.#3)@0 as #4], schema=[#4:Int64;N] + AggregateExec: mode=Final, gby=[], aggr=[min(s.#3)], schema=[min(s.#3):Int64;N] + AggregateExec: mode=Partial, gby=[], aggr=[min(s.#3)], schema=[min(s.#3)[value]:Int64;N] + ProjectionExec: expr=[column1@0 as #3], schema=[#3:Int32;N] + DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N] + + + physical_plan after OutputRequirements: + OutputRequirementExec: order_by=[], dist_by=Unspecified + ScalarSubqueryExec: subqueries=1 + SortExec: expr=[#5@0 ASC], preserve_partitioning=[false] + ProjectionExec: expr=[#1@0 as #5, CAST(#1@0 AS Int64) + scalar_subquery() as #6] + ProjectionExec: expr=[column1@0 as #1] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#3)@0 as #4] + AggregateExec: mode=Final, gby=[], aggr=[min(s.#3)] + AggregateExec: mode=Partial, gby=[], aggr=[min(s.#3)] + ProjectionExec: expr=[column1@0 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after aggregate_statistics: + SAME TEXT AS ABOVE + + physical_plan after join_selection: + SAME TEXT AS ABOVE + + physical_plan after LimitedDistinctAggregation: + SAME TEXT AS ABOVE + + physical_plan after FilterPushdown: + SAME TEXT AS ABOVE + + physical_plan after EnforceDistribution: + SAME TEXT AS ABOVE + + physical_plan after CombinePartialFinalAggregate: + OutputRequirementExec: order_by=[], dist_by=Unspecified + ScalarSubqueryExec: subqueries=1 + SortExec: expr=[#5@0 ASC], preserve_partitioning=[false] + ProjectionExec: expr=[#1@0 as #5, CAST(#1@0 AS Int64) + scalar_subquery() as #6] + ProjectionExec: expr=[column1@0 as #1] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#3)@0 as #4] + AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)] + ProjectionExec: expr=[column1@0 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after EnforceSorting: + SAME TEXT AS ABOVE + + physical_plan after OptimizeAggregateOrder: + SAME TEXT AS ABOVE + + physical_plan after WindowTopN: + SAME TEXT AS ABOVE + + physical_plan after ProjectionPushdown: + OutputRequirementExec: order_by=[], dist_by=Unspecified + ScalarSubqueryExec: subqueries=1 + SortExec: expr=[#5@0 ASC], preserve_partitioning=[false] + ProjectionExec: expr=[column1@0 as #5, CAST(column1@0 AS Int64) + scalar_subquery() as #6] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#3)@0 as #4] + AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)] + ProjectionExec: expr=[column1@0 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after OutputRequirements: + ScalarSubqueryExec: subqueries=1 + SortExec: expr=[#5@0 ASC], preserve_partitioning=[false] + ProjectionExec: expr=[column1@0 as #5, CAST(column1@0 AS Int64) + scalar_subquery() as #6] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#3)@0 as #4] + AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)] + ProjectionExec: expr=[column1@0 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after LimitAggregation: + SAME TEXT AS ABOVE + + physical_plan after LimitPushPastWindows: + SAME TEXT AS ABOVE + + physical_plan after HashJoinBuffering: + SAME TEXT AS ABOVE + + physical_plan after LimitPushdown: + SAME TEXT AS ABOVE + + physical_plan after TopKRepartition: + SAME TEXT AS ABOVE + + physical_plan after ProjectionPushdown: + SAME TEXT AS ABOVE + + physical_plan after PushdownSort: + SAME TEXT AS ABOVE + + physical_plan after EnsureCooperative: + SAME TEXT AS ABOVE + + physical_plan after FilterPushdown(Post): + SAME TEXT AS ABOVE + + physical_plan after RewriteExplicitRepartition: + SAME TEXT AS ABOVE + + physical_plan after RewriteCollectLeftHashJoin: + SAME TEXT AS ABOVE + + physical_plan after EnforceBarrierPartitioning: + SAME TEXT AS ABOVE + + physical_plan after SanityCheckPlan: + SAME TEXT AS ABOVE + + physical_plan: + ProjectionExec: expr=[#5@0 as v, #6@1 as maybe_shifted] + ScalarSubqueryExec: subqueries=1 + SortExec: expr=[#5@0 ASC], preserve_partitioning=[false] + ProjectionExec: expr=[column1@0 as #5, CAST(column1@0 AS Int64) + scalar_subquery() as #6] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#3)@0 as #4] + AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)] + ProjectionExec: expr=[column1@0 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + + + == Physical Plan == + ProjectionExec: expr=[#5@0 as v, #6@1 as maybe_shifted] + ScalarSubqueryExec: subqueries=1 + SortExec: expr=[#5@0 ASC], preserve_partitioning=[false] + ProjectionExec: expr=[column1@0 as #5, CAST(column1@0 AS Int64) + scalar_subquery() as #6] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#3)@0 as #4] + AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)] + ProjectionExec: expr=[column1@0 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + + + == Distributed Plan == + === stage 0 === + inputs=[] + mode=Pipelined + partitions=1 + distribution=RoundRobin(channels=1) + placement=Worker + ProjectionExec: expr=[min(s.#3)@0 as #4] + AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)] + ProjectionExec: expr=[column1@0 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + + === stage 1 === + inputs=[StageInput(stage=0, mode=Broadcast)] + mode=Pipelined + partitions=1 + distribution=RoundRobin(channels=1) + placement=Worker + ProjectionExec: expr=[#5@0 as v, #6@1 as maybe_shifted] + ScalarSubqueryExec: subqueries=1 + SortExec: expr=[#5@0 ASC], preserve_partitioning=[false] + ProjectionExec: expr=[column1@0 as #5, CAST(column1@0 AS Int64) + scalar_subquery() as #6] + DataSourceExec: partitions=1, partition_sizes=[1] + StageInputExec: input=0, partitioning=UnknownPartitioning(1) +--- +name: "test_scalar_subquery_in_aggregate_filter" +data: + |- + == Codegen == + Whole-stage codegen is not supported; showing physical plan instead. + + == Plan Steps == + initial_logical_plan: + Projection: t.#2 AS #7, count(Int64(1)) FILTER (WHERE t.#3 > ()) AS n AS #8 + Aggregate: groupBy=[[t.#2]], aggr=[[count(Int64(1)) FILTER (WHERE t.#3 > ())]] + Subquery: + Projection: min(s.#5) AS #6 + Aggregate: groupBy=[[]], aggr=[[min(s.#5)]] + SubqueryAlias: s + Projection: #4 AS #5 + Projection: column1 AS #4 + Values: (Int32(1)) + SubqueryAlias: t + Projection: #0 AS #2, #1 AS #3 + Projection: column1 AS #0, column2 AS #1 + Values: (Int32(1), Int32(1)), (Int32(1), Int32(2)), (Int32(2), Int32(3)) + + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE + + logical_plan after resolve_grouping_function: + SAME TEXT AS ABOVE + + logical_plan after type_coercion: + SAME TEXT AS ABOVE + + analyzed_logical_plan: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_projection: + SAME TEXT AS ABOVE + + logical_plan after rewrite_set_comparison: + SAME TEXT AS ABOVE + + logical_plan after optimize_unions: + SAME TEXT AS ABOVE + + logical_plan after unions_to_filter: + SAME TEXT AS ABOVE + + logical_plan after simplify_expressions: + SAME TEXT AS ABOVE + + logical_plan after replace_distinct_aggregate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_predicate_subquery: + SAME TEXT AS ABOVE + + logical_plan after scalar_subquery_to_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_join: + SAME TEXT AS ABOVE + + logical_plan after extract_equijoin_predicate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_duplicated_expr: + SAME TEXT AS ABOVE + + logical_plan after eliminate_filter: + SAME TEXT AS ABOVE + + logical_plan after eliminate_cross_join: + SAME TEXT AS ABOVE + + logical_plan after eliminate_limit: + SAME TEXT AS ABOVE + + logical_plan after propagate_empty_relation: + SAME TEXT AS ABOVE + + logical_plan after filter_null_join_keys: + SAME TEXT AS ABOVE + + logical_plan after eliminate_outer_join: + SAME TEXT AS ABOVE + + logical_plan after push_down_limit: + SAME TEXT AS ABOVE + + logical_plan after push_down_filter: + SAME TEXT AS ABOVE + + logical_plan after single_distinct_aggregation_to_group_by: + SAME TEXT AS ABOVE + + logical_plan after eliminate_group_by_constant: + SAME TEXT AS ABOVE + + logical_plan after common_sub_expression_eliminate: + SAME TEXT AS ABOVE + + logical_plan after extract_leaf_expressions: + SAME TEXT AS ABOVE + + logical_plan after optimize_projections: + Projection: t.#2 AS #7, count(Int64(1)) FILTER (WHERE t.#3 > ()) AS n AS #8 + Aggregate: groupBy=[[t.#2]], aggr=[[count(Int64(1)) FILTER (WHERE t.#3 > ())]] + Subquery: + Projection: min(s.#5) AS #6 + Aggregate: groupBy=[[]], aggr=[[min(s.#5)]] + SubqueryAlias: s + Projection: column1 AS #5 + Values: (Int32(1)) + SubqueryAlias: t + Projection: column1 AS #2, column2 AS #3 + Values: (Int32(1), Int32(1)), (Int32(1), Int32(2)), (Int32(2), Int32(3)) + + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_projection: + SAME TEXT AS ABOVE + + logical_plan after rewrite_set_comparison: + SAME TEXT AS ABOVE + + logical_plan after optimize_unions: + SAME TEXT AS ABOVE + + logical_plan after unions_to_filter: + SAME TEXT AS ABOVE + + logical_plan after simplify_expressions: + SAME TEXT AS ABOVE + + logical_plan after replace_distinct_aggregate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_predicate_subquery: + SAME TEXT AS ABOVE + + logical_plan after scalar_subquery_to_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_join: + SAME TEXT AS ABOVE + + logical_plan after extract_equijoin_predicate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_duplicated_expr: + SAME TEXT AS ABOVE + + logical_plan after eliminate_filter: + SAME TEXT AS ABOVE + + logical_plan after eliminate_cross_join: + SAME TEXT AS ABOVE + + logical_plan after eliminate_limit: + SAME TEXT AS ABOVE + + logical_plan after propagate_empty_relation: + SAME TEXT AS ABOVE + + logical_plan after filter_null_join_keys: + SAME TEXT AS ABOVE + + logical_plan after eliminate_outer_join: + SAME TEXT AS ABOVE + + logical_plan after push_down_limit: + SAME TEXT AS ABOVE + + logical_plan after push_down_filter: + SAME TEXT AS ABOVE + + logical_plan after single_distinct_aggregation_to_group_by: + SAME TEXT AS ABOVE + + logical_plan after eliminate_group_by_constant: + SAME TEXT AS ABOVE + + logical_plan after common_sub_expression_eliminate: + SAME TEXT AS ABOVE + + logical_plan after extract_leaf_expressions: + SAME TEXT AS ABOVE + + logical_plan after optimize_projections: + SAME TEXT AS ABOVE + + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE + + logical_plan: + Projection: t.#2 AS #7, count(Int64(1)) FILTER (WHERE t.#3 > ()) AS n AS #8 + Aggregate: groupBy=[[t.#2]], aggr=[[count(Int64(1)) FILTER (WHERE t.#3 > ())]] + Subquery: + Projection: min(s.#5) AS #6 + Aggregate: groupBy=[[]], aggr=[[min(s.#5)]] + SubqueryAlias: s + Projection: column1 AS #5 + Values: (Int32(1)) + SubqueryAlias: t + Projection: column1 AS #2, column2 AS #3 + Values: (Int32(1), Int32(1)), (Int32(1), Int32(2)), (Int32(2), Int32(3)) + + initial_physical_plan: + ScalarSubqueryExec: subqueries=1 + ProjectionExec: expr=[#2@0 as #7, count(Int64(1)) FILTER (WHERE t.#3 > ())@1 as #8] + AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[count(Int64(1)) FILTER (WHERE t.#3 > ())] + AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[count(Int64(1)) FILTER (WHERE t.#3 > ())] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#5)@0 as #6] + AggregateExec: mode=Final, gby=[], aggr=[min(s.#5)] + AggregateExec: mode=Partial, gby=[], aggr=[min(s.#5)] + ProjectionExec: expr=[column1@0 as #5] + DataSourceExec: partitions=1, partition_sizes=[1] + + + initial_physical_plan_with_stats: + ScalarSubqueryExec: subqueries=1, statistics=[Rows=Inexact(3), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]] + ProjectionExec: expr=[#2@0 as #7, count(Int64(1)) FILTER (WHERE t.#3 > ())@1 as #8], statistics=[Rows=Inexact(3), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]] + AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[count(Int64(1)) FILTER (WHERE t.#3 > ())], statistics=[Rows=Inexact(3), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]] + AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[count(Int64(1)) FILTER (WHERE t.#3 > ())], statistics=[Rows=Inexact(3), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3], statistics=[Rows=Exact(3), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]] + DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(3), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]] + ProjectionExec: expr=[min(s.#5)@0 as #6], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]:)]] + AggregateExec: mode=Final, gby=[], aggr=[min(s.#5)], statistics=[Rows=Exact(1), Bytes=Absent, [(Col[0]:)]] + AggregateExec: mode=Partial, gby=[], aggr=[min(s.#5)], statistics=[Rows=Exact(0), Bytes=Inexact(), [(Col[0]:)]] + ProjectionExec: expr=[column1@0 as #5], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]: Null=Exact(0))]] + DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]: Null=Exact(0))]] + + + initial_physical_plan_with_schema: + ScalarSubqueryExec: subqueries=1, schema=[#7:Int32;N, #8:Int64] + ProjectionExec: expr=[#2@0 as #7, count(Int64(1)) FILTER (WHERE t.#3 > ())@1 as #8], schema=[#7:Int32;N, #8:Int64] + AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[count(Int64(1)) FILTER (WHERE t.#3 > ())], schema=[#2:Int32;N, count(Int64(1)) FILTER (WHERE t.#3 > ()):Int64] + AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[count(Int64(1)) FILTER (WHERE t.#3 > ())], schema=[#2:Int32;N, count(Int64(1)) FILTER (WHERE t.#3 > ())[count]:Int64] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3], schema=[#2:Int32;N, #3:Int32;N] + DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N, column2:Int32;N] + ProjectionExec: expr=[min(s.#5)@0 as #6], schema=[#6:Int32;N] + AggregateExec: mode=Final, gby=[], aggr=[min(s.#5)], schema=[min(s.#5):Int32;N] + AggregateExec: mode=Partial, gby=[], aggr=[min(s.#5)], schema=[min(s.#5)[value]:Int32;N] + ProjectionExec: expr=[column1@0 as #5], schema=[#5:Int32;N] + DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N] + + + physical_plan after OutputRequirements: + OutputRequirementExec: order_by=[], dist_by=Unspecified + ScalarSubqueryExec: subqueries=1 + ProjectionExec: expr=[#2@0 as #7, count(Int64(1)) FILTER (WHERE t.#3 > ())@1 as #8] + AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[count(Int64(1)) FILTER (WHERE t.#3 > ())] + AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[count(Int64(1)) FILTER (WHERE t.#3 > ())] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#5)@0 as #6] + AggregateExec: mode=Final, gby=[], aggr=[min(s.#5)] + AggregateExec: mode=Partial, gby=[], aggr=[min(s.#5)] + ProjectionExec: expr=[column1@0 as #5] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after aggregate_statistics: + SAME TEXT AS ABOVE + + physical_plan after join_selection: + SAME TEXT AS ABOVE + + physical_plan after LimitedDistinctAggregation: + SAME TEXT AS ABOVE + + physical_plan after FilterPushdown: + SAME TEXT AS ABOVE + + physical_plan after EnforceDistribution: + OutputRequirementExec: order_by=[], dist_by=Unspecified + ScalarSubqueryExec: subqueries=1 + ProjectionExec: expr=[#2@0 as #7, count(Int64(1)) FILTER (WHERE t.#3 > ())@1 as #8] + AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[count(Int64(1)) FILTER (WHERE t.#3 > ())] + RepartitionExec: partitioning=Hash([#2@0], 4), input_partitions=1 + AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[count(Int64(1)) FILTER (WHERE t.#3 > ())] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#5)@0 as #6] + AggregateExec: mode=Final, gby=[], aggr=[min(s.#5)] + AggregateExec: mode=Partial, gby=[], aggr=[min(s.#5)] + ProjectionExec: expr=[column1@0 as #5] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after CombinePartialFinalAggregate: + OutputRequirementExec: order_by=[], dist_by=Unspecified + ScalarSubqueryExec: subqueries=1 + ProjectionExec: expr=[#2@0 as #7, count(Int64(1)) FILTER (WHERE t.#3 > ())@1 as #8] + AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[count(Int64(1)) FILTER (WHERE t.#3 > ())] + RepartitionExec: partitioning=Hash([#2@0], 4), input_partitions=1 + AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[count(Int64(1)) FILTER (WHERE t.#3 > ())] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#5)@0 as #6] + AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)] + ProjectionExec: expr=[column1@0 as #5] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after EnforceSorting: + SAME TEXT AS ABOVE + + physical_plan after OptimizeAggregateOrder: + SAME TEXT AS ABOVE + + physical_plan after WindowTopN: + SAME TEXT AS ABOVE + + physical_plan after ProjectionPushdown: + SAME TEXT AS ABOVE + + physical_plan after OutputRequirements: + ScalarSubqueryExec: subqueries=1 + ProjectionExec: expr=[#2@0 as #7, count(Int64(1)) FILTER (WHERE t.#3 > ())@1 as #8] + AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[count(Int64(1)) FILTER (WHERE t.#3 > ())] + RepartitionExec: partitioning=Hash([#2@0], 4), input_partitions=1 + AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[count(Int64(1)) FILTER (WHERE t.#3 > ())] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#5)@0 as #6] + AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)] + ProjectionExec: expr=[column1@0 as #5] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after LimitAggregation: + SAME TEXT AS ABOVE + + physical_plan after LimitPushPastWindows: + SAME TEXT AS ABOVE + + physical_plan after HashJoinBuffering: + SAME TEXT AS ABOVE + + physical_plan after LimitPushdown: + SAME TEXT AS ABOVE + + physical_plan after TopKRepartition: + SAME TEXT AS ABOVE + + physical_plan after ProjectionPushdown: + SAME TEXT AS ABOVE + + physical_plan after PushdownSort: + SAME TEXT AS ABOVE + + physical_plan after EnsureCooperative: + SAME TEXT AS ABOVE + + physical_plan after FilterPushdown(Post): + SAME TEXT AS ABOVE + + physical_plan after RewriteExplicitRepartition: + SAME TEXT AS ABOVE + + physical_plan after RewriteCollectLeftHashJoin: + SAME TEXT AS ABOVE + + physical_plan after EnforceBarrierPartitioning: + SAME TEXT AS ABOVE + + physical_plan after SanityCheckPlan: + SAME TEXT AS ABOVE + + physical_plan: + ProjectionExec: expr=[#7@0 as k, #8@1 as n] + ScalarSubqueryExec: subqueries=1 + ProjectionExec: expr=[#2@0 as #7, count(Int64(1)) FILTER (WHERE t.#3 > ())@1 as #8] + AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[count(Int64(1)) FILTER (WHERE t.#3 > ())] + RepartitionExec: partitioning=Hash([#2@0], 4), input_partitions=1 + AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[count(Int64(1)) FILTER (WHERE t.#3 > ())] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#5)@0 as #6] + AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)] + ProjectionExec: expr=[column1@0 as #5] + DataSourceExec: partitions=1, partition_sizes=[1] + + + == Physical Plan == + ProjectionExec: expr=[#7@0 as k, #8@1 as n] + ScalarSubqueryExec: subqueries=1 + ProjectionExec: expr=[#2@0 as #7, count(Int64(1)) FILTER (WHERE t.#3 > ())@1 as #8] + AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[count(Int64(1)) FILTER (WHERE t.#3 > ())] + RepartitionExec: partitioning=Hash([#2@0], 4), input_partitions=1 + AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[count(Int64(1)) FILTER (WHERE t.#3 > ())] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#5)@0 as #6] + AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)] + ProjectionExec: expr=[column1@0 as #5] + DataSourceExec: partitions=1, partition_sizes=[1] + + + == Distributed Plan == + === stage 0 === + inputs=[] + mode=Pipelined + partitions=1 + distribution=RoundRobin(channels=1) + placement=Worker + ProjectionExec: expr=[min(s.#5)@0 as #6] + AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)] + ProjectionExec: expr=[column1@0 as #5] + DataSourceExec: partitions=1, partition_sizes=[1] + + === stage 1 === + inputs=[StageInput(stage=0, mode=Broadcast)] + mode=Pipelined + partitions=1 + distribution=Hash(keys=[#2@0], channels=4) + placement=Worker + ScalarSubqueryExec: subqueries=1 + AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[count(Int64(1)) FILTER (WHERE t.#3 > ())] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + StageInputExec: input=0, partitioning=UnknownPartitioning(1) + + === stage 2 === + inputs=[StageInput(stage=1, mode=Shuffle), StageInput(stage=0, mode=Broadcast)] + mode=Pipelined + partitions=4 + distribution=RoundRobin(channels=1) + placement=Worker + ProjectionExec: expr=[#7@0 as k, #8@1 as n] + ScalarSubqueryExec: subqueries=1 + ProjectionExec: expr=[#2@0 as #7, count(Int64(1)) FILTER (WHERE t.#3 > ())@1 as #8] + AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[count(Int64(1)) FILTER (WHERE t.#3 > ())] + StageInputExec: input=0, partitioning=Hash([#2@0], 4) + StageInputExec: input=1, partitioning=UnknownPartitioning(1) +--- +name: "test_scalar_subquery_in_aggregate_order_by" +data: + |- + == Codegen == + Whole-stage codegen is not supported; showing physical plan instead. + + == Plan Steps == + initial_logical_plan: + Projection: CASE WHEN array_to_string(array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST], Utf8(",")) IS NOT NULL THEN array_to_string(array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST], Utf8(",")) WHEN Boolean(true) THEN NULL END AS joined AS #7 + Aggregate: groupBy=[[]], aggr=[[array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST]]] + Subquery: + Projection: min(s.#5) AS #6 + Aggregate: groupBy=[[]], aggr=[[min(s.#5)]] + SubqueryAlias: s + Projection: #4 AS #5 + Projection: column1 AS #4 + Values: (Int32(1)) + SubqueryAlias: t + Projection: #0 AS #2, #1 AS #3 + Projection: column1 AS #0, column2 AS #1 + Values: (Utf8("a"), Int32(2)), (Utf8("b"), Int32(1)), (Utf8("c"), Int32(3)) + + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE + + logical_plan after resolve_grouping_function: + SAME TEXT AS ABOVE + + logical_plan after type_coercion: + Projection: CASE WHEN array_to_string(array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST], Utf8(",")) IS NOT NULL THEN array_to_string(array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST], Utf8(",")) WHEN Boolean(true) THEN CAST(NULL AS Utf8) END AS joined AS #7 + Aggregate: groupBy=[[]], aggr=[[array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST]]] + Subquery: + Projection: min(s.#5) AS #6 + Aggregate: groupBy=[[]], aggr=[[min(s.#5)]] + SubqueryAlias: s + Projection: #4 AS #5 + Projection: column1 AS #4 + Values: (Int32(1)) + SubqueryAlias: t + Projection: #0 AS #2, #1 AS #3 + Projection: column1 AS #0, column2 AS #1 + Values: (Utf8("a"), Int32(2)), (Utf8("b"), Int32(1)), (Utf8("c"), Int32(3)) + + analyzed_logical_plan: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_projection: + SAME TEXT AS ABOVE + + logical_plan after rewrite_set_comparison: + SAME TEXT AS ABOVE + + logical_plan after optimize_unions: + SAME TEXT AS ABOVE + + logical_plan after unions_to_filter: + SAME TEXT AS ABOVE + + logical_plan after simplify_expressions: + Projection: CASE WHEN array_to_string(array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST], Utf8(",")) IS NOT NULL THEN array_to_string(array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST], Utf8(",")) ELSE Utf8(NULL) END AS joined AS #7 + Aggregate: groupBy=[[]], aggr=[[array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 ELSE negative(t.#3) END ASC NULLS FIRST] AS array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST]]] + Subquery: + Projection: min(s.#5) AS #6 + Aggregate: groupBy=[[]], aggr=[[min(s.#5)]] + SubqueryAlias: s + Projection: #4 AS #5 + Projection: column1 AS #4 + Values: (Int32(1)) + SubqueryAlias: t + Projection: #0 AS #2, #1 AS #3 + Projection: column1 AS #0, column2 AS #1 + Values: (Utf8("a"), Int32(2)), (Utf8("b"), Int32(1)), (Utf8("c"), Int32(3)) + + logical_plan after replace_distinct_aggregate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_predicate_subquery: + SAME TEXT AS ABOVE + + logical_plan after scalar_subquery_to_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_join: + SAME TEXT AS ABOVE + + logical_plan after extract_equijoin_predicate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_duplicated_expr: + SAME TEXT AS ABOVE + + logical_plan after eliminate_filter: + SAME TEXT AS ABOVE + + logical_plan after eliminate_cross_join: + SAME TEXT AS ABOVE + + logical_plan after eliminate_limit: + SAME TEXT AS ABOVE + + logical_plan after propagate_empty_relation: + SAME TEXT AS ABOVE + + logical_plan after filter_null_join_keys: + SAME TEXT AS ABOVE + + logical_plan after eliminate_outer_join: + SAME TEXT AS ABOVE + + logical_plan after push_down_limit: + SAME TEXT AS ABOVE + + logical_plan after push_down_filter: + SAME TEXT AS ABOVE + + logical_plan after single_distinct_aggregation_to_group_by: + SAME TEXT AS ABOVE + + logical_plan after eliminate_group_by_constant: + SAME TEXT AS ABOVE + + logical_plan after common_sub_expression_eliminate: + Projection: CASE WHEN __common_expr_1 IS NOT NULL THEN __common_expr_1 ELSE Utf8(NULL) END AS joined AS #7 + Projection: array_to_string(array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST], Utf8(",")) AS __common_expr_1, array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST] + Aggregate: groupBy=[[]], aggr=[[array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 ELSE negative(t.#3) END ASC NULLS FIRST] AS array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST]]] + Subquery: + Projection: min(s.#5) AS #6 + Aggregate: groupBy=[[]], aggr=[[min(s.#5)]] + SubqueryAlias: s + Projection: #4 AS #5 + Projection: column1 AS #4 + Values: (Int32(1)) + SubqueryAlias: t + Projection: #0 AS #2, #1 AS #3 + Projection: column1 AS #0, column2 AS #1 + Values: (Utf8("a"), Int32(2)), (Utf8("b"), Int32(1)), (Utf8("c"), Int32(3)) + + logical_plan after extract_leaf_expressions: + SAME TEXT AS ABOVE + + logical_plan after optimize_projections: + Projection: CASE WHEN __common_expr_1 IS NOT NULL THEN __common_expr_1 ELSE Utf8(NULL) END AS joined AS #7 + Projection: array_to_string(array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST], Utf8(",")) AS __common_expr_1 + Aggregate: groupBy=[[]], aggr=[[array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 ELSE negative(t.#3) END ASC NULLS FIRST] AS array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST]]] + Subquery: + Projection: min(s.#5) AS #6 + Aggregate: groupBy=[[]], aggr=[[min(s.#5)]] + SubqueryAlias: s + Projection: column1 AS #5 + Values: (Int32(1)) + SubqueryAlias: t + Projection: column1 AS #2, column2 AS #3 + Values: (Utf8("a"), Int32(2)), (Utf8("b"), Int32(1)), (Utf8("c"), Int32(3)) + + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_projection: + SAME TEXT AS ABOVE + + logical_plan after rewrite_set_comparison: + SAME TEXT AS ABOVE + + logical_plan after optimize_unions: + SAME TEXT AS ABOVE + + logical_plan after unions_to_filter: + SAME TEXT AS ABOVE + + logical_plan after simplify_expressions: + SAME TEXT AS ABOVE + + logical_plan after replace_distinct_aggregate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_predicate_subquery: + SAME TEXT AS ABOVE + + logical_plan after scalar_subquery_to_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_join: + SAME TEXT AS ABOVE + + logical_plan after extract_equijoin_predicate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_duplicated_expr: + SAME TEXT AS ABOVE + + logical_plan after eliminate_filter: + SAME TEXT AS ABOVE + + logical_plan after eliminate_cross_join: + SAME TEXT AS ABOVE + + logical_plan after eliminate_limit: + SAME TEXT AS ABOVE + + logical_plan after propagate_empty_relation: + SAME TEXT AS ABOVE + + logical_plan after filter_null_join_keys: + SAME TEXT AS ABOVE + + logical_plan after eliminate_outer_join: + SAME TEXT AS ABOVE + + logical_plan after push_down_limit: + SAME TEXT AS ABOVE + + logical_plan after push_down_filter: + SAME TEXT AS ABOVE + + logical_plan after single_distinct_aggregation_to_group_by: + SAME TEXT AS ABOVE + + logical_plan after eliminate_group_by_constant: + SAME TEXT AS ABOVE + + logical_plan after common_sub_expression_eliminate: + SAME TEXT AS ABOVE + + logical_plan after extract_leaf_expressions: + SAME TEXT AS ABOVE + + logical_plan after optimize_projections: + SAME TEXT AS ABOVE + + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE + + logical_plan: + Projection: CASE WHEN __common_expr_1 IS NOT NULL THEN __common_expr_1 ELSE Utf8(NULL) END AS joined AS #7 + Projection: array_to_string(array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST], Utf8(",")) AS __common_expr_1 + Aggregate: groupBy=[[]], aggr=[[array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 ELSE negative(t.#3) END ASC NULLS FIRST] AS array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST]]] + Subquery: + Projection: min(s.#5) AS #6 + Aggregate: groupBy=[[]], aggr=[[min(s.#5)]] + SubqueryAlias: s + Projection: column1 AS #5 + Values: (Int32(1)) + SubqueryAlias: t + Projection: column1 AS #2, column2 AS #3 + Values: (Utf8("a"), Int32(2)), (Utf8("b"), Int32(1)), (Utf8("c"), Int32(3)) + + initial_physical_plan: + ScalarSubqueryExec: subqueries=1 + ProjectionExec: expr=[CASE WHEN __common_expr_1@0 IS NOT NULL THEN __common_expr_1@0 END as #7] + ProjectionExec: expr=[array_to_string(array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST]@0, ,) as __common_expr_1] + AggregateExec: mode=Final, gby=[], aggr=[array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 ELSE negative(t.#3) END ASC NULLS FIRST] as array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST]] + AggregateExec: mode=Partial, gby=[], aggr=[array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 ELSE negative(t.#3) END ASC NULLS FIRST] as array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST]] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#5)@0 as #6] + AggregateExec: mode=Final, gby=[], aggr=[min(s.#5)] + AggregateExec: mode=Partial, gby=[], aggr=[min(s.#5)] + ProjectionExec: expr=[column1@0 as #5] + DataSourceExec: partitions=1, partition_sizes=[1] + + + initial_physical_plan_with_stats: + ScalarSubqueryExec: subqueries=1, statistics=[Rows=Exact(1), Bytes=Inexact(), [(Col[0]:)]] + ProjectionExec: expr=[CASE WHEN __common_expr_1@0 IS NOT NULL THEN __common_expr_1@0 END as #7], statistics=[Rows=Exact(1), Bytes=Inexact(), [(Col[0]:)]] + ProjectionExec: expr=[array_to_string(array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST]@0, ,) as __common_expr_1], statistics=[Rows=Exact(1), Bytes=Inexact(), [(Col[0]:)]] + AggregateExec: mode=Final, gby=[], aggr=[array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 ELSE negative(t.#3) END ASC NULLS FIRST] as array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST]], statistics=[Rows=Exact(1), Bytes=Inexact(), [(Col[0]:)]] + AggregateExec: mode=Partial, gby=[], aggr=[array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 ELSE negative(t.#3) END ASC NULLS FIRST] as array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST]], statistics=[Rows=Inexact(3), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3], statistics=[Rows=Exact(3), Bytes=Inexact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]] + DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(3), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]] + ProjectionExec: expr=[min(s.#5)@0 as #6], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]:)]] + AggregateExec: mode=Final, gby=[], aggr=[min(s.#5)], statistics=[Rows=Exact(1), Bytes=Absent, [(Col[0]:)]] + AggregateExec: mode=Partial, gby=[], aggr=[min(s.#5)], statistics=[Rows=Exact(0), Bytes=Inexact(), [(Col[0]:)]] + ProjectionExec: expr=[column1@0 as #5], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]: Null=Exact(0))]] + DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]: Null=Exact(0))]] + + + initial_physical_plan_with_schema: + ScalarSubqueryExec: subqueries=1, schema=[#7:Utf8;N] + ProjectionExec: expr=[CASE WHEN __common_expr_1@0 IS NOT NULL THEN __common_expr_1@0 END as #7], schema=[#7:Utf8;N] + ProjectionExec: expr=[array_to_string(array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST]@0, ,) as __common_expr_1], schema=[__common_expr_1:Utf8;N] + AggregateExec: mode=Final, gby=[], aggr=[array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 ELSE negative(t.#3) END ASC NULLS FIRST] as array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST]], schema=[array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST]:List(Utf8);N] + AggregateExec: mode=Partial, gby=[], aggr=[array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 ELSE negative(t.#3) END ASC NULLS FIRST] as array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST]], schema=[array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST][array_agg]:List(Utf8);N, array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST][array_agg_orderings]:List(Struct("CASE WHEN scalar_subquery() = 1 THEN #3@1 ELSE negative(#3@1) END": Int32))] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3], schema=[#2:Utf8;N, #3:Int32;N] + DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Utf8;N, column2:Int32;N] + ProjectionExec: expr=[min(s.#5)@0 as #6], schema=[#6:Int32;N] + AggregateExec: mode=Final, gby=[], aggr=[min(s.#5)], schema=[min(s.#5):Int32;N] + AggregateExec: mode=Partial, gby=[], aggr=[min(s.#5)], schema=[min(s.#5)[value]:Int32;N] + ProjectionExec: expr=[column1@0 as #5], schema=[#5:Int32;N] + DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N] + + + physical_plan after OutputRequirements: + OutputRequirementExec: order_by=[], dist_by=Unspecified + ScalarSubqueryExec: subqueries=1 + ProjectionExec: expr=[CASE WHEN __common_expr_1@0 IS NOT NULL THEN __common_expr_1@0 END as #7] + ProjectionExec: expr=[array_to_string(array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST]@0, ,) as __common_expr_1] + AggregateExec: mode=Final, gby=[], aggr=[array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 ELSE negative(t.#3) END ASC NULLS FIRST] as array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST]] + AggregateExec: mode=Partial, gby=[], aggr=[array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 ELSE negative(t.#3) END ASC NULLS FIRST] as array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST]] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#5)@0 as #6] + AggregateExec: mode=Final, gby=[], aggr=[min(s.#5)] + AggregateExec: mode=Partial, gby=[], aggr=[min(s.#5)] + ProjectionExec: expr=[column1@0 as #5] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after aggregate_statistics: + SAME TEXT AS ABOVE + + physical_plan after join_selection: + SAME TEXT AS ABOVE + + physical_plan after LimitedDistinctAggregation: + SAME TEXT AS ABOVE + + physical_plan after FilterPushdown: + SAME TEXT AS ABOVE + + physical_plan after EnforceDistribution: + SAME TEXT AS ABOVE + + physical_plan after CombinePartialFinalAggregate: + OutputRequirementExec: order_by=[], dist_by=Unspecified + ScalarSubqueryExec: subqueries=1 + ProjectionExec: expr=[CASE WHEN __common_expr_1@0 IS NOT NULL THEN __common_expr_1@0 END as #7] + ProjectionExec: expr=[array_to_string(array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST]@0, ,) as __common_expr_1] + AggregateExec: mode=Single, gby=[], aggr=[array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 ELSE negative(t.#3) END ASC NULLS FIRST] as array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST]] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#5)@0 as #6] + AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)] + ProjectionExec: expr=[column1@0 as #5] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after EnforceSorting: + OutputRequirementExec: order_by=[], dist_by=Unspecified + ScalarSubqueryExec: subqueries=1 + ProjectionExec: expr=[CASE WHEN __common_expr_1@0 IS NOT NULL THEN __common_expr_1@0 END as #7] + ProjectionExec: expr=[array_to_string(array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST]@0, ,) as __common_expr_1] + AggregateExec: mode=Single, gby=[], aggr=[array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 ELSE negative(t.#3) END ASC NULLS FIRST] as array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST]] + SortExec: expr=[CASE WHEN scalar_subquery() = 1 THEN #3@1 ELSE negative(#3@1) END ASC], preserve_partitioning=[false] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#5)@0 as #6] + AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)] + ProjectionExec: expr=[column1@0 as #5] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after OptimizeAggregateOrder: + SAME TEXT AS ABOVE + + physical_plan after WindowTopN: + SAME TEXT AS ABOVE + + physical_plan after ProjectionPushdown: + SAME TEXT AS ABOVE + + physical_plan after OutputRequirements: + ScalarSubqueryExec: subqueries=1 + ProjectionExec: expr=[CASE WHEN __common_expr_1@0 IS NOT NULL THEN __common_expr_1@0 END as #7] + ProjectionExec: expr=[array_to_string(array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST]@0, ,) as __common_expr_1] + AggregateExec: mode=Single, gby=[], aggr=[array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 ELSE negative(t.#3) END ASC NULLS FIRST] as array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST]] + SortExec: expr=[CASE WHEN scalar_subquery() = 1 THEN #3@1 ELSE negative(#3@1) END ASC], preserve_partitioning=[false] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#5)@0 as #6] + AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)] + ProjectionExec: expr=[column1@0 as #5] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after LimitAggregation: + SAME TEXT AS ABOVE + + physical_plan after LimitPushPastWindows: + SAME TEXT AS ABOVE + + physical_plan after HashJoinBuffering: + SAME TEXT AS ABOVE + + physical_plan after LimitPushdown: + SAME TEXT AS ABOVE + + physical_plan after TopKRepartition: + SAME TEXT AS ABOVE + + physical_plan after ProjectionPushdown: + SAME TEXT AS ABOVE + + physical_plan after PushdownSort: + SAME TEXT AS ABOVE + + physical_plan after EnsureCooperative: + SAME TEXT AS ABOVE + + physical_plan after FilterPushdown(Post): + SAME TEXT AS ABOVE + + physical_plan after RewriteExplicitRepartition: + SAME TEXT AS ABOVE + + physical_plan after RewriteCollectLeftHashJoin: + SAME TEXT AS ABOVE + + physical_plan after EnforceBarrierPartitioning: + SAME TEXT AS ABOVE + + physical_plan after SanityCheckPlan: + SAME TEXT AS ABOVE + + physical_plan: + ProjectionExec: expr=[#7@0 as joined] + ScalarSubqueryExec: subqueries=1 + ProjectionExec: expr=[CASE WHEN __common_expr_1@0 IS NOT NULL THEN __common_expr_1@0 END as #7] + ProjectionExec: expr=[array_to_string(array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST]@0, ,) as __common_expr_1] + AggregateExec: mode=Single, gby=[], aggr=[array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 ELSE negative(t.#3) END ASC NULLS FIRST] as array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST]] + SortExec: expr=[CASE WHEN scalar_subquery() = 1 THEN #3@1 ELSE negative(#3@1) END ASC], preserve_partitioning=[false] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#5)@0 as #6] + AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)] + ProjectionExec: expr=[column1@0 as #5] + DataSourceExec: partitions=1, partition_sizes=[1] + + + == Physical Plan == + ProjectionExec: expr=[#7@0 as joined] + ScalarSubqueryExec: subqueries=1 + ProjectionExec: expr=[CASE WHEN __common_expr_1@0 IS NOT NULL THEN __common_expr_1@0 END as #7] + ProjectionExec: expr=[array_to_string(array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST]@0, ,) as __common_expr_1] + AggregateExec: mode=Single, gby=[], aggr=[array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 ELSE negative(t.#3) END ASC NULLS FIRST] as array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST]] + SortExec: expr=[CASE WHEN scalar_subquery() = 1 THEN #3@1 ELSE negative(#3@1) END ASC], preserve_partitioning=[false] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#5)@0 as #6] + AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)] + ProjectionExec: expr=[column1@0 as #5] + DataSourceExec: partitions=1, partition_sizes=[1] + + + == Distributed Plan == + === stage 0 === + inputs=[] + mode=Pipelined + partitions=1 + distribution=RoundRobin(channels=1) + placement=Worker + ProjectionExec: expr=[min(s.#5)@0 as #6] + AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)] + ProjectionExec: expr=[column1@0 as #5] + DataSourceExec: partitions=1, partition_sizes=[1] + + === stage 1 === + inputs=[StageInput(stage=0, mode=Broadcast)] + mode=Pipelined + partitions=1 + distribution=RoundRobin(channels=1) + placement=Worker + ProjectionExec: expr=[#7@0 as joined] + ScalarSubqueryExec: subqueries=1 + ProjectionExec: expr=[CASE WHEN __common_expr_1@0 IS NOT NULL THEN __common_expr_1@0 END as #7] + ProjectionExec: expr=[array_to_string(array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST]@0, ,) as __common_expr_1] + AggregateExec: mode=Single, gby=[], aggr=[array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 ELSE negative(t.#3) END ASC NULLS FIRST] as array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST]] + SortExec: expr=[CASE WHEN scalar_subquery() = 1 THEN #3@1 ELSE negative(#3@1) END ASC], preserve_partitioning=[false] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + StageInputExec: input=0, partitioning=UnknownPartitioning(1) +--- +name: "test_scalar_subquery_in_filter_before_aggregate" +data: + |- + == Codegen == + Whole-stage codegen is not supported; showing physical plan instead. + + == Plan Steps == + initial_logical_plan: + Projection: t.#2 AS #7, sum(t.#3) AS total AS #8 + Aggregate: groupBy=[[t.#2]], aggr=[[sum(t.#3)]] + Filter: t.#3 > () + Subquery: + Projection: max(s.#5) AS #6 + Aggregate: groupBy=[[]], aggr=[[max(s.#5)]] + SubqueryAlias: s + Projection: #4 AS #5 + Projection: column1 AS #4 + Values: (Int32(1)), (Int32(2)) + SubqueryAlias: t + Projection: #0 AS #2, #1 AS #3 + Projection: column1 AS #0, column2 AS #1 + Values: (Int32(1), Int32(2)), (Int32(1), Int32(3)), (Int32(2), Int32(10)), (Int32(3), Int32(1)) + + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE + + logical_plan after resolve_grouping_function: + SAME TEXT AS ABOVE + + logical_plan after type_coercion: + Projection: t.#2 AS #7, sum(t.#3) AS total AS #8 + Aggregate: groupBy=[[t.#2]], aggr=[[sum(CAST(t.#3 AS Int64))]] + Filter: t.#3 > () + Subquery: + Projection: max(s.#5) AS #6 + Aggregate: groupBy=[[]], aggr=[[max(s.#5)]] + SubqueryAlias: s + Projection: #4 AS #5 + Projection: column1 AS #4 + Values: (Int32(1)), (Int32(2)) + SubqueryAlias: t + Projection: #0 AS #2, #1 AS #3 + Projection: column1 AS #0, column2 AS #1 + Values: (Int32(1), Int32(2)), (Int32(1), Int32(3)), (Int32(2), Int32(10)), (Int32(3), Int32(1)) + + analyzed_logical_plan: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_projection: + SAME TEXT AS ABOVE + + logical_plan after rewrite_set_comparison: + SAME TEXT AS ABOVE + + logical_plan after optimize_unions: + SAME TEXT AS ABOVE + + logical_plan after unions_to_filter: + SAME TEXT AS ABOVE + + logical_plan after simplify_expressions: + SAME TEXT AS ABOVE + + logical_plan after replace_distinct_aggregate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_predicate_subquery: + SAME TEXT AS ABOVE + + logical_plan after scalar_subquery_to_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_join: + SAME TEXT AS ABOVE + + logical_plan after extract_equijoin_predicate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_duplicated_expr: + SAME TEXT AS ABOVE + + logical_plan after eliminate_filter: + SAME TEXT AS ABOVE + + logical_plan after eliminate_cross_join: + SAME TEXT AS ABOVE + + logical_plan after eliminate_limit: + SAME TEXT AS ABOVE + + logical_plan after propagate_empty_relation: + SAME TEXT AS ABOVE + + logical_plan after filter_null_join_keys: + SAME TEXT AS ABOVE + + logical_plan after eliminate_outer_join: + SAME TEXT AS ABOVE + + logical_plan after push_down_limit: + SAME TEXT AS ABOVE + + logical_plan after push_down_filter: + Projection: t.#2 AS #7, sum(t.#3) AS total AS #8 + Aggregate: groupBy=[[t.#2]], aggr=[[sum(CAST(t.#3 AS Int64))]] + SubqueryAlias: t + Projection: #0 AS #2, #1 AS #3 + Projection: column1 AS #0, column2 AS #1 + Filter: column2 > () + Subquery: + Projection: max(s.#5) AS #6 + Aggregate: groupBy=[[]], aggr=[[max(s.#5)]] + SubqueryAlias: s + Projection: #4 AS #5 + Projection: column1 AS #4 + Values: (Int32(1)), (Int32(2)) + Values: (Int32(1), Int32(2)), (Int32(1), Int32(3)), (Int32(2), Int32(10)), (Int32(3), Int32(1)) + + logical_plan after single_distinct_aggregation_to_group_by: + SAME TEXT AS ABOVE + + logical_plan after eliminate_group_by_constant: + SAME TEXT AS ABOVE + + logical_plan after common_sub_expression_eliminate: + SAME TEXT AS ABOVE + + logical_plan after extract_leaf_expressions: + SAME TEXT AS ABOVE + + logical_plan after optimize_projections: + Projection: t.#2 AS #7, sum(t.#3) AS total AS #8 + Aggregate: groupBy=[[t.#2]], aggr=[[sum(CAST(t.#3 AS Int64))]] + SubqueryAlias: t + Projection: column1 AS #2, column2 AS #3 + Filter: column2 > () + Subquery: + Projection: max(s.#5) AS #6 + Aggregate: groupBy=[[]], aggr=[[max(s.#5)]] + SubqueryAlias: s + Projection: column1 AS #5 + Values: (Int32(1)), (Int32(2)) + Values: (Int32(1), Int32(2)), (Int32(1), Int32(3)), (Int32(2), Int32(10)), (Int32(3), Int32(1)) + + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_projection: + SAME TEXT AS ABOVE + + logical_plan after rewrite_set_comparison: + SAME TEXT AS ABOVE + + logical_plan after optimize_unions: + SAME TEXT AS ABOVE + + logical_plan after unions_to_filter: + SAME TEXT AS ABOVE + + logical_plan after simplify_expressions: + SAME TEXT AS ABOVE + + logical_plan after replace_distinct_aggregate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_predicate_subquery: + SAME TEXT AS ABOVE + + logical_plan after scalar_subquery_to_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_join: + SAME TEXT AS ABOVE + + logical_plan after extract_equijoin_predicate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_duplicated_expr: + SAME TEXT AS ABOVE + + logical_plan after eliminate_filter: + SAME TEXT AS ABOVE + + logical_plan after eliminate_cross_join: + SAME TEXT AS ABOVE + + logical_plan after eliminate_limit: + SAME TEXT AS ABOVE + + logical_plan after propagate_empty_relation: + SAME TEXT AS ABOVE + + logical_plan after filter_null_join_keys: + SAME TEXT AS ABOVE + + logical_plan after eliminate_outer_join: + SAME TEXT AS ABOVE + + logical_plan after push_down_limit: + SAME TEXT AS ABOVE + + logical_plan after push_down_filter: + SAME TEXT AS ABOVE + + logical_plan after single_distinct_aggregation_to_group_by: + SAME TEXT AS ABOVE + + logical_plan after eliminate_group_by_constant: + SAME TEXT AS ABOVE + + logical_plan after common_sub_expression_eliminate: + SAME TEXT AS ABOVE + + logical_plan after extract_leaf_expressions: + SAME TEXT AS ABOVE + + logical_plan after optimize_projections: + SAME TEXT AS ABOVE + + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE + + logical_plan: + Projection: t.#2 AS #7, sum(t.#3) AS total AS #8 + Aggregate: groupBy=[[t.#2]], aggr=[[sum(CAST(t.#3 AS Int64))]] + SubqueryAlias: t + Projection: column1 AS #2, column2 AS #3 + Filter: column2 > () + Subquery: + Projection: max(s.#5) AS #6 + Aggregate: groupBy=[[]], aggr=[[max(s.#5)]] + SubqueryAlias: s + Projection: column1 AS #5 + Values: (Int32(1)), (Int32(2)) + Values: (Int32(1), Int32(2)), (Int32(1), Int32(3)), (Int32(2), Int32(10)), (Int32(3), Int32(1)) + + initial_physical_plan: + ScalarSubqueryExec: subqueries=1 + ProjectionExec: expr=[#2@0 as #7, sum(t.#3)@1 as #8] + AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[sum(t.#3)] + AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3)] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + FilterExec: column2@1 > scalar_subquery() + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[max(s.#5)@0 as #6] + AggregateExec: mode=Final, gby=[], aggr=[max(s.#5)] + AggregateExec: mode=Partial, gby=[], aggr=[max(s.#5)] + ProjectionExec: expr=[column1@0 as #5] + DataSourceExec: partitions=1, partition_sizes=[1] + + + initial_physical_plan_with_stats: + ScalarSubqueryExec: subqueries=1, statistics=[Rows=Inexact(1), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]] + ProjectionExec: expr=[#2@0 as #7, sum(t.#3)@1 as #8], statistics=[Rows=Inexact(1), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]] + AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[sum(t.#3)], statistics=[Rows=Inexact(1), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]] + AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3)], statistics=[Rows=Inexact(1), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3], statistics=[Rows=Inexact(1), Bytes=Inexact(), [(Col[0]: Null=Inexact(0)),(Col[1]: Null=Inexact(0))]] + FilterExec: column2@1 > scalar_subquery(), statistics=[Rows=Inexact(1), Bytes=Inexact(), [(Col[0]: Null=Inexact(0)),(Col[1]: Null=Inexact(0))]] + DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(4), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]] + ProjectionExec: expr=[max(s.#5)@0 as #6], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]:)]] + AggregateExec: mode=Final, gby=[], aggr=[max(s.#5)], statistics=[Rows=Exact(1), Bytes=Inexact(), [(Col[0]:)]] + AggregateExec: mode=Partial, gby=[], aggr=[max(s.#5)], statistics=[Rows=Inexact(2), Bytes=Inexact(), [(Col[0]:)]] + ProjectionExec: expr=[column1@0 as #5], statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0))]] + DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0))]] + + + initial_physical_plan_with_schema: + ScalarSubqueryExec: subqueries=1, schema=[#7:Int32;N, #8:Int64;N] + ProjectionExec: expr=[#2@0 as #7, sum(t.#3)@1 as #8], schema=[#7:Int32;N, #8:Int64;N] + AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[sum(t.#3)], schema=[#2:Int32;N, sum(t.#3):Int64;N] + AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3)], schema=[#2:Int32;N, sum(t.#3)[sum]:Int64;N] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3], schema=[#2:Int32;N, #3:Int32;N] + FilterExec: column2@1 > scalar_subquery(), schema=[column1:Int32;N, column2:Int32;N] + DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N, column2:Int32;N] + ProjectionExec: expr=[max(s.#5)@0 as #6], schema=[#6:Int32;N] + AggregateExec: mode=Final, gby=[], aggr=[max(s.#5)], schema=[max(s.#5):Int32;N] + AggregateExec: mode=Partial, gby=[], aggr=[max(s.#5)], schema=[max(s.#5)[value]:Int32;N] + ProjectionExec: expr=[column1@0 as #5], schema=[#5:Int32;N] + DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N] + + + physical_plan after OutputRequirements: + OutputRequirementExec: order_by=[], dist_by=Unspecified + ScalarSubqueryExec: subqueries=1 + ProjectionExec: expr=[#2@0 as #7, sum(t.#3)@1 as #8] + AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[sum(t.#3)] + AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3)] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + FilterExec: column2@1 > scalar_subquery() + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[max(s.#5)@0 as #6] + AggregateExec: mode=Final, gby=[], aggr=[max(s.#5)] + AggregateExec: mode=Partial, gby=[], aggr=[max(s.#5)] + ProjectionExec: expr=[column1@0 as #5] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after aggregate_statistics: + SAME TEXT AS ABOVE + + physical_plan after join_selection: + SAME TEXT AS ABOVE + + physical_plan after LimitedDistinctAggregation: + SAME TEXT AS ABOVE + + physical_plan after FilterPushdown: + SAME TEXT AS ABOVE + + physical_plan after EnforceDistribution: + OutputRequirementExec: order_by=[], dist_by=Unspecified + ScalarSubqueryExec: subqueries=1 + ProjectionExec: expr=[#2@0 as #7, sum(t.#3)@1 as #8] + AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[sum(t.#3)] + RepartitionExec: partitioning=Hash([#2@0], 4), input_partitions=4 + AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3)] + RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + FilterExec: column2@1 > scalar_subquery() + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[max(s.#5)@0 as #6] + AggregateExec: mode=Final, gby=[], aggr=[max(s.#5)] + AggregateExec: mode=Partial, gby=[], aggr=[max(s.#5)] + ProjectionExec: expr=[column1@0 as #5] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after CombinePartialFinalAggregate: + OutputRequirementExec: order_by=[], dist_by=Unspecified + ScalarSubqueryExec: subqueries=1 + ProjectionExec: expr=[#2@0 as #7, sum(t.#3)@1 as #8] + AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[sum(t.#3)] + RepartitionExec: partitioning=Hash([#2@0], 4), input_partitions=4 + AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3)] + RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + FilterExec: column2@1 > scalar_subquery() + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[max(s.#5)@0 as #6] + AggregateExec: mode=Single, gby=[], aggr=[max(s.#5)] + ProjectionExec: expr=[column1@0 as #5] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after EnforceSorting: + SAME TEXT AS ABOVE + + physical_plan after OptimizeAggregateOrder: + SAME TEXT AS ABOVE + + physical_plan after WindowTopN: + SAME TEXT AS ABOVE + + physical_plan after ProjectionPushdown: + SAME TEXT AS ABOVE + + physical_plan after OutputRequirements: + ScalarSubqueryExec: subqueries=1 + ProjectionExec: expr=[#2@0 as #7, sum(t.#3)@1 as #8] + AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[sum(t.#3)] + RepartitionExec: partitioning=Hash([#2@0], 4), input_partitions=4 + AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3)] + RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + FilterExec: column2@1 > scalar_subquery() + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[max(s.#5)@0 as #6] + AggregateExec: mode=Single, gby=[], aggr=[max(s.#5)] + ProjectionExec: expr=[column1@0 as #5] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after LimitAggregation: + SAME TEXT AS ABOVE + + physical_plan after LimitPushPastWindows: + SAME TEXT AS ABOVE + + physical_plan after HashJoinBuffering: + SAME TEXT AS ABOVE + + physical_plan after LimitPushdown: + SAME TEXT AS ABOVE + + physical_plan after TopKRepartition: + SAME TEXT AS ABOVE + + physical_plan after ProjectionPushdown: + SAME TEXT AS ABOVE + + physical_plan after PushdownSort: + SAME TEXT AS ABOVE + + physical_plan after EnsureCooperative: + SAME TEXT AS ABOVE + + physical_plan after FilterPushdown(Post): + SAME TEXT AS ABOVE + + physical_plan after RewriteExplicitRepartition: + SAME TEXT AS ABOVE + + physical_plan after RewriteCollectLeftHashJoin: + SAME TEXT AS ABOVE + + physical_plan after EnforceBarrierPartitioning: + SAME TEXT AS ABOVE + + physical_plan after SanityCheckPlan: + SAME TEXT AS ABOVE + + physical_plan: + ProjectionExec: expr=[#7@0 as k, #8@1 as total] + ScalarSubqueryExec: subqueries=1 + ProjectionExec: expr=[#2@0 as #7, sum(t.#3)@1 as #8] + AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[sum(t.#3)] + RepartitionExec: partitioning=Hash([#2@0], 4), input_partitions=4 + AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3)] + RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + FilterExec: column2@1 > scalar_subquery() + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[max(s.#5)@0 as #6] + AggregateExec: mode=Single, gby=[], aggr=[max(s.#5)] + ProjectionExec: expr=[column1@0 as #5] + DataSourceExec: partitions=1, partition_sizes=[1] + + + == Physical Plan == + ProjectionExec: expr=[#7@0 as k, #8@1 as total] + ScalarSubqueryExec: subqueries=1 + ProjectionExec: expr=[#2@0 as #7, sum(t.#3)@1 as #8] + AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[sum(t.#3)] + RepartitionExec: partitioning=Hash([#2@0], 4), input_partitions=4 + AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3)] + RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + FilterExec: column2@1 > scalar_subquery() + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[max(s.#5)@0 as #6] + AggregateExec: mode=Single, gby=[], aggr=[max(s.#5)] + ProjectionExec: expr=[column1@0 as #5] + DataSourceExec: partitions=1, partition_sizes=[1] + + + == Distributed Plan == + === stage 0 === + inputs=[] + mode=Pipelined + partitions=1 + distribution=RoundRobin(channels=1) + placement=Worker + ProjectionExec: expr=[max(s.#5)@0 as #6] + AggregateExec: mode=Single, gby=[], aggr=[max(s.#5)] + ProjectionExec: expr=[column1@0 as #5] + DataSourceExec: partitions=1, partition_sizes=[1] + + === stage 1 === + inputs=[StageInput(stage=0, mode=Broadcast)] + mode=Pipelined + partitions=1 + distribution=RoundRobin(channels=4) + placement=Worker + ScalarSubqueryExec: subqueries=1 + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + FilterExec: column2@1 > scalar_subquery() + DataSourceExec: partitions=1, partition_sizes=[1] + StageInputExec: input=0, partitioning=UnknownPartitioning(1) + + === stage 2 === + inputs=[StageInput(stage=1, mode=Shuffle)] + mode=Pipelined + partitions=4 + distribution=Hash(keys=[#2@0], channels=4) + placement=Worker + AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3)] + StageInputExec: input=0, partitioning=RoundRobinBatch(4) + + === stage 3 === + inputs=[StageInput(stage=2, mode=Shuffle)] + mode=Pipelined + partitions=4 + distribution=RoundRobin(channels=1) + placement=Worker + ProjectionExec: expr=[#7@0 as k, #8@1 as total] + ProjectionExec: expr=[#2@0 as #7, sum(t.#3)@1 as #8] + AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[sum(t.#3)] + StageInputExec: input=0, partitioning=Hash([#2@0], 4) +--- +name: "test_scalar_subquery_in_group_by_expression" +data: + |- + == Codegen == + Whole-stage codegen is not supported; showing physical plan instead. + + == Plan Steps == + initial_logical_plan: + Projection: g AS #5, count(Int64(1)) AS n AS #6 + Aggregate: groupBy=[[CAST(t.#1 AS Int64) + () AS g]], aggr=[[count(Int64(1))]] + Subquery: + Projection: min(s.#3) AS #4 + Aggregate: groupBy=[[]], aggr=[[min(CAST(s.#3 AS Int64))]] + SubqueryAlias: s + Projection: #2 AS #3 + Projection: column1 AS #2 + Values: (Int32(10)), (Int32(20)) + SubqueryAlias: t + Projection: #0 AS #1 + Projection: column1 AS #0 + Values: (Int32(1)), (Int32(2)), (Int32(1)) + + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE + + logical_plan after resolve_grouping_function: + SAME TEXT AS ABOVE + + logical_plan after type_coercion: + SAME TEXT AS ABOVE + + analyzed_logical_plan: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_projection: + SAME TEXT AS ABOVE + + logical_plan after rewrite_set_comparison: + SAME TEXT AS ABOVE + + logical_plan after optimize_unions: + SAME TEXT AS ABOVE + + logical_plan after unions_to_filter: + SAME TEXT AS ABOVE + + logical_plan after simplify_expressions: + SAME TEXT AS ABOVE + + logical_plan after replace_distinct_aggregate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_predicate_subquery: + SAME TEXT AS ABOVE + + logical_plan after scalar_subquery_to_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_join: + SAME TEXT AS ABOVE + + logical_plan after extract_equijoin_predicate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_duplicated_expr: + SAME TEXT AS ABOVE + + logical_plan after eliminate_filter: + SAME TEXT AS ABOVE + + logical_plan after eliminate_cross_join: + SAME TEXT AS ABOVE + + logical_plan after eliminate_limit: + SAME TEXT AS ABOVE + + logical_plan after propagate_empty_relation: + SAME TEXT AS ABOVE + + logical_plan after filter_null_join_keys: + SAME TEXT AS ABOVE + + logical_plan after eliminate_outer_join: + SAME TEXT AS ABOVE + + logical_plan after push_down_limit: + SAME TEXT AS ABOVE + + logical_plan after push_down_filter: + SAME TEXT AS ABOVE + + logical_plan after single_distinct_aggregation_to_group_by: + SAME TEXT AS ABOVE + + logical_plan after eliminate_group_by_constant: + SAME TEXT AS ABOVE + + logical_plan after common_sub_expression_eliminate: + SAME TEXT AS ABOVE + + logical_plan after extract_leaf_expressions: + SAME TEXT AS ABOVE + + logical_plan after optimize_projections: + Projection: g AS #5, count(Int64(1)) AS n AS #6 + Aggregate: groupBy=[[CAST(t.#1 AS Int64) + () AS g]], aggr=[[count(Int64(1))]] + Subquery: + Projection: min(s.#3) AS #4 + Aggregate: groupBy=[[]], aggr=[[min(CAST(s.#3 AS Int64))]] + SubqueryAlias: s + Projection: column1 AS #3 + Values: (Int32(10)), (Int32(20)) + SubqueryAlias: t + Projection: column1 AS #1 + Values: (Int32(1)), (Int32(2)), (Int32(1)) + + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_projection: + SAME TEXT AS ABOVE + + logical_plan after rewrite_set_comparison: + SAME TEXT AS ABOVE + + logical_plan after optimize_unions: + SAME TEXT AS ABOVE + + logical_plan after unions_to_filter: + SAME TEXT AS ABOVE + + logical_plan after simplify_expressions: + SAME TEXT AS ABOVE + + logical_plan after replace_distinct_aggregate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_predicate_subquery: + SAME TEXT AS ABOVE + + logical_plan after scalar_subquery_to_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_join: + SAME TEXT AS ABOVE + + logical_plan after extract_equijoin_predicate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_duplicated_expr: + SAME TEXT AS ABOVE + + logical_plan after eliminate_filter: + SAME TEXT AS ABOVE + + logical_plan after eliminate_cross_join: + SAME TEXT AS ABOVE + + logical_plan after eliminate_limit: + SAME TEXT AS ABOVE + + logical_plan after propagate_empty_relation: + SAME TEXT AS ABOVE + + logical_plan after filter_null_join_keys: + SAME TEXT AS ABOVE + + logical_plan after eliminate_outer_join: + SAME TEXT AS ABOVE + + logical_plan after push_down_limit: + SAME TEXT AS ABOVE + + logical_plan after push_down_filter: + SAME TEXT AS ABOVE + + logical_plan after single_distinct_aggregation_to_group_by: + SAME TEXT AS ABOVE + + logical_plan after eliminate_group_by_constant: + SAME TEXT AS ABOVE + + logical_plan after common_sub_expression_eliminate: + SAME TEXT AS ABOVE + + logical_plan after extract_leaf_expressions: + SAME TEXT AS ABOVE + + logical_plan after optimize_projections: + SAME TEXT AS ABOVE + + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE + + logical_plan: + Projection: g AS #5, count(Int64(1)) AS n AS #6 + Aggregate: groupBy=[[CAST(t.#1 AS Int64) + () AS g]], aggr=[[count(Int64(1))]] + Subquery: + Projection: min(s.#3) AS #4 + Aggregate: groupBy=[[]], aggr=[[min(CAST(s.#3 AS Int64))]] + SubqueryAlias: s + Projection: column1 AS #3 + Values: (Int32(10)), (Int32(20)) + SubqueryAlias: t + Projection: column1 AS #1 + Values: (Int32(1)), (Int32(2)), (Int32(1)) + + initial_physical_plan: + ScalarSubqueryExec: subqueries=1 + ProjectionExec: expr=[g@0 as #5, count(Int64(1))@1 as #6] + AggregateExec: mode=FinalPartitioned, gby=[g@0 as g], aggr=[count(Int64(1))] + AggregateExec: mode=Partial, gby=[CAST(#1@0 AS Int64) + scalar_subquery() as g], aggr=[count(Int64(1))] + ProjectionExec: expr=[column1@0 as #1] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#3)@0 as #4] + AggregateExec: mode=Final, gby=[], aggr=[min(s.#3)] + AggregateExec: mode=Partial, gby=[], aggr=[min(s.#3)] + ProjectionExec: expr=[column1@0 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + + + initial_physical_plan_with_stats: + ScalarSubqueryExec: subqueries=1, statistics=[Rows=Inexact(3), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]] + ProjectionExec: expr=[g@0 as #5, count(Int64(1))@1 as #6], statistics=[Rows=Inexact(3), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]] + AggregateExec: mode=FinalPartitioned, gby=[g@0 as g], aggr=[count(Int64(1))], statistics=[Rows=Inexact(3), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]] + AggregateExec: mode=Partial, gby=[CAST(#1@0 AS Int64) + scalar_subquery() as g], aggr=[count(Int64(1))], statistics=[Rows=Inexact(3), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]] + ProjectionExec: expr=[column1@0 as #1], statistics=[Rows=Exact(3), Bytes=Exact(), [(Col[0]: Null=Exact(0))]] + DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(3), Bytes=Exact(), [(Col[0]: Null=Exact(0))]] + ProjectionExec: expr=[min(s.#3)@0 as #4], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]:)]] + AggregateExec: mode=Final, gby=[], aggr=[min(s.#3)], statistics=[Rows=Exact(1), Bytes=Inexact(), [(Col[0]:)]] + AggregateExec: mode=Partial, gby=[], aggr=[min(s.#3)], statistics=[Rows=Inexact(2), Bytes=Inexact(), [(Col[0]:)]] + ProjectionExec: expr=[column1@0 as #3], statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0))]] + DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0))]] + + + initial_physical_plan_with_schema: + ScalarSubqueryExec: subqueries=1, schema=[#5:Int64;N, #6:Int64] + ProjectionExec: expr=[g@0 as #5, count(Int64(1))@1 as #6], schema=[#5:Int64;N, #6:Int64] + AggregateExec: mode=FinalPartitioned, gby=[g@0 as g], aggr=[count(Int64(1))], schema=[g:Int64;N, count(Int64(1)):Int64] + AggregateExec: mode=Partial, gby=[CAST(#1@0 AS Int64) + scalar_subquery() as g], aggr=[count(Int64(1))], schema=[g:Int64;N, count(Int64(1))[count]:Int64] + ProjectionExec: expr=[column1@0 as #1], schema=[#1:Int32;N] + DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N] + ProjectionExec: expr=[min(s.#3)@0 as #4], schema=[#4:Int64;N] + AggregateExec: mode=Final, gby=[], aggr=[min(s.#3)], schema=[min(s.#3):Int64;N] + AggregateExec: mode=Partial, gby=[], aggr=[min(s.#3)], schema=[min(s.#3)[value]:Int64;N] + ProjectionExec: expr=[column1@0 as #3], schema=[#3:Int32;N] + DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N] + + + physical_plan after OutputRequirements: + OutputRequirementExec: order_by=[], dist_by=Unspecified + ScalarSubqueryExec: subqueries=1 + ProjectionExec: expr=[g@0 as #5, count(Int64(1))@1 as #6] + AggregateExec: mode=FinalPartitioned, gby=[g@0 as g], aggr=[count(Int64(1))] + AggregateExec: mode=Partial, gby=[CAST(#1@0 AS Int64) + scalar_subquery() as g], aggr=[count(Int64(1))] + ProjectionExec: expr=[column1@0 as #1] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#3)@0 as #4] + AggregateExec: mode=Final, gby=[], aggr=[min(s.#3)] + AggregateExec: mode=Partial, gby=[], aggr=[min(s.#3)] + ProjectionExec: expr=[column1@0 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after aggregate_statistics: + SAME TEXT AS ABOVE + + physical_plan after join_selection: + SAME TEXT AS ABOVE + + physical_plan after LimitedDistinctAggregation: + SAME TEXT AS ABOVE + + physical_plan after FilterPushdown: + SAME TEXT AS ABOVE + + physical_plan after EnforceDistribution: + OutputRequirementExec: order_by=[], dist_by=Unspecified + ScalarSubqueryExec: subqueries=1 + ProjectionExec: expr=[g@0 as #5, count(Int64(1))@1 as #6] + AggregateExec: mode=FinalPartitioned, gby=[g@0 as g], aggr=[count(Int64(1))] + RepartitionExec: partitioning=Hash([g@0], 4), input_partitions=1 + AggregateExec: mode=Partial, gby=[CAST(#1@0 AS Int64) + scalar_subquery() as g], aggr=[count(Int64(1))] + ProjectionExec: expr=[column1@0 as #1] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#3)@0 as #4] + AggregateExec: mode=Final, gby=[], aggr=[min(s.#3)] + AggregateExec: mode=Partial, gby=[], aggr=[min(s.#3)] + ProjectionExec: expr=[column1@0 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after CombinePartialFinalAggregate: + OutputRequirementExec: order_by=[], dist_by=Unspecified + ScalarSubqueryExec: subqueries=1 + ProjectionExec: expr=[g@0 as #5, count(Int64(1))@1 as #6] + AggregateExec: mode=FinalPartitioned, gby=[g@0 as g], aggr=[count(Int64(1))] + RepartitionExec: partitioning=Hash([g@0], 4), input_partitions=1 + AggregateExec: mode=Partial, gby=[CAST(#1@0 AS Int64) + scalar_subquery() as g], aggr=[count(Int64(1))] + ProjectionExec: expr=[column1@0 as #1] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#3)@0 as #4] + AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)] + ProjectionExec: expr=[column1@0 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after EnforceSorting: + SAME TEXT AS ABOVE + + physical_plan after OptimizeAggregateOrder: + SAME TEXT AS ABOVE + + physical_plan after WindowTopN: + SAME TEXT AS ABOVE + + physical_plan after ProjectionPushdown: + SAME TEXT AS ABOVE + + physical_plan after OutputRequirements: + ScalarSubqueryExec: subqueries=1 + ProjectionExec: expr=[g@0 as #5, count(Int64(1))@1 as #6] + AggregateExec: mode=FinalPartitioned, gby=[g@0 as g], aggr=[count(Int64(1))] + RepartitionExec: partitioning=Hash([g@0], 4), input_partitions=1 + AggregateExec: mode=Partial, gby=[CAST(#1@0 AS Int64) + scalar_subquery() as g], aggr=[count(Int64(1))] + ProjectionExec: expr=[column1@0 as #1] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#3)@0 as #4] + AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)] + ProjectionExec: expr=[column1@0 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after LimitAggregation: + SAME TEXT AS ABOVE + + physical_plan after LimitPushPastWindows: + SAME TEXT AS ABOVE + + physical_plan after HashJoinBuffering: + SAME TEXT AS ABOVE + + physical_plan after LimitPushdown: + SAME TEXT AS ABOVE + + physical_plan after TopKRepartition: + SAME TEXT AS ABOVE + + physical_plan after ProjectionPushdown: + SAME TEXT AS ABOVE + + physical_plan after PushdownSort: + SAME TEXT AS ABOVE + + physical_plan after EnsureCooperative: + SAME TEXT AS ABOVE + + physical_plan after FilterPushdown(Post): + SAME TEXT AS ABOVE + + physical_plan after RewriteExplicitRepartition: + SAME TEXT AS ABOVE + + physical_plan after RewriteCollectLeftHashJoin: + SAME TEXT AS ABOVE + + physical_plan after EnforceBarrierPartitioning: + SAME TEXT AS ABOVE + + physical_plan after SanityCheckPlan: + SAME TEXT AS ABOVE + + physical_plan: + ProjectionExec: expr=[#5@0 as g, #6@1 as n] + ScalarSubqueryExec: subqueries=1 + ProjectionExec: expr=[g@0 as #5, count(Int64(1))@1 as #6] + AggregateExec: mode=FinalPartitioned, gby=[g@0 as g], aggr=[count(Int64(1))] + RepartitionExec: partitioning=Hash([g@0], 4), input_partitions=1 + AggregateExec: mode=Partial, gby=[CAST(#1@0 AS Int64) + scalar_subquery() as g], aggr=[count(Int64(1))] + ProjectionExec: expr=[column1@0 as #1] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#3)@0 as #4] + AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)] + ProjectionExec: expr=[column1@0 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + + + == Physical Plan == + ProjectionExec: expr=[#5@0 as g, #6@1 as n] + ScalarSubqueryExec: subqueries=1 + ProjectionExec: expr=[g@0 as #5, count(Int64(1))@1 as #6] + AggregateExec: mode=FinalPartitioned, gby=[g@0 as g], aggr=[count(Int64(1))] + RepartitionExec: partitioning=Hash([g@0], 4), input_partitions=1 + AggregateExec: mode=Partial, gby=[CAST(#1@0 AS Int64) + scalar_subquery() as g], aggr=[count(Int64(1))] + ProjectionExec: expr=[column1@0 as #1] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#3)@0 as #4] + AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)] + ProjectionExec: expr=[column1@0 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + + + == Distributed Plan == + === stage 0 === + inputs=[] + mode=Pipelined + partitions=1 + distribution=RoundRobin(channels=1) + placement=Worker + ProjectionExec: expr=[min(s.#3)@0 as #4] + AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)] + ProjectionExec: expr=[column1@0 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + + === stage 1 === + inputs=[StageInput(stage=0, mode=Broadcast)] + mode=Pipelined + partitions=1 + distribution=Hash(keys=[g@0], channels=4) + placement=Worker + ScalarSubqueryExec: subqueries=1 + AggregateExec: mode=Partial, gby=[CAST(#1@0 AS Int64) + scalar_subquery() as g], aggr=[count(Int64(1))] + ProjectionExec: expr=[column1@0 as #1] + DataSourceExec: partitions=1, partition_sizes=[1] + StageInputExec: input=0, partitioning=UnknownPartitioning(1) + + === stage 2 === + inputs=[StageInput(stage=1, mode=Shuffle)] + mode=Pipelined + partitions=4 + distribution=RoundRobin(channels=1) + placement=Worker + ProjectionExec: expr=[#5@0 as g, #6@1 as n] + ProjectionExec: expr=[g@0 as #5, count(Int64(1))@1 as #6] + AggregateExec: mode=FinalPartitioned, gby=[g@0 as g], aggr=[count(Int64(1))] + StageInputExec: input=0, partitioning=Hash([g@0], 4) +--- +name: "test_scalar_subquery_in_grouped_aggregate_expression" +data: + |- + == Codegen == + Whole-stage codegen is not supported; showing physical plan instead. + + == Plan Steps == + initial_logical_plan: + Projection: t.#2 AS #7, sum(t.#3 + #6) AS total AS #8 + Aggregate: groupBy=[[t.#2]], aggr=[[sum(CAST(t.#3 AS Int64) + ())]] + Subquery: + Projection: min(s.#5) AS #6 + Aggregate: groupBy=[[]], aggr=[[min(CAST(s.#5 AS Int64))]] + SubqueryAlias: s + Projection: #4 AS #5 + Projection: column1 AS #4 + Values: (Int32(10)), (Int32(20)) + SubqueryAlias: t + Projection: #0 AS #2, #1 AS #3 + Projection: column1 AS #0, column2 AS #1 + Values: (Int32(1), Int32(1)), (Int32(1), Int32(2)), (Int32(2), Int32(3)) + + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE + + logical_plan after resolve_grouping_function: + SAME TEXT AS ABOVE + + logical_plan after type_coercion: + SAME TEXT AS ABOVE + + analyzed_logical_plan: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_projection: + SAME TEXT AS ABOVE + + logical_plan after rewrite_set_comparison: + SAME TEXT AS ABOVE + + logical_plan after optimize_unions: + SAME TEXT AS ABOVE + + logical_plan after unions_to_filter: + SAME TEXT AS ABOVE + + logical_plan after simplify_expressions: + SAME TEXT AS ABOVE + + logical_plan after replace_distinct_aggregate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_predicate_subquery: + SAME TEXT AS ABOVE + + logical_plan after scalar_subquery_to_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_join: + SAME TEXT AS ABOVE + + logical_plan after extract_equijoin_predicate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_duplicated_expr: + SAME TEXT AS ABOVE + + logical_plan after eliminate_filter: + SAME TEXT AS ABOVE + + logical_plan after eliminate_cross_join: + SAME TEXT AS ABOVE + + logical_plan after eliminate_limit: + SAME TEXT AS ABOVE + + logical_plan after propagate_empty_relation: + SAME TEXT AS ABOVE + + logical_plan after filter_null_join_keys: + SAME TEXT AS ABOVE + + logical_plan after eliminate_outer_join: + SAME TEXT AS ABOVE + + logical_plan after push_down_limit: + SAME TEXT AS ABOVE + + logical_plan after push_down_filter: + SAME TEXT AS ABOVE + + logical_plan after single_distinct_aggregation_to_group_by: + SAME TEXT AS ABOVE + + logical_plan after eliminate_group_by_constant: + SAME TEXT AS ABOVE + + logical_plan after common_sub_expression_eliminate: + SAME TEXT AS ABOVE + + logical_plan after extract_leaf_expressions: + SAME TEXT AS ABOVE + + logical_plan after optimize_projections: + Projection: t.#2 AS #7, sum(t.#3 + #6) AS total AS #8 + Aggregate: groupBy=[[t.#2]], aggr=[[sum(CAST(t.#3 AS Int64) + ())]] + Subquery: + Projection: min(s.#5) AS #6 + Aggregate: groupBy=[[]], aggr=[[min(CAST(s.#5 AS Int64))]] + SubqueryAlias: s + Projection: column1 AS #5 + Values: (Int32(10)), (Int32(20)) + SubqueryAlias: t + Projection: column1 AS #2, column2 AS #3 + Values: (Int32(1), Int32(1)), (Int32(1), Int32(2)), (Int32(2), Int32(3)) + + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_projection: + SAME TEXT AS ABOVE + + logical_plan after rewrite_set_comparison: + SAME TEXT AS ABOVE + + logical_plan after optimize_unions: + SAME TEXT AS ABOVE + + logical_plan after unions_to_filter: + SAME TEXT AS ABOVE + + logical_plan after simplify_expressions: + SAME TEXT AS ABOVE + + logical_plan after replace_distinct_aggregate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_predicate_subquery: + SAME TEXT AS ABOVE + + logical_plan after scalar_subquery_to_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_join: + SAME TEXT AS ABOVE + + logical_plan after extract_equijoin_predicate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_duplicated_expr: + SAME TEXT AS ABOVE + + logical_plan after eliminate_filter: + SAME TEXT AS ABOVE + + logical_plan after eliminate_cross_join: + SAME TEXT AS ABOVE + + logical_plan after eliminate_limit: + SAME TEXT AS ABOVE + + logical_plan after propagate_empty_relation: + SAME TEXT AS ABOVE + + logical_plan after filter_null_join_keys: + SAME TEXT AS ABOVE + + logical_plan after eliminate_outer_join: + SAME TEXT AS ABOVE + + logical_plan after push_down_limit: + SAME TEXT AS ABOVE + + logical_plan after push_down_filter: + SAME TEXT AS ABOVE + + logical_plan after single_distinct_aggregation_to_group_by: + SAME TEXT AS ABOVE + + logical_plan after eliminate_group_by_constant: + SAME TEXT AS ABOVE + + logical_plan after common_sub_expression_eliminate: + SAME TEXT AS ABOVE + + logical_plan after extract_leaf_expressions: + SAME TEXT AS ABOVE + + logical_plan after optimize_projections: + SAME TEXT AS ABOVE + + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE + + logical_plan: + Projection: t.#2 AS #7, sum(t.#3 + #6) AS total AS #8 + Aggregate: groupBy=[[t.#2]], aggr=[[sum(CAST(t.#3 AS Int64) + ())]] + Subquery: + Projection: min(s.#5) AS #6 + Aggregate: groupBy=[[]], aggr=[[min(CAST(s.#5 AS Int64))]] + SubqueryAlias: s + Projection: column1 AS #5 + Values: (Int32(10)), (Int32(20)) + SubqueryAlias: t + Projection: column1 AS #2, column2 AS #3 + Values: (Int32(1), Int32(1)), (Int32(1), Int32(2)), (Int32(2), Int32(3)) + + initial_physical_plan: + ScalarSubqueryExec: subqueries=1 + ProjectionExec: expr=[#2@0 as #7, sum(t.#3 + #6)@1 as #8] + AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[sum(t.#3 + #6)] + AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3 + #6)] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#5)@0 as #6] + AggregateExec: mode=Final, gby=[], aggr=[min(s.#5)] + AggregateExec: mode=Partial, gby=[], aggr=[min(s.#5)] + ProjectionExec: expr=[column1@0 as #5] + DataSourceExec: partitions=1, partition_sizes=[1] + + + initial_physical_plan_with_stats: + ScalarSubqueryExec: subqueries=1, statistics=[Rows=Inexact(3), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]] + ProjectionExec: expr=[#2@0 as #7, sum(t.#3 + #6)@1 as #8], statistics=[Rows=Inexact(3), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]] + AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[sum(t.#3 + #6)], statistics=[Rows=Inexact(3), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]] + AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3 + #6)], statistics=[Rows=Inexact(3), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3], statistics=[Rows=Exact(3), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]] + DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(3), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]] + ProjectionExec: expr=[min(s.#5)@0 as #6], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]:)]] + AggregateExec: mode=Final, gby=[], aggr=[min(s.#5)], statistics=[Rows=Exact(1), Bytes=Inexact(), [(Col[0]:)]] + AggregateExec: mode=Partial, gby=[], aggr=[min(s.#5)], statistics=[Rows=Inexact(2), Bytes=Inexact(), [(Col[0]:)]] + ProjectionExec: expr=[column1@0 as #5], statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0))]] + DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0))]] + + + initial_physical_plan_with_schema: + ScalarSubqueryExec: subqueries=1, schema=[#7:Int32;N, #8:Int64;N] + ProjectionExec: expr=[#2@0 as #7, sum(t.#3 + #6)@1 as #8], schema=[#7:Int32;N, #8:Int64;N] + AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[sum(t.#3 + #6)], schema=[#2:Int32;N, sum(t.#3 + #6):Int64;N] + AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3 + #6)], schema=[#2:Int32;N, sum(t.#3 + #6)[sum]:Int64;N] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3], schema=[#2:Int32;N, #3:Int32;N] + DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N, column2:Int32;N] + ProjectionExec: expr=[min(s.#5)@0 as #6], schema=[#6:Int64;N] + AggregateExec: mode=Final, gby=[], aggr=[min(s.#5)], schema=[min(s.#5):Int64;N] + AggregateExec: mode=Partial, gby=[], aggr=[min(s.#5)], schema=[min(s.#5)[value]:Int64;N] + ProjectionExec: expr=[column1@0 as #5], schema=[#5:Int32;N] + DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N] + + + physical_plan after OutputRequirements: + OutputRequirementExec: order_by=[], dist_by=Unspecified + ScalarSubqueryExec: subqueries=1 + ProjectionExec: expr=[#2@0 as #7, sum(t.#3 + #6)@1 as #8] + AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[sum(t.#3 + #6)] + AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3 + #6)] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#5)@0 as #6] + AggregateExec: mode=Final, gby=[], aggr=[min(s.#5)] + AggregateExec: mode=Partial, gby=[], aggr=[min(s.#5)] + ProjectionExec: expr=[column1@0 as #5] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after aggregate_statistics: + SAME TEXT AS ABOVE + + physical_plan after join_selection: + SAME TEXT AS ABOVE + + physical_plan after LimitedDistinctAggregation: + SAME TEXT AS ABOVE + + physical_plan after FilterPushdown: + SAME TEXT AS ABOVE + + physical_plan after EnforceDistribution: + OutputRequirementExec: order_by=[], dist_by=Unspecified + ScalarSubqueryExec: subqueries=1 + ProjectionExec: expr=[#2@0 as #7, sum(t.#3 + #6)@1 as #8] + AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[sum(t.#3 + #6)] + RepartitionExec: partitioning=Hash([#2@0], 4), input_partitions=1 + AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3 + #6)] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#5)@0 as #6] + AggregateExec: mode=Final, gby=[], aggr=[min(s.#5)] + AggregateExec: mode=Partial, gby=[], aggr=[min(s.#5)] + ProjectionExec: expr=[column1@0 as #5] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after CombinePartialFinalAggregate: + OutputRequirementExec: order_by=[], dist_by=Unspecified + ScalarSubqueryExec: subqueries=1 + ProjectionExec: expr=[#2@0 as #7, sum(t.#3 + #6)@1 as #8] + AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[sum(t.#3 + #6)] + RepartitionExec: partitioning=Hash([#2@0], 4), input_partitions=1 + AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3 + #6)] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#5)@0 as #6] + AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)] + ProjectionExec: expr=[column1@0 as #5] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after EnforceSorting: + SAME TEXT AS ABOVE + + physical_plan after OptimizeAggregateOrder: + SAME TEXT AS ABOVE + + physical_plan after WindowTopN: + SAME TEXT AS ABOVE + + physical_plan after ProjectionPushdown: + SAME TEXT AS ABOVE + + physical_plan after OutputRequirements: + ScalarSubqueryExec: subqueries=1 + ProjectionExec: expr=[#2@0 as #7, sum(t.#3 + #6)@1 as #8] + AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[sum(t.#3 + #6)] + RepartitionExec: partitioning=Hash([#2@0], 4), input_partitions=1 + AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3 + #6)] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#5)@0 as #6] + AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)] + ProjectionExec: expr=[column1@0 as #5] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after LimitAggregation: + SAME TEXT AS ABOVE + + physical_plan after LimitPushPastWindows: + SAME TEXT AS ABOVE + + physical_plan after HashJoinBuffering: + SAME TEXT AS ABOVE + + physical_plan after LimitPushdown: + SAME TEXT AS ABOVE + + physical_plan after TopKRepartition: + SAME TEXT AS ABOVE + + physical_plan after ProjectionPushdown: + SAME TEXT AS ABOVE + + physical_plan after PushdownSort: + SAME TEXT AS ABOVE + + physical_plan after EnsureCooperative: + SAME TEXT AS ABOVE + + physical_plan after FilterPushdown(Post): + SAME TEXT AS ABOVE + + physical_plan after RewriteExplicitRepartition: + SAME TEXT AS ABOVE + + physical_plan after RewriteCollectLeftHashJoin: + SAME TEXT AS ABOVE + + physical_plan after EnforceBarrierPartitioning: + SAME TEXT AS ABOVE + + physical_plan after SanityCheckPlan: + SAME TEXT AS ABOVE + + physical_plan: + ProjectionExec: expr=[#7@0 as k, #8@1 as total] + ScalarSubqueryExec: subqueries=1 + ProjectionExec: expr=[#2@0 as #7, sum(t.#3 + #6)@1 as #8] + AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[sum(t.#3 + #6)] + RepartitionExec: partitioning=Hash([#2@0], 4), input_partitions=1 + AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3 + #6)] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#5)@0 as #6] + AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)] + ProjectionExec: expr=[column1@0 as #5] + DataSourceExec: partitions=1, partition_sizes=[1] + + + == Physical Plan == + ProjectionExec: expr=[#7@0 as k, #8@1 as total] + ScalarSubqueryExec: subqueries=1 + ProjectionExec: expr=[#2@0 as #7, sum(t.#3 + #6)@1 as #8] + AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[sum(t.#3 + #6)] + RepartitionExec: partitioning=Hash([#2@0], 4), input_partitions=1 + AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3 + #6)] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#5)@0 as #6] + AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)] + ProjectionExec: expr=[column1@0 as #5] + DataSourceExec: partitions=1, partition_sizes=[1] + + + == Distributed Plan == + === stage 0 === + inputs=[] + mode=Pipelined + partitions=1 + distribution=RoundRobin(channels=1) + placement=Worker + ProjectionExec: expr=[min(s.#5)@0 as #6] + AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)] + ProjectionExec: expr=[column1@0 as #5] + DataSourceExec: partitions=1, partition_sizes=[1] + + === stage 1 === + inputs=[StageInput(stage=0, mode=Broadcast)] + mode=Pipelined + partitions=1 + distribution=Hash(keys=[#2@0], channels=4) + placement=Worker + ScalarSubqueryExec: subqueries=1 + AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3 + #6)] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + StageInputExec: input=0, partitioning=UnknownPartitioning(1) + + === stage 2 === + inputs=[StageInput(stage=1, mode=Shuffle), StageInput(stage=0, mode=Broadcast)] + mode=Pipelined + partitions=4 + distribution=RoundRobin(channels=1) + placement=Worker + ProjectionExec: expr=[#7@0 as k, #8@1 as total] + ScalarSubqueryExec: subqueries=1 + ProjectionExec: expr=[#2@0 as #7, sum(t.#3 + #6)@1 as #8] + AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[sum(t.#3 + #6)] + StageInputExec: input=0, partitioning=Hash([#2@0], 4) + StageInputExec: input=1, partitioning=UnknownPartitioning(1) +--- +name: "test_scalar_subquery_in_having_filter" +data: + |- + == Codegen == + Whole-stage codegen is not supported; showing physical plan instead. + + == Plan Steps == + initial_logical_plan: + Projection: t.#2 AS #7, sum(t.#3) AS total AS #8 + Filter: sum(t.#3) > () + Subquery: + Projection: min(s.#5) AS #6 + Aggregate: groupBy=[[]], aggr=[[min(s.#5)]] + SubqueryAlias: s + Projection: #4 AS #5 + Projection: column1 AS #4 + Values: (Int32(3)) + Aggregate: groupBy=[[t.#2]], aggr=[[sum(t.#3)]] + SubqueryAlias: t + Projection: #0 AS #2, #1 AS #3 + Projection: column1 AS #0, column2 AS #1 + Values: (Int32(1), Int32(1)), (Int32(1), Int32(2)), (Int32(2), Int32(3)), (Int32(2), Int32(4)) + + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE + + logical_plan after resolve_grouping_function: + SAME TEXT AS ABOVE + + logical_plan after type_coercion: + Projection: t.#2 AS #7, sum(t.#3) AS total AS #8 + Filter: sum(t.#3) > () + Subquery: + Projection: CAST(min(s.#5) AS #6 AS Int64) + Aggregate: groupBy=[[]], aggr=[[min(s.#5)]] + SubqueryAlias: s + Projection: #4 AS #5 + Projection: column1 AS #4 + Values: (Int32(3)) + Aggregate: groupBy=[[t.#2]], aggr=[[sum(CAST(t.#3 AS Int64))]] + SubqueryAlias: t + Projection: #0 AS #2, #1 AS #3 + Projection: column1 AS #0, column2 AS #1 + Values: (Int32(1), Int32(1)), (Int32(1), Int32(2)), (Int32(2), Int32(3)), (Int32(2), Int32(4)) + + analyzed_logical_plan: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_projection: + SAME TEXT AS ABOVE + + logical_plan after rewrite_set_comparison: + SAME TEXT AS ABOVE + + logical_plan after optimize_unions: + SAME TEXT AS ABOVE + + logical_plan after unions_to_filter: + SAME TEXT AS ABOVE + + logical_plan after simplify_expressions: + SAME TEXT AS ABOVE + + logical_plan after replace_distinct_aggregate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_predicate_subquery: + SAME TEXT AS ABOVE + + logical_plan after scalar_subquery_to_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_join: + SAME TEXT AS ABOVE + + logical_plan after extract_equijoin_predicate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_duplicated_expr: + SAME TEXT AS ABOVE + + logical_plan after eliminate_filter: + SAME TEXT AS ABOVE + + logical_plan after eliminate_cross_join: + SAME TEXT AS ABOVE + + logical_plan after eliminate_limit: + SAME TEXT AS ABOVE + + logical_plan after propagate_empty_relation: + SAME TEXT AS ABOVE + + logical_plan after filter_null_join_keys: + SAME TEXT AS ABOVE + + logical_plan after eliminate_outer_join: + SAME TEXT AS ABOVE + + logical_plan after push_down_limit: + SAME TEXT AS ABOVE + + logical_plan after push_down_filter: + SAME TEXT AS ABOVE + + logical_plan after single_distinct_aggregation_to_group_by: + SAME TEXT AS ABOVE + + logical_plan after eliminate_group_by_constant: + SAME TEXT AS ABOVE + + logical_plan after common_sub_expression_eliminate: + SAME TEXT AS ABOVE + + logical_plan after extract_leaf_expressions: + SAME TEXT AS ABOVE + + logical_plan after optimize_projections: + Projection: t.#2 AS #7, sum(t.#3) AS total AS #8 + Filter: sum(t.#3) > () + Subquery: + Projection: CAST(min(s.#5) AS #6 AS Int64) + Aggregate: groupBy=[[]], aggr=[[min(s.#5)]] + SubqueryAlias: s + Projection: column1 AS #5 + Values: (Int32(3)) + Aggregate: groupBy=[[t.#2]], aggr=[[sum(CAST(t.#3 AS Int64))]] + SubqueryAlias: t + Projection: column1 AS #2, column2 AS #3 + Values: (Int32(1), Int32(1)), (Int32(1), Int32(2)), (Int32(2), Int32(3)), (Int32(2), Int32(4)) + + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_projection: + SAME TEXT AS ABOVE + + logical_plan after rewrite_set_comparison: + SAME TEXT AS ABOVE + + logical_plan after optimize_unions: + SAME TEXT AS ABOVE + + logical_plan after unions_to_filter: + SAME TEXT AS ABOVE + + logical_plan after simplify_expressions: + SAME TEXT AS ABOVE + + logical_plan after replace_distinct_aggregate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_predicate_subquery: + SAME TEXT AS ABOVE + + logical_plan after scalar_subquery_to_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_join: + SAME TEXT AS ABOVE + + logical_plan after extract_equijoin_predicate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_duplicated_expr: + SAME TEXT AS ABOVE + + logical_plan after eliminate_filter: + SAME TEXT AS ABOVE + + logical_plan after eliminate_cross_join: + SAME TEXT AS ABOVE + + logical_plan after eliminate_limit: + SAME TEXT AS ABOVE + + logical_plan after propagate_empty_relation: + SAME TEXT AS ABOVE + + logical_plan after filter_null_join_keys: + SAME TEXT AS ABOVE + + logical_plan after eliminate_outer_join: + SAME TEXT AS ABOVE + + logical_plan after push_down_limit: + SAME TEXT AS ABOVE + + logical_plan after push_down_filter: + SAME TEXT AS ABOVE + + logical_plan after single_distinct_aggregation_to_group_by: + SAME TEXT AS ABOVE + + logical_plan after eliminate_group_by_constant: + SAME TEXT AS ABOVE + + logical_plan after common_sub_expression_eliminate: + SAME TEXT AS ABOVE + + logical_plan after extract_leaf_expressions: + SAME TEXT AS ABOVE + + logical_plan after optimize_projections: + SAME TEXT AS ABOVE + + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE + + logical_plan: + Projection: t.#2 AS #7, sum(t.#3) AS total AS #8 + Filter: sum(t.#3) > () + Subquery: + Projection: CAST(min(s.#5) AS #6 AS Int64) + Aggregate: groupBy=[[]], aggr=[[min(s.#5)]] + SubqueryAlias: s + Projection: column1 AS #5 + Values: (Int32(3)) + Aggregate: groupBy=[[t.#2]], aggr=[[sum(CAST(t.#3 AS Int64))]] + SubqueryAlias: t + Projection: column1 AS #2, column2 AS #3 + Values: (Int32(1), Int32(1)), (Int32(1), Int32(2)), (Int32(2), Int32(3)), (Int32(2), Int32(4)) + + initial_physical_plan: + ScalarSubqueryExec: subqueries=1 + ProjectionExec: expr=[#2@0 as #7, sum(t.#3)@1 as #8] + FilterExec: sum(t.#3)@1 > scalar_subquery() + AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[sum(t.#3)] + AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3)] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[CAST(min(s.#5)@0 AS Int64) as #6] + AggregateExec: mode=Final, gby=[], aggr=[min(s.#5)] + AggregateExec: mode=Partial, gby=[], aggr=[min(s.#5)] + ProjectionExec: expr=[column1@0 as #5] + DataSourceExec: partitions=1, partition_sizes=[1] + + + initial_physical_plan_with_stats: + ScalarSubqueryExec: subqueries=1, statistics=[Rows=Inexact(1), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]] + ProjectionExec: expr=[#2@0 as #7, sum(t.#3)@1 as #8], statistics=[Rows=Inexact(1), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]] + FilterExec: sum(t.#3)@1 > scalar_subquery(), statistics=[Rows=Inexact(1), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]] + AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[sum(t.#3)], statistics=[Rows=Inexact(4), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]] + AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3)], statistics=[Rows=Inexact(4), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3], statistics=[Rows=Exact(4), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]] + DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(4), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]] + ProjectionExec: expr=[CAST(min(s.#5)@0 AS Int64) as #6], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]:)]] + AggregateExec: mode=Final, gby=[], aggr=[min(s.#5)], statistics=[Rows=Exact(1), Bytes=Absent, [(Col[0]:)]] + AggregateExec: mode=Partial, gby=[], aggr=[min(s.#5)], statistics=[Rows=Exact(0), Bytes=Inexact(), [(Col[0]:)]] + ProjectionExec: expr=[column1@0 as #5], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]: Null=Exact(0))]] + DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]: Null=Exact(0))]] + + + initial_physical_plan_with_schema: + ScalarSubqueryExec: subqueries=1, schema=[#7:Int32;N, #8:Int64;N] + ProjectionExec: expr=[#2@0 as #7, sum(t.#3)@1 as #8], schema=[#7:Int32;N, #8:Int64;N] + FilterExec: sum(t.#3)@1 > scalar_subquery(), schema=[#2:Int32;N, sum(t.#3):Int64;N] + AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[sum(t.#3)], schema=[#2:Int32;N, sum(t.#3):Int64;N] + AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3)], schema=[#2:Int32;N, sum(t.#3)[sum]:Int64;N] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3], schema=[#2:Int32;N, #3:Int32;N] + DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N, column2:Int32;N] + ProjectionExec: expr=[CAST(min(s.#5)@0 AS Int64) as #6], schema=[#6:Int64;N] + AggregateExec: mode=Final, gby=[], aggr=[min(s.#5)], schema=[min(s.#5):Int32;N] + AggregateExec: mode=Partial, gby=[], aggr=[min(s.#5)], schema=[min(s.#5)[value]:Int32;N] + ProjectionExec: expr=[column1@0 as #5], schema=[#5:Int32;N] + DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N] + + + physical_plan after OutputRequirements: + OutputRequirementExec: order_by=[], dist_by=Unspecified + ScalarSubqueryExec: subqueries=1 + ProjectionExec: expr=[#2@0 as #7, sum(t.#3)@1 as #8] + FilterExec: sum(t.#3)@1 > scalar_subquery() + AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[sum(t.#3)] + AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3)] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[CAST(min(s.#5)@0 AS Int64) as #6] + AggregateExec: mode=Final, gby=[], aggr=[min(s.#5)] + AggregateExec: mode=Partial, gby=[], aggr=[min(s.#5)] + ProjectionExec: expr=[column1@0 as #5] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after aggregate_statistics: + SAME TEXT AS ABOVE + + physical_plan after join_selection: + SAME TEXT AS ABOVE + + physical_plan after LimitedDistinctAggregation: + SAME TEXT AS ABOVE + + physical_plan after FilterPushdown: + SAME TEXT AS ABOVE + + physical_plan after EnforceDistribution: + OutputRequirementExec: order_by=[], dist_by=Unspecified + ScalarSubqueryExec: subqueries=1 + ProjectionExec: expr=[#2@0 as #7, sum(t.#3)@1 as #8] + FilterExec: sum(t.#3)@1 > scalar_subquery() + AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[sum(t.#3)] + RepartitionExec: partitioning=Hash([#2@0], 4), input_partitions=1 + AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3)] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[CAST(min(s.#5)@0 AS Int64) as #6] + AggregateExec: mode=Final, gby=[], aggr=[min(s.#5)] + AggregateExec: mode=Partial, gby=[], aggr=[min(s.#5)] + ProjectionExec: expr=[column1@0 as #5] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after CombinePartialFinalAggregate: + OutputRequirementExec: order_by=[], dist_by=Unspecified + ScalarSubqueryExec: subqueries=1 + ProjectionExec: expr=[#2@0 as #7, sum(t.#3)@1 as #8] + FilterExec: sum(t.#3)@1 > scalar_subquery() + AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[sum(t.#3)] + RepartitionExec: partitioning=Hash([#2@0], 4), input_partitions=1 + AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3)] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[CAST(min(s.#5)@0 AS Int64) as #6] + AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)] + ProjectionExec: expr=[column1@0 as #5] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after EnforceSorting: + SAME TEXT AS ABOVE + + physical_plan after OptimizeAggregateOrder: + SAME TEXT AS ABOVE + + physical_plan after WindowTopN: + SAME TEXT AS ABOVE + + physical_plan after ProjectionPushdown: + SAME TEXT AS ABOVE + + physical_plan after OutputRequirements: + ScalarSubqueryExec: subqueries=1 + ProjectionExec: expr=[#2@0 as #7, sum(t.#3)@1 as #8] + FilterExec: sum(t.#3)@1 > scalar_subquery() + AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[sum(t.#3)] + RepartitionExec: partitioning=Hash([#2@0], 4), input_partitions=1 + AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3)] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[CAST(min(s.#5)@0 AS Int64) as #6] + AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)] + ProjectionExec: expr=[column1@0 as #5] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after LimitAggregation: + SAME TEXT AS ABOVE + + physical_plan after LimitPushPastWindows: + SAME TEXT AS ABOVE + + physical_plan after HashJoinBuffering: + SAME TEXT AS ABOVE + + physical_plan after LimitPushdown: + SAME TEXT AS ABOVE + + physical_plan after TopKRepartition: + SAME TEXT AS ABOVE + + physical_plan after ProjectionPushdown: + SAME TEXT AS ABOVE + + physical_plan after PushdownSort: + SAME TEXT AS ABOVE + + physical_plan after EnsureCooperative: + SAME TEXT AS ABOVE + + physical_plan after FilterPushdown(Post): + SAME TEXT AS ABOVE + + physical_plan after RewriteExplicitRepartition: + SAME TEXT AS ABOVE + + physical_plan after RewriteCollectLeftHashJoin: + SAME TEXT AS ABOVE + + physical_plan after EnforceBarrierPartitioning: + SAME TEXT AS ABOVE + + physical_plan after SanityCheckPlan: + SAME TEXT AS ABOVE + + physical_plan: + ProjectionExec: expr=[#7@0 as k, #8@1 as total] + ScalarSubqueryExec: subqueries=1 + ProjectionExec: expr=[#2@0 as #7, sum(t.#3)@1 as #8] + FilterExec: sum(t.#3)@1 > scalar_subquery() + AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[sum(t.#3)] + RepartitionExec: partitioning=Hash([#2@0], 4), input_partitions=1 + AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3)] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[CAST(min(s.#5)@0 AS Int64) as #6] + AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)] + ProjectionExec: expr=[column1@0 as #5] + DataSourceExec: partitions=1, partition_sizes=[1] + + + == Physical Plan == + ProjectionExec: expr=[#7@0 as k, #8@1 as total] + ScalarSubqueryExec: subqueries=1 + ProjectionExec: expr=[#2@0 as #7, sum(t.#3)@1 as #8] + FilterExec: sum(t.#3)@1 > scalar_subquery() + AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[sum(t.#3)] + RepartitionExec: partitioning=Hash([#2@0], 4), input_partitions=1 + AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3)] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[CAST(min(s.#5)@0 AS Int64) as #6] + AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)] + ProjectionExec: expr=[column1@0 as #5] + DataSourceExec: partitions=1, partition_sizes=[1] + + + == Distributed Plan == + === stage 0 === + inputs=[] + mode=Pipelined + partitions=1 + distribution=RoundRobin(channels=1) + placement=Worker + ProjectionExec: expr=[CAST(min(s.#5)@0 AS Int64) as #6] + AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)] + ProjectionExec: expr=[column1@0 as #5] + DataSourceExec: partitions=1, partition_sizes=[1] + + === stage 1 === + inputs=[] + mode=Pipelined + partitions=1 + distribution=Hash(keys=[#2@0], channels=4) + placement=Worker + AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3)] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + + === stage 2 === + inputs=[StageInput(stage=1, mode=Shuffle), StageInput(stage=0, mode=Broadcast)] + mode=Pipelined + partitions=4 + distribution=RoundRobin(channels=1) + placement=Worker + ProjectionExec: expr=[#7@0 as k, #8@1 as total] + ScalarSubqueryExec: subqueries=1 + ProjectionExec: expr=[#2@0 as #7, sum(t.#3)@1 as #8] + FilterExec: sum(t.#3)@1 > scalar_subquery() + AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[sum(t.#3)] + StageInputExec: input=0, partitioning=Hash([#2@0], 4) + StageInputExec: input=1, partitioning=UnknownPartitioning(1) +--- +name: "test_scalar_subquery_in_join_condition" +data: + |- + == Codegen == + Whole-stage codegen is not supported; showing physical plan instead. + + == Plan Steps == + initial_logical_plan: + Sort: #7 ASC NULLS FIRST + Projection: l.#1 AS #7, r.#3 AS #8 + Inner Join: Filter: CAST(l.#1 AS Int64) + () = CAST(r.#3 AS Int64) + Subquery: + Projection: min(s.#5) AS #6 + Aggregate: groupBy=[[]], aggr=[[min(CAST(s.#5 AS Int64))]] + SubqueryAlias: s + Projection: #4 AS #5 + Projection: column1 AS #4 + Values: (Int32(10)), (Int32(20)) + SubqueryAlias: l + Projection: #0 AS #1 + Projection: column1 AS #0 + Values: (Int32(1)), (Int32(2)), (Int32(3)) + SubqueryAlias: r + Projection: #2 AS #3 + Projection: column1 AS #2 + Values: (Int32(11)), (Int32(12)), (Int32(13)) + + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE + + logical_plan after resolve_grouping_function: + SAME TEXT AS ABOVE + + logical_plan after type_coercion: + SAME TEXT AS ABOVE + + analyzed_logical_plan: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_projection: + SAME TEXT AS ABOVE + + logical_plan after rewrite_set_comparison: + SAME TEXT AS ABOVE + + logical_plan after optimize_unions: + SAME TEXT AS ABOVE + + logical_plan after unions_to_filter: + SAME TEXT AS ABOVE + + logical_plan after simplify_expressions: + SAME TEXT AS ABOVE + + logical_plan after replace_distinct_aggregate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_predicate_subquery: + SAME TEXT AS ABOVE + + logical_plan after scalar_subquery_to_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_join: + SAME TEXT AS ABOVE + + logical_plan after extract_equijoin_predicate: + Sort: #7 ASC NULLS FIRST + Projection: l.#1 AS #7, r.#3 AS #8 + Inner Join: CAST(l.#1 AS Int64) + () = CAST(r.#3 AS Int64) + Subquery: + Projection: min(s.#5) AS #6 + Aggregate: groupBy=[[]], aggr=[[min(CAST(s.#5 AS Int64))]] + SubqueryAlias: s + Projection: #4 AS #5 + Projection: column1 AS #4 + Values: (Int32(10)), (Int32(20)) + SubqueryAlias: l + Projection: #0 AS #1 + Projection: column1 AS #0 + Values: (Int32(1)), (Int32(2)), (Int32(3)) + SubqueryAlias: r + Projection: #2 AS #3 + Projection: column1 AS #2 + Values: (Int32(11)), (Int32(12)), (Int32(13)) + + logical_plan after eliminate_duplicated_expr: + SAME TEXT AS ABOVE + + logical_plan after eliminate_filter: + SAME TEXT AS ABOVE + + logical_plan after eliminate_cross_join: + SAME TEXT AS ABOVE + + logical_plan after eliminate_limit: + SAME TEXT AS ABOVE + + logical_plan after propagate_empty_relation: + SAME TEXT AS ABOVE + + logical_plan after filter_null_join_keys: + SAME TEXT AS ABOVE + + logical_plan after eliminate_outer_join: + SAME TEXT AS ABOVE + + logical_plan after push_down_limit: + SAME TEXT AS ABOVE + + logical_plan after push_down_filter: + SAME TEXT AS ABOVE + + logical_plan after single_distinct_aggregation_to_group_by: + SAME TEXT AS ABOVE + + logical_plan after eliminate_group_by_constant: + SAME TEXT AS ABOVE + + logical_plan after common_sub_expression_eliminate: + SAME TEXT AS ABOVE + + logical_plan after extract_leaf_expressions: + SAME TEXT AS ABOVE + + logical_plan after optimize_projections: + Sort: #7 ASC NULLS FIRST + Projection: l.#1 AS #7, r.#3 AS #8 + Inner Join: CAST(l.#1 AS Int64) + () = CAST(r.#3 AS Int64) + Subquery: + Projection: min(s.#5) AS #6 + Aggregate: groupBy=[[]], aggr=[[min(CAST(s.#5 AS Int64))]] + SubqueryAlias: s + Projection: column1 AS #5 + Values: (Int32(10)), (Int32(20)) + SubqueryAlias: l + Projection: column1 AS #1 + Values: (Int32(1)), (Int32(2)), (Int32(3)) + SubqueryAlias: r + Projection: column1 AS #3 + Values: (Int32(11)), (Int32(12)), (Int32(13)) + + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_projection: + SAME TEXT AS ABOVE + + logical_plan after rewrite_set_comparison: + SAME TEXT AS ABOVE + + logical_plan after optimize_unions: + SAME TEXT AS ABOVE + + logical_plan after unions_to_filter: + SAME TEXT AS ABOVE + + logical_plan after simplify_expressions: + SAME TEXT AS ABOVE + + logical_plan after replace_distinct_aggregate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_predicate_subquery: + SAME TEXT AS ABOVE + + logical_plan after scalar_subquery_to_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_join: + SAME TEXT AS ABOVE + + logical_plan after extract_equijoin_predicate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_duplicated_expr: + SAME TEXT AS ABOVE + + logical_plan after eliminate_filter: + SAME TEXT AS ABOVE + + logical_plan after eliminate_cross_join: + SAME TEXT AS ABOVE + + logical_plan after eliminate_limit: + SAME TEXT AS ABOVE + + logical_plan after propagate_empty_relation: + SAME TEXT AS ABOVE + + logical_plan after filter_null_join_keys: + SAME TEXT AS ABOVE + + logical_plan after eliminate_outer_join: + SAME TEXT AS ABOVE + + logical_plan after push_down_limit: + SAME TEXT AS ABOVE + + logical_plan after push_down_filter: + SAME TEXT AS ABOVE + + logical_plan after single_distinct_aggregation_to_group_by: + SAME TEXT AS ABOVE + + logical_plan after eliminate_group_by_constant: + SAME TEXT AS ABOVE + + logical_plan after common_sub_expression_eliminate: + SAME TEXT AS ABOVE + + logical_plan after extract_leaf_expressions: + SAME TEXT AS ABOVE + + logical_plan after optimize_projections: + SAME TEXT AS ABOVE + + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE + + logical_plan: + Sort: #7 ASC NULLS FIRST + Projection: l.#1 AS #7, r.#3 AS #8 + Inner Join: CAST(l.#1 AS Int64) + () = CAST(r.#3 AS Int64) + Subquery: + Projection: min(s.#5) AS #6 + Aggregate: groupBy=[[]], aggr=[[min(CAST(s.#5 AS Int64))]] + SubqueryAlias: s + Projection: column1 AS #5 + Values: (Int32(10)), (Int32(20)) + SubqueryAlias: l + Projection: column1 AS #1 + Values: (Int32(1)), (Int32(2)), (Int32(3)) + SubqueryAlias: r + Projection: column1 AS #3 + Values: (Int32(11)), (Int32(12)), (Int32(13)) + + initial_physical_plan: + ScalarSubqueryExec: subqueries=1 + SortExec: expr=[#7@0 ASC], preserve_partitioning=[false] + ProjectionExec: expr=[#1@0 as #7, #3@1 as #8] + ProjectionExec: expr=[#1@0 as #1, #3@2 as #3] + HashJoinExec: mode=Auto, join_type=Inner, on=[(l.#1 + #6@1, CAST(r.#3 AS Int64)@1)] + ProjectionExec: expr=[#1@0 as #1, CAST(#1@0 AS Int64) + scalar_subquery() as l.#1 + #6] + ProjectionExec: expr=[column1@0 as #1] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[#3@0 as #3, CAST(#3@0 AS Int64) as CAST(r.#3 AS Int64)] + ProjectionExec: expr=[column1@0 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#5)@0 as #6] + AggregateExec: mode=Final, gby=[], aggr=[min(s.#5)] + AggregateExec: mode=Partial, gby=[], aggr=[min(s.#5)] + ProjectionExec: expr=[column1@0 as #5] + DataSourceExec: partitions=1, partition_sizes=[1] + + + initial_physical_plan_with_stats: + ScalarSubqueryExec: subqueries=1, statistics=[Rows=Inexact(3), Bytes=Inexact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]] + SortExec: expr=[#7@0 ASC], preserve_partitioning=[false], statistics=[Rows=Inexact(3), Bytes=Inexact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]] + ProjectionExec: expr=[#1@0 as #7, #3@1 as #8], statistics=[Rows=Inexact(3), Bytes=Inexact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]] + ProjectionExec: expr=[#1@0 as #1, #3@2 as #3], statistics=[Rows=Inexact(3), Bytes=Inexact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]] + HashJoinExec: mode=Auto, join_type=Inner, on=[(l.#1 + #6@1, CAST(r.#3 AS Int64)@1)], statistics=[Rows=Inexact(3), Bytes=Absent, [(Col[0]: Null=Exact(0)),(Col[1]:),(Col[2]: Null=Exact(0)),(Col[3]:)]] + ProjectionExec: expr=[#1@0 as #1, CAST(#1@0 AS Int64) + scalar_subquery() as l.#1 + #6], statistics=[Rows=Exact(3), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]:)]] + ProjectionExec: expr=[column1@0 as #1], statistics=[Rows=Exact(3), Bytes=Exact(), [(Col[0]: Null=Exact(0))]] + DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(3), Bytes=Exact(), [(Col[0]: Null=Exact(0))]] + ProjectionExec: expr=[#3@0 as #3, CAST(#3@0 AS Int64) as CAST(r.#3 AS Int64)], statistics=[Rows=Exact(3), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]:)]] + ProjectionExec: expr=[column1@0 as #3], statistics=[Rows=Exact(3), Bytes=Exact(), [(Col[0]: Null=Exact(0))]] + DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(3), Bytes=Exact(), [(Col[0]: Null=Exact(0))]] + ProjectionExec: expr=[min(s.#5)@0 as #6], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]:)]] + AggregateExec: mode=Final, gby=[], aggr=[min(s.#5)], statistics=[Rows=Exact(1), Bytes=Inexact(), [(Col[0]:)]] + AggregateExec: mode=Partial, gby=[], aggr=[min(s.#5)], statistics=[Rows=Inexact(2), Bytes=Inexact(), [(Col[0]:)]] + ProjectionExec: expr=[column1@0 as #5], statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0))]] + DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0))]] + + + initial_physical_plan_with_schema: + ScalarSubqueryExec: subqueries=1, schema=[#7:Int32;N, #8:Int32;N] + SortExec: expr=[#7@0 ASC], preserve_partitioning=[false], schema=[#7:Int32;N, #8:Int32;N] + ProjectionExec: expr=[#1@0 as #7, #3@1 as #8], schema=[#7:Int32;N, #8:Int32;N] + ProjectionExec: expr=[#1@0 as #1, #3@2 as #3], schema=[#1:Int32;N, #3:Int32;N] + HashJoinExec: mode=Auto, join_type=Inner, on=[(l.#1 + #6@1, CAST(r.#3 AS Int64)@1)], schema=[#1:Int32;N, l.#1 + #6:Int64;N, #3:Int32;N, CAST(r.#3 AS Int64):Int64;N] + ProjectionExec: expr=[#1@0 as #1, CAST(#1@0 AS Int64) + scalar_subquery() as l.#1 + #6], schema=[#1:Int32;N, l.#1 + #6:Int64;N] + ProjectionExec: expr=[column1@0 as #1], schema=[#1:Int32;N] + DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N] + ProjectionExec: expr=[#3@0 as #3, CAST(#3@0 AS Int64) as CAST(r.#3 AS Int64)], schema=[#3:Int32;N, CAST(r.#3 AS Int64):Int64;N] + ProjectionExec: expr=[column1@0 as #3], schema=[#3:Int32;N] + DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N] + ProjectionExec: expr=[min(s.#5)@0 as #6], schema=[#6:Int64;N] + AggregateExec: mode=Final, gby=[], aggr=[min(s.#5)], schema=[min(s.#5):Int64;N] + AggregateExec: mode=Partial, gby=[], aggr=[min(s.#5)], schema=[min(s.#5)[value]:Int64;N] + ProjectionExec: expr=[column1@0 as #5], schema=[#5:Int32;N] + DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N] + + + physical_plan after OutputRequirements: + OutputRequirementExec: order_by=[], dist_by=Unspecified + ScalarSubqueryExec: subqueries=1 + SortExec: expr=[#7@0 ASC], preserve_partitioning=[false] + ProjectionExec: expr=[#1@0 as #7, #3@1 as #8] + ProjectionExec: expr=[#1@0 as #1, #3@2 as #3] + HashJoinExec: mode=Auto, join_type=Inner, on=[(l.#1 + #6@1, CAST(r.#3 AS Int64)@1)] + ProjectionExec: expr=[#1@0 as #1, CAST(#1@0 AS Int64) + scalar_subquery() as l.#1 + #6] + ProjectionExec: expr=[column1@0 as #1] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[#3@0 as #3, CAST(#3@0 AS Int64) as CAST(r.#3 AS Int64)] + ProjectionExec: expr=[column1@0 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#5)@0 as #6] + AggregateExec: mode=Final, gby=[], aggr=[min(s.#5)] + AggregateExec: mode=Partial, gby=[], aggr=[min(s.#5)] + ProjectionExec: expr=[column1@0 as #5] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after aggregate_statistics: + SAME TEXT AS ABOVE + + physical_plan after join_selection: + OutputRequirementExec: order_by=[], dist_by=Unspecified + ScalarSubqueryExec: subqueries=1 + SortExec: expr=[#7@0 ASC], preserve_partitioning=[false] + ProjectionExec: expr=[#1@0 as #7, #3@1 as #8] + ProjectionExec: expr=[#1@0 as #1, #3@2 as #3] + HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(l.#1 + #6@1, CAST(r.#3 AS Int64)@1)] + ProjectionExec: expr=[#1@0 as #1, CAST(#1@0 AS Int64) + scalar_subquery() as l.#1 + #6] + ProjectionExec: expr=[column1@0 as #1] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[#3@0 as #3, CAST(#3@0 AS Int64) as CAST(r.#3 AS Int64)] + ProjectionExec: expr=[column1@0 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#5)@0 as #6] + AggregateExec: mode=Final, gby=[], aggr=[min(s.#5)] + AggregateExec: mode=Partial, gby=[], aggr=[min(s.#5)] + ProjectionExec: expr=[column1@0 as #5] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after LimitedDistinctAggregation: + SAME TEXT AS ABOVE + + physical_plan after FilterPushdown: + SAME TEXT AS ABOVE + + physical_plan after EnforceDistribution: + SAME TEXT AS ABOVE + + physical_plan after CombinePartialFinalAggregate: + OutputRequirementExec: order_by=[], dist_by=Unspecified + ScalarSubqueryExec: subqueries=1 + SortExec: expr=[#7@0 ASC], preserve_partitioning=[false] + ProjectionExec: expr=[#1@0 as #7, #3@1 as #8] + ProjectionExec: expr=[#1@0 as #1, #3@2 as #3] + HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(l.#1 + #6@1, CAST(r.#3 AS Int64)@1)] + ProjectionExec: expr=[#1@0 as #1, CAST(#1@0 AS Int64) + scalar_subquery() as l.#1 + #6] + ProjectionExec: expr=[column1@0 as #1] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[#3@0 as #3, CAST(#3@0 AS Int64) as CAST(r.#3 AS Int64)] + ProjectionExec: expr=[column1@0 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#5)@0 as #6] + AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)] + ProjectionExec: expr=[column1@0 as #5] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after EnforceSorting: + SAME TEXT AS ABOVE + + physical_plan after OptimizeAggregateOrder: + SAME TEXT AS ABOVE + + physical_plan after WindowTopN: + SAME TEXT AS ABOVE + + physical_plan after ProjectionPushdown: + OutputRequirementExec: order_by=[], dist_by=Unspecified + ScalarSubqueryExec: subqueries=1 + SortExec: expr=[#7@0 ASC], preserve_partitioning=[false] + ProjectionExec: expr=[#1@0 as #7, #3@1 as #8] + HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(l.#1 + #6@1, CAST(r.#3 AS Int64)@1)], projection=[#1@0, #3@2] + ProjectionExec: expr=[column1@0 as #1, CAST(column1@0 AS Int64) + scalar_subquery() as l.#1 + #6] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[column1@0 as #3, CAST(column1@0 AS Int64) as CAST(r.#3 AS Int64)] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#5)@0 as #6] + AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)] + ProjectionExec: expr=[column1@0 as #5] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after OutputRequirements: + ScalarSubqueryExec: subqueries=1 + SortExec: expr=[#7@0 ASC], preserve_partitioning=[false] + ProjectionExec: expr=[#1@0 as #7, #3@1 as #8] + HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(l.#1 + #6@1, CAST(r.#3 AS Int64)@1)], projection=[#1@0, #3@2] + ProjectionExec: expr=[column1@0 as #1, CAST(column1@0 AS Int64) + scalar_subquery() as l.#1 + #6] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[column1@0 as #3, CAST(column1@0 AS Int64) as CAST(r.#3 AS Int64)] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#5)@0 as #6] + AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)] + ProjectionExec: expr=[column1@0 as #5] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after LimitAggregation: + SAME TEXT AS ABOVE + + physical_plan after LimitPushPastWindows: + SAME TEXT AS ABOVE + + physical_plan after HashJoinBuffering: + SAME TEXT AS ABOVE + + physical_plan after LimitPushdown: + SAME TEXT AS ABOVE + + physical_plan after TopKRepartition: + SAME TEXT AS ABOVE + + physical_plan after ProjectionPushdown: + SAME TEXT AS ABOVE + + physical_plan after PushdownSort: + SAME TEXT AS ABOVE + + physical_plan after EnsureCooperative: + SAME TEXT AS ABOVE + + physical_plan after FilterPushdown(Post): + SAME TEXT AS ABOVE + + physical_plan after RewriteExplicitRepartition: + SAME TEXT AS ABOVE + + physical_plan after RewriteCollectLeftHashJoin: + SAME TEXT AS ABOVE + + physical_plan after EnforceBarrierPartitioning: + SAME TEXT AS ABOVE + + physical_plan after SanityCheckPlan: + SAME TEXT AS ABOVE + + physical_plan: + ProjectionExec: expr=[#7@0 as v, #8@1 as w] + ScalarSubqueryExec: subqueries=1 + SortExec: expr=[#7@0 ASC], preserve_partitioning=[false] + ProjectionExec: expr=[#1@0 as #7, #3@1 as #8] + HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(l.#1 + #6@1, CAST(r.#3 AS Int64)@1)], projection=[#1@0, #3@2] + ProjectionExec: expr=[column1@0 as #1, CAST(column1@0 AS Int64) + scalar_subquery() as l.#1 + #6] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[column1@0 as #3, CAST(column1@0 AS Int64) as CAST(r.#3 AS Int64)] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#5)@0 as #6] + AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)] + ProjectionExec: expr=[column1@0 as #5] + DataSourceExec: partitions=1, partition_sizes=[1] + + + == Physical Plan == + ProjectionExec: expr=[#7@0 as v, #8@1 as w] + ScalarSubqueryExec: subqueries=1 + SortExec: expr=[#7@0 ASC], preserve_partitioning=[false] + ProjectionExec: expr=[#1@0 as #7, #3@1 as #8] + HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(l.#1 + #6@1, CAST(r.#3 AS Int64)@1)], projection=[#1@0, #3@2] + ProjectionExec: expr=[column1@0 as #1, CAST(column1@0 AS Int64) + scalar_subquery() as l.#1 + #6] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[column1@0 as #3, CAST(column1@0 AS Int64) as CAST(r.#3 AS Int64)] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#5)@0 as #6] + AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)] + ProjectionExec: expr=[column1@0 as #5] + DataSourceExec: partitions=1, partition_sizes=[1] + + + == Distributed Plan == + === stage 0 === + inputs=[] + mode=Pipelined + partitions=1 + distribution=RoundRobin(channels=1) + placement=Worker + ProjectionExec: expr=[min(s.#5)@0 as #6] + AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)] + ProjectionExec: expr=[column1@0 as #5] + DataSourceExec: partitions=1, partition_sizes=[1] + + === stage 1 === + inputs=[StageInput(stage=0, mode=Broadcast)] + mode=Pipelined + partitions=1 + distribution=RoundRobin(channels=1) + placement=Worker + ProjectionExec: expr=[#7@0 as v, #8@1 as w] + ScalarSubqueryExec: subqueries=1 + SortExec: expr=[#7@0 ASC], preserve_partitioning=[false] + ProjectionExec: expr=[#1@0 as #7, #3@1 as #8] + HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(l.#1 + #6@1, CAST(r.#3 AS Int64)@1)], projection=[#1@0, #3@2] + ProjectionExec: expr=[column1@0 as #1, CAST(column1@0 AS Int64) + scalar_subquery() as l.#1 + #6] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[column1@0 as #3, CAST(column1@0 AS Int64) as CAST(r.#3 AS Int64)] + DataSourceExec: partitions=1, partition_sizes=[1] + StageInputExec: input=0, partitioning=UnknownPartitioning(1) +--- +name: "test_scalar_subquery_in_nonequi_join_condition" +data: + |- + == Codegen == + Whole-stage codegen is not supported; showing physical plan instead. + + == Plan Steps == + initial_logical_plan: + Sort: #7 ASC NULLS FIRST, #8 ASC NULLS FIRST + Projection: l.#1 AS #7, r.#3 AS #8 + Inner Join: Filter: CAST(l.#1 AS Int64) + () < CAST(r.#3 AS Int64) + Subquery: + Projection: min(s.#5) AS #6 + Aggregate: groupBy=[[]], aggr=[[min(CAST(s.#5 AS Int64))]] + SubqueryAlias: s + Projection: #4 AS #5 + Projection: column1 AS #4 + Values: (Int32(10)), (Int32(20)) + SubqueryAlias: l + Projection: #0 AS #1 + Projection: column1 AS #0 + Values: (Int32(1)), (Int32(2)), (Int32(3)) + SubqueryAlias: r + Projection: #2 AS #3 + Projection: column1 AS #2 + Values: (Int32(11)), (Int32(12)), (Int32(13)) + + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE + + logical_plan after resolve_grouping_function: + SAME TEXT AS ABOVE + + logical_plan after type_coercion: + SAME TEXT AS ABOVE + + analyzed_logical_plan: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_projection: + SAME TEXT AS ABOVE + + logical_plan after rewrite_set_comparison: + SAME TEXT AS ABOVE + + logical_plan after optimize_unions: + SAME TEXT AS ABOVE + + logical_plan after unions_to_filter: + SAME TEXT AS ABOVE + + logical_plan after simplify_expressions: + SAME TEXT AS ABOVE + + logical_plan after replace_distinct_aggregate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_predicate_subquery: + SAME TEXT AS ABOVE + + logical_plan after scalar_subquery_to_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_join: + SAME TEXT AS ABOVE + + logical_plan after extract_equijoin_predicate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_duplicated_expr: + SAME TEXT AS ABOVE + + logical_plan after eliminate_filter: + SAME TEXT AS ABOVE + + logical_plan after eliminate_cross_join: + Sort: #7 ASC NULLS FIRST, #8 ASC NULLS FIRST + Projection: l.#1 AS #7, r.#3 AS #8 + Filter: CAST(l.#1 AS Int64) + () < CAST(r.#3 AS Int64) + Subquery: + Projection: min(s.#5) AS #6 + Aggregate: groupBy=[[]], aggr=[[min(CAST(s.#5 AS Int64))]] + SubqueryAlias: s + Projection: #4 AS #5 + Projection: column1 AS #4 + Values: (Int32(10)), (Int32(20)) + Cross Join: + SubqueryAlias: l + Projection: #0 AS #1 + Projection: column1 AS #0 + Values: (Int32(1)), (Int32(2)), (Int32(3)) + SubqueryAlias: r + Projection: #2 AS #3 + Projection: column1 AS #2 + Values: (Int32(11)), (Int32(12)), (Int32(13)) + + logical_plan after eliminate_limit: + SAME TEXT AS ABOVE + + logical_plan after propagate_empty_relation: + SAME TEXT AS ABOVE + + logical_plan after filter_null_join_keys: + SAME TEXT AS ABOVE + + logical_plan after eliminate_outer_join: + SAME TEXT AS ABOVE + + logical_plan after push_down_limit: + SAME TEXT AS ABOVE + + logical_plan after push_down_filter: + SAME TEXT AS ABOVE + + logical_plan after single_distinct_aggregation_to_group_by: + SAME TEXT AS ABOVE + + logical_plan after eliminate_group_by_constant: + SAME TEXT AS ABOVE + + logical_plan after common_sub_expression_eliminate: + SAME TEXT AS ABOVE + + logical_plan after extract_leaf_expressions: + SAME TEXT AS ABOVE + + logical_plan after optimize_projections: + Sort: #7 ASC NULLS FIRST, #8 ASC NULLS FIRST + Projection: l.#1 AS #7, r.#3 AS #8 + Filter: CAST(l.#1 AS Int64) + () < CAST(r.#3 AS Int64) + Subquery: + Projection: min(s.#5) AS #6 + Aggregate: groupBy=[[]], aggr=[[min(CAST(s.#5 AS Int64))]] + SubqueryAlias: s + Projection: column1 AS #5 + Values: (Int32(10)), (Int32(20)) + Cross Join: + SubqueryAlias: l + Projection: column1 AS #1 + Values: (Int32(1)), (Int32(2)), (Int32(3)) + SubqueryAlias: r + Projection: column1 AS #3 + Values: (Int32(11)), (Int32(12)), (Int32(13)) + + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_projection: + SAME TEXT AS ABOVE + + logical_plan after rewrite_set_comparison: + SAME TEXT AS ABOVE + + logical_plan after optimize_unions: + SAME TEXT AS ABOVE + + logical_plan after unions_to_filter: + SAME TEXT AS ABOVE + + logical_plan after simplify_expressions: + SAME TEXT AS ABOVE + + logical_plan after replace_distinct_aggregate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_predicate_subquery: + SAME TEXT AS ABOVE + + logical_plan after scalar_subquery_to_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_join: + SAME TEXT AS ABOVE + + logical_plan after extract_equijoin_predicate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_duplicated_expr: + SAME TEXT AS ABOVE + + logical_plan after eliminate_filter: + SAME TEXT AS ABOVE + + logical_plan after eliminate_cross_join: + SAME TEXT AS ABOVE + + logical_plan after eliminate_limit: + SAME TEXT AS ABOVE + + logical_plan after propagate_empty_relation: + SAME TEXT AS ABOVE + + logical_plan after filter_null_join_keys: + SAME TEXT AS ABOVE + + logical_plan after eliminate_outer_join: + SAME TEXT AS ABOVE + + logical_plan after push_down_limit: + SAME TEXT AS ABOVE + + logical_plan after push_down_filter: + SAME TEXT AS ABOVE + + logical_plan after single_distinct_aggregation_to_group_by: + SAME TEXT AS ABOVE + + logical_plan after eliminate_group_by_constant: + SAME TEXT AS ABOVE + + logical_plan after common_sub_expression_eliminate: + SAME TEXT AS ABOVE + + logical_plan after extract_leaf_expressions: + SAME TEXT AS ABOVE + + logical_plan after optimize_projections: + SAME TEXT AS ABOVE + + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE + + logical_plan: + Sort: #7 ASC NULLS FIRST, #8 ASC NULLS FIRST + Projection: l.#1 AS #7, r.#3 AS #8 + Filter: CAST(l.#1 AS Int64) + () < CAST(r.#3 AS Int64) + Subquery: + Projection: min(s.#5) AS #6 + Aggregate: groupBy=[[]], aggr=[[min(CAST(s.#5 AS Int64))]] + SubqueryAlias: s + Projection: column1 AS #5 + Values: (Int32(10)), (Int32(20)) + Cross Join: + SubqueryAlias: l + Projection: column1 AS #1 + Values: (Int32(1)), (Int32(2)), (Int32(3)) + SubqueryAlias: r + Projection: column1 AS #3 + Values: (Int32(11)), (Int32(12)), (Int32(13)) + + initial_physical_plan: + ScalarSubqueryExec: subqueries=1 + SortExec: expr=[#7@0 ASC, #8@1 ASC], preserve_partitioning=[false] + ProjectionExec: expr=[#1@0 as #7, #3@1 as #8] + FilterExec: CAST(#1@0 AS Int64) + scalar_subquery() < CAST(#3@1 AS Int64) + CrossJoinExec + ProjectionExec: expr=[column1@0 as #1] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[column1@0 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#5)@0 as #6] + AggregateExec: mode=Final, gby=[], aggr=[min(s.#5)] + AggregateExec: mode=Partial, gby=[], aggr=[min(s.#5)] + ProjectionExec: expr=[column1@0 as #5] + DataSourceExec: partitions=1, partition_sizes=[1] + + + initial_physical_plan_with_stats: + ScalarSubqueryExec: subqueries=1, statistics=[Rows=Inexact(2), Bytes=Inexact(), [(Col[0]: Null=Inexact(0)),(Col[1]: Null=Inexact(0))]] + SortExec: expr=[#7@0 ASC, #8@1 ASC], preserve_partitioning=[false], statistics=[Rows=Inexact(2), Bytes=Inexact(), [(Col[0]: Null=Inexact(0)),(Col[1]: Null=Inexact(0))]] + ProjectionExec: expr=[#1@0 as #7, #3@1 as #8], statistics=[Rows=Inexact(2), Bytes=Inexact(), [(Col[0]: Null=Inexact(0)),(Col[1]: Null=Inexact(0))]] + FilterExec: CAST(#1@0 AS Int64) + scalar_subquery() < CAST(#3@1 AS Int64), statistics=[Rows=Inexact(2), Bytes=Inexact(), [(Col[0]: Null=Inexact(0)),(Col[1]: Null=Inexact(0))]] + CrossJoinExec, statistics=[Rows=Exact(9), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]] + ProjectionExec: expr=[column1@0 as #1], statistics=[Rows=Exact(3), Bytes=Exact(), [(Col[0]: Null=Exact(0))]] + DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(3), Bytes=Exact(), [(Col[0]: Null=Exact(0))]] + ProjectionExec: expr=[column1@0 as #3], statistics=[Rows=Exact(3), Bytes=Exact(), [(Col[0]: Null=Exact(0))]] + DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(3), Bytes=Exact(), [(Col[0]: Null=Exact(0))]] + ProjectionExec: expr=[min(s.#5)@0 as #6], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]:)]] + AggregateExec: mode=Final, gby=[], aggr=[min(s.#5)], statistics=[Rows=Exact(1), Bytes=Inexact(), [(Col[0]:)]] + AggregateExec: mode=Partial, gby=[], aggr=[min(s.#5)], statistics=[Rows=Inexact(2), Bytes=Inexact(), [(Col[0]:)]] + ProjectionExec: expr=[column1@0 as #5], statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0))]] + DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0))]] + + + initial_physical_plan_with_schema: + ScalarSubqueryExec: subqueries=1, schema=[#7:Int32;N, #8:Int32;N] + SortExec: expr=[#7@0 ASC, #8@1 ASC], preserve_partitioning=[false], schema=[#7:Int32;N, #8:Int32;N] + ProjectionExec: expr=[#1@0 as #7, #3@1 as #8], schema=[#7:Int32;N, #8:Int32;N] + FilterExec: CAST(#1@0 AS Int64) + scalar_subquery() < CAST(#3@1 AS Int64), schema=[#1:Int32;N, #3:Int32;N] + CrossJoinExec, schema=[#1:Int32;N, #3:Int32;N] + ProjectionExec: expr=[column1@0 as #1], schema=[#1:Int32;N] + DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N] + ProjectionExec: expr=[column1@0 as #3], schema=[#3:Int32;N] + DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N] + ProjectionExec: expr=[min(s.#5)@0 as #6], schema=[#6:Int64;N] + AggregateExec: mode=Final, gby=[], aggr=[min(s.#5)], schema=[min(s.#5):Int64;N] + AggregateExec: mode=Partial, gby=[], aggr=[min(s.#5)], schema=[min(s.#5)[value]:Int64;N] + ProjectionExec: expr=[column1@0 as #5], schema=[#5:Int32;N] + DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N] + + + physical_plan after OutputRequirements: + OutputRequirementExec: order_by=[], dist_by=Unspecified + ScalarSubqueryExec: subqueries=1 + SortExec: expr=[#7@0 ASC, #8@1 ASC], preserve_partitioning=[false] + ProjectionExec: expr=[#1@0 as #7, #3@1 as #8] + FilterExec: CAST(#1@0 AS Int64) + scalar_subquery() < CAST(#3@1 AS Int64) + CrossJoinExec + ProjectionExec: expr=[column1@0 as #1] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[column1@0 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#5)@0 as #6] + AggregateExec: mode=Final, gby=[], aggr=[min(s.#5)] + AggregateExec: mode=Partial, gby=[], aggr=[min(s.#5)] + ProjectionExec: expr=[column1@0 as #5] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after aggregate_statistics: + SAME TEXT AS ABOVE + + physical_plan after join_selection: + SAME TEXT AS ABOVE + + physical_plan after LimitedDistinctAggregation: + SAME TEXT AS ABOVE + + physical_plan after FilterPushdown: + SAME TEXT AS ABOVE + + physical_plan after EnforceDistribution: + SAME TEXT AS ABOVE + + physical_plan after CombinePartialFinalAggregate: + OutputRequirementExec: order_by=[], dist_by=Unspecified + ScalarSubqueryExec: subqueries=1 + SortExec: expr=[#7@0 ASC, #8@1 ASC], preserve_partitioning=[false] + ProjectionExec: expr=[#1@0 as #7, #3@1 as #8] + FilterExec: CAST(#1@0 AS Int64) + scalar_subquery() < CAST(#3@1 AS Int64) + CrossJoinExec + ProjectionExec: expr=[column1@0 as #1] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[column1@0 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#5)@0 as #6] + AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)] + ProjectionExec: expr=[column1@0 as #5] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after EnforceSorting: + SAME TEXT AS ABOVE + + physical_plan after OptimizeAggregateOrder: + SAME TEXT AS ABOVE + + physical_plan after WindowTopN: + SAME TEXT AS ABOVE + + physical_plan after ProjectionPushdown: + SAME TEXT AS ABOVE + + physical_plan after OutputRequirements: + ScalarSubqueryExec: subqueries=1 + SortExec: expr=[#7@0 ASC, #8@1 ASC], preserve_partitioning=[false] + ProjectionExec: expr=[#1@0 as #7, #3@1 as #8] + FilterExec: CAST(#1@0 AS Int64) + scalar_subquery() < CAST(#3@1 AS Int64) + CrossJoinExec + ProjectionExec: expr=[column1@0 as #1] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[column1@0 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#5)@0 as #6] + AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)] + ProjectionExec: expr=[column1@0 as #5] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after LimitAggregation: + SAME TEXT AS ABOVE + + physical_plan after LimitPushPastWindows: + SAME TEXT AS ABOVE + + physical_plan after HashJoinBuffering: + SAME TEXT AS ABOVE + + physical_plan after LimitPushdown: + SAME TEXT AS ABOVE + + physical_plan after TopKRepartition: + SAME TEXT AS ABOVE + + physical_plan after ProjectionPushdown: + SAME TEXT AS ABOVE + + physical_plan after PushdownSort: + SAME TEXT AS ABOVE + + physical_plan after EnsureCooperative: + SAME TEXT AS ABOVE + + physical_plan after FilterPushdown(Post): + SAME TEXT AS ABOVE + + physical_plan after RewriteExplicitRepartition: + SAME TEXT AS ABOVE + + physical_plan after RewriteCollectLeftHashJoin: + SAME TEXT AS ABOVE + + physical_plan after EnforceBarrierPartitioning: + SAME TEXT AS ABOVE + + physical_plan after SanityCheckPlan: + SAME TEXT AS ABOVE + + physical_plan: + ProjectionExec: expr=[#7@0 as v, #8@1 as w] + ScalarSubqueryExec: subqueries=1 + SortExec: expr=[#7@0 ASC, #8@1 ASC], preserve_partitioning=[false] + ProjectionExec: expr=[#1@0 as #7, #3@1 as #8] + FilterExec: CAST(#1@0 AS Int64) + scalar_subquery() < CAST(#3@1 AS Int64) + CrossJoinExec + ProjectionExec: expr=[column1@0 as #1] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[column1@0 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#5)@0 as #6] + AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)] + ProjectionExec: expr=[column1@0 as #5] + DataSourceExec: partitions=1, partition_sizes=[1] + + + == Physical Plan == + ProjectionExec: expr=[#7@0 as v, #8@1 as w] + ScalarSubqueryExec: subqueries=1 + SortExec: expr=[#7@0 ASC, #8@1 ASC], preserve_partitioning=[false] + ProjectionExec: expr=[#1@0 as #7, #3@1 as #8] + FilterExec: CAST(#1@0 AS Int64) + scalar_subquery() < CAST(#3@1 AS Int64) + CrossJoinExec + ProjectionExec: expr=[column1@0 as #1] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[column1@0 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#5)@0 as #6] + AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)] + ProjectionExec: expr=[column1@0 as #5] + DataSourceExec: partitions=1, partition_sizes=[1] + + + == Distributed Plan == + === stage 0 === + inputs=[] + mode=Pipelined + partitions=1 + distribution=RoundRobin(channels=1) + placement=Worker + ProjectionExec: expr=[min(s.#5)@0 as #6] + AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)] + ProjectionExec: expr=[column1@0 as #5] + DataSourceExec: partitions=1, partition_sizes=[1] + + === stage 1 === + inputs=[StageInput(stage=0, mode=Broadcast)] + mode=Pipelined + partitions=1 + distribution=RoundRobin(channels=1) + placement=Worker + ProjectionExec: expr=[#7@0 as v, #8@1 as w] + ScalarSubqueryExec: subqueries=1 + SortExec: expr=[#7@0 ASC, #8@1 ASC], preserve_partitioning=[false] + ProjectionExec: expr=[#1@0 as #7, #3@1 as #8] + FilterExec: CAST(#1@0 AS Int64) + scalar_subquery() < CAST(#3@1 AS Int64) + CrossJoinExec + ProjectionExec: expr=[column1@0 as #1] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[column1@0 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + StageInputExec: input=0, partitioning=UnknownPartitioning(1) --- -name: "test_multiple_scalar_subqueries_in_one_expression" +name: "test_scalar_subquery_in_projection_before_aggregate" data: |- - == Physical Plan == - ProjectionExec: expr=[#8@0 as total] - ScalarSubqueryExec: subqueries=2 - ProjectionExec: expr=[sum(t.#1 + #4 + #7)@0 as #8] - AggregateExec: mode=Single, gby=[], aggr=[sum(t.#1 + #4 + #7)] - ProjectionExec: expr=[column1@0 as #1] + == Codegen == + Whole-stage codegen is not supported; showing physical plan instead. + + == Plan Steps == + initial_logical_plan: + Projection: p.#7 AS #9, sum(p.#8) AS total AS #10 + Aggregate: groupBy=[[p.#7]], aggr=[[sum(p.#8)]] + SubqueryAlias: p + Projection: t.#2 AS #7, CAST(t.#3 AS Int64) + () AS shifted AS #8 + Subquery: + Projection: min(s.#5) AS #6 + Aggregate: groupBy=[[]], aggr=[[min(CAST(s.#5 AS Int64))]] + SubqueryAlias: s + Projection: #4 AS #5 + Projection: column1 AS #4 + Values: (Int32(10)), (Int32(20)) + SubqueryAlias: t + Projection: #0 AS #2, #1 AS #3 + Projection: column1 AS #0, column2 AS #1 + Values: (Int32(1), Int32(1)), (Int32(1), Int32(2)), (Int32(2), Int32(3)) + + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE + + logical_plan after resolve_grouping_function: + SAME TEXT AS ABOVE + + logical_plan after type_coercion: + SAME TEXT AS ABOVE + + analyzed_logical_plan: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_projection: + SAME TEXT AS ABOVE + + logical_plan after rewrite_set_comparison: + SAME TEXT AS ABOVE + + logical_plan after optimize_unions: + SAME TEXT AS ABOVE + + logical_plan after unions_to_filter: + SAME TEXT AS ABOVE + + logical_plan after simplify_expressions: + SAME TEXT AS ABOVE + + logical_plan after replace_distinct_aggregate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_predicate_subquery: + SAME TEXT AS ABOVE + + logical_plan after scalar_subquery_to_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_join: + SAME TEXT AS ABOVE + + logical_plan after extract_equijoin_predicate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_duplicated_expr: + SAME TEXT AS ABOVE + + logical_plan after eliminate_filter: + SAME TEXT AS ABOVE + + logical_plan after eliminate_cross_join: + SAME TEXT AS ABOVE + + logical_plan after eliminate_limit: + SAME TEXT AS ABOVE + + logical_plan after propagate_empty_relation: + SAME TEXT AS ABOVE + + logical_plan after filter_null_join_keys: + SAME TEXT AS ABOVE + + logical_plan after eliminate_outer_join: + SAME TEXT AS ABOVE + + logical_plan after push_down_limit: + SAME TEXT AS ABOVE + + logical_plan after push_down_filter: + SAME TEXT AS ABOVE + + logical_plan after single_distinct_aggregation_to_group_by: + SAME TEXT AS ABOVE + + logical_plan after eliminate_group_by_constant: + SAME TEXT AS ABOVE + + logical_plan after common_sub_expression_eliminate: + SAME TEXT AS ABOVE + + logical_plan after extract_leaf_expressions: + SAME TEXT AS ABOVE + + logical_plan after optimize_projections: + Projection: p.#7 AS #9, sum(p.#8) AS total AS #10 + Aggregate: groupBy=[[p.#7]], aggr=[[sum(p.#8)]] + SubqueryAlias: p + Projection: t.#2 AS #7, CAST(t.#3 AS Int64) + () AS shifted AS #8 + Subquery: + Projection: min(s.#5) AS #6 + Aggregate: groupBy=[[]], aggr=[[min(CAST(s.#5 AS Int64))]] + SubqueryAlias: s + Projection: column1 AS #5 + Values: (Int32(10)), (Int32(20)) + SubqueryAlias: t + Projection: column1 AS #2, column2 AS #3 + Values: (Int32(1), Int32(1)), (Int32(1), Int32(2)), (Int32(2), Int32(3)) + + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_projection: + SAME TEXT AS ABOVE + + logical_plan after rewrite_set_comparison: + SAME TEXT AS ABOVE + + logical_plan after optimize_unions: + SAME TEXT AS ABOVE + + logical_plan after unions_to_filter: + SAME TEXT AS ABOVE + + logical_plan after simplify_expressions: + SAME TEXT AS ABOVE + + logical_plan after replace_distinct_aggregate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_predicate_subquery: + SAME TEXT AS ABOVE + + logical_plan after scalar_subquery_to_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_join: + SAME TEXT AS ABOVE + + logical_plan after extract_equijoin_predicate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_duplicated_expr: + SAME TEXT AS ABOVE + + logical_plan after eliminate_filter: + SAME TEXT AS ABOVE + + logical_plan after eliminate_cross_join: + SAME TEXT AS ABOVE + + logical_plan after eliminate_limit: + SAME TEXT AS ABOVE + + logical_plan after propagate_empty_relation: + SAME TEXT AS ABOVE + + logical_plan after filter_null_join_keys: + SAME TEXT AS ABOVE + + logical_plan after eliminate_outer_join: + SAME TEXT AS ABOVE + + logical_plan after push_down_limit: + SAME TEXT AS ABOVE + + logical_plan after push_down_filter: + SAME TEXT AS ABOVE + + logical_plan after single_distinct_aggregation_to_group_by: + SAME TEXT AS ABOVE + + logical_plan after eliminate_group_by_constant: + SAME TEXT AS ABOVE + + logical_plan after common_sub_expression_eliminate: + SAME TEXT AS ABOVE + + logical_plan after extract_leaf_expressions: + SAME TEXT AS ABOVE + + logical_plan after optimize_projections: + SAME TEXT AS ABOVE + + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE + + logical_plan: + Projection: p.#7 AS #9, sum(p.#8) AS total AS #10 + Aggregate: groupBy=[[p.#7]], aggr=[[sum(p.#8)]] + SubqueryAlias: p + Projection: t.#2 AS #7, CAST(t.#3 AS Int64) + () AS shifted AS #8 + Subquery: + Projection: min(s.#5) AS #6 + Aggregate: groupBy=[[]], aggr=[[min(CAST(s.#5 AS Int64))]] + SubqueryAlias: s + Projection: column1 AS #5 + Values: (Int32(10)), (Int32(20)) + SubqueryAlias: t + Projection: column1 AS #2, column2 AS #3 + Values: (Int32(1), Int32(1)), (Int32(1), Int32(2)), (Int32(2), Int32(3)) + + initial_physical_plan: + ScalarSubqueryExec: subqueries=1 + ProjectionExec: expr=[#7@0 as #9, sum(p.#8)@1 as #10] + AggregateExec: mode=FinalPartitioned, gby=[#7@0 as #7], aggr=[sum(p.#8)] + AggregateExec: mode=Partial, gby=[#7@0 as #7], aggr=[sum(p.#8)] + ProjectionExec: expr=[#2@0 as #7, CAST(#3@1 AS Int64) + scalar_subquery() as #8] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#5)@0 as #6] + AggregateExec: mode=Final, gby=[], aggr=[min(s.#5)] + AggregateExec: mode=Partial, gby=[], aggr=[min(s.#5)] + ProjectionExec: expr=[column1@0 as #5] DataSourceExec: partitions=1, partition_sizes=[1] - ProjectionExec: expr=[min(s.#3)@0 as #4] - AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)] - ProjectionExec: expr=[column1@0 as #3] + + + initial_physical_plan_with_stats: + ScalarSubqueryExec: subqueries=1, statistics=[Rows=Inexact(3), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]] + ProjectionExec: expr=[#7@0 as #9, sum(p.#8)@1 as #10], statistics=[Rows=Inexact(3), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]] + AggregateExec: mode=FinalPartitioned, gby=[#7@0 as #7], aggr=[sum(p.#8)], statistics=[Rows=Inexact(3), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]] + AggregateExec: mode=Partial, gby=[#7@0 as #7], aggr=[sum(p.#8)], statistics=[Rows=Inexact(3), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]] + ProjectionExec: expr=[#2@0 as #7, CAST(#3@1 AS Int64) + scalar_subquery() as #8], statistics=[Rows=Exact(3), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]:)]] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3], statistics=[Rows=Exact(3), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]] + DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(3), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]] + ProjectionExec: expr=[min(s.#5)@0 as #6], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]:)]] + AggregateExec: mode=Final, gby=[], aggr=[min(s.#5)], statistics=[Rows=Exact(1), Bytes=Inexact(), [(Col[0]:)]] + AggregateExec: mode=Partial, gby=[], aggr=[min(s.#5)], statistics=[Rows=Inexact(2), Bytes=Inexact(), [(Col[0]:)]] + ProjectionExec: expr=[column1@0 as #5], statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0))]] + DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0))]] + + + initial_physical_plan_with_schema: + ScalarSubqueryExec: subqueries=1, schema=[#9:Int32;N, #10:Int64;N] + ProjectionExec: expr=[#7@0 as #9, sum(p.#8)@1 as #10], schema=[#9:Int32;N, #10:Int64;N] + AggregateExec: mode=FinalPartitioned, gby=[#7@0 as #7], aggr=[sum(p.#8)], schema=[#7:Int32;N, sum(p.#8):Int64;N] + AggregateExec: mode=Partial, gby=[#7@0 as #7], aggr=[sum(p.#8)], schema=[#7:Int32;N, sum(p.#8)[sum]:Int64;N] + ProjectionExec: expr=[#2@0 as #7, CAST(#3@1 AS Int64) + scalar_subquery() as #8], schema=[#7:Int32;N, #8:Int64;N] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3], schema=[#2:Int32;N, #3:Int32;N] + DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N, column2:Int32;N] + ProjectionExec: expr=[min(s.#5)@0 as #6], schema=[#6:Int64;N] + AggregateExec: mode=Final, gby=[], aggr=[min(s.#5)], schema=[min(s.#5):Int64;N] + AggregateExec: mode=Partial, gby=[], aggr=[min(s.#5)], schema=[min(s.#5)[value]:Int64;N] + ProjectionExec: expr=[column1@0 as #5], schema=[#5:Int32;N] + DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N] + + + physical_plan after OutputRequirements: + OutputRequirementExec: order_by=[], dist_by=Unspecified + ScalarSubqueryExec: subqueries=1 + ProjectionExec: expr=[#7@0 as #9, sum(p.#8)@1 as #10] + AggregateExec: mode=FinalPartitioned, gby=[#7@0 as #7], aggr=[sum(p.#8)] + AggregateExec: mode=Partial, gby=[#7@0 as #7], aggr=[sum(p.#8)] + ProjectionExec: expr=[#2@0 as #7, CAST(#3@1 AS Int64) + scalar_subquery() as #8] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#5)@0 as #6] + AggregateExec: mode=Final, gby=[], aggr=[min(s.#5)] + AggregateExec: mode=Partial, gby=[], aggr=[min(s.#5)] + ProjectionExec: expr=[column1@0 as #5] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after aggregate_statistics: + SAME TEXT AS ABOVE + + physical_plan after join_selection: + SAME TEXT AS ABOVE + + physical_plan after LimitedDistinctAggregation: + SAME TEXT AS ABOVE + + physical_plan after FilterPushdown: + SAME TEXT AS ABOVE + + physical_plan after EnforceDistribution: + OutputRequirementExec: order_by=[], dist_by=Unspecified + ScalarSubqueryExec: subqueries=1 + ProjectionExec: expr=[#7@0 as #9, sum(p.#8)@1 as #10] + AggregateExec: mode=FinalPartitioned, gby=[#7@0 as #7], aggr=[sum(p.#8)] + RepartitionExec: partitioning=Hash([#7@0], 4), input_partitions=1 + AggregateExec: mode=Partial, gby=[#7@0 as #7], aggr=[sum(p.#8)] + ProjectionExec: expr=[#2@0 as #7, CAST(#3@1 AS Int64) + scalar_subquery() as #8] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#5)@0 as #6] + AggregateExec: mode=Final, gby=[], aggr=[min(s.#5)] + AggregateExec: mode=Partial, gby=[], aggr=[min(s.#5)] + ProjectionExec: expr=[column1@0 as #5] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after CombinePartialFinalAggregate: + OutputRequirementExec: order_by=[], dist_by=Unspecified + ScalarSubqueryExec: subqueries=1 + ProjectionExec: expr=[#7@0 as #9, sum(p.#8)@1 as #10] + AggregateExec: mode=FinalPartitioned, gby=[#7@0 as #7], aggr=[sum(p.#8)] + RepartitionExec: partitioning=Hash([#7@0], 4), input_partitions=1 + AggregateExec: mode=Partial, gby=[#7@0 as #7], aggr=[sum(p.#8)] + ProjectionExec: expr=[#2@0 as #7, CAST(#3@1 AS Int64) + scalar_subquery() as #8] + ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#5)@0 as #6] + AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)] + ProjectionExec: expr=[column1@0 as #5] DataSourceExec: partitions=1, partition_sizes=[1] - ProjectionExec: expr=[max(u.#6)@0 as #7] - AggregateExec: mode=Single, gby=[], aggr=[max(u.#6)] - ProjectionExec: expr=[column1@0 as #6] + + + physical_plan after EnforceSorting: + SAME TEXT AS ABOVE + + physical_plan after OptimizeAggregateOrder: + SAME TEXT AS ABOVE + + physical_plan after WindowTopN: + SAME TEXT AS ABOVE + + physical_plan after ProjectionPushdown: + OutputRequirementExec: order_by=[], dist_by=Unspecified + ScalarSubqueryExec: subqueries=1 + ProjectionExec: expr=[#7@0 as #9, sum(p.#8)@1 as #10] + AggregateExec: mode=FinalPartitioned, gby=[#7@0 as #7], aggr=[sum(p.#8)] + RepartitionExec: partitioning=Hash([#7@0], 4), input_partitions=1 + AggregateExec: mode=Partial, gby=[#7@0 as #7], aggr=[sum(p.#8)] + ProjectionExec: expr=[column1@0 as #7, CAST(column2@1 AS Int64) + scalar_subquery() as #8] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#5)@0 as #6] + AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)] + ProjectionExec: expr=[column1@0 as #5] DataSourceExec: partitions=1, partition_sizes=[1] - == Distributed Plan == - === stage 0 === - inputs=[] - mode=Pipelined - partitions=1 - distribution=RoundRobin(channels=1) - placement=Worker - ProjectionExec: expr=[min(s.#3)@0 as #4] - AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)] - ProjectionExec: expr=[column1@0 as #3] - DataSourceExec: partitions=1, partition_sizes=[1] + physical_plan after OutputRequirements: + ScalarSubqueryExec: subqueries=1 + ProjectionExec: expr=[#7@0 as #9, sum(p.#8)@1 as #10] + AggregateExec: mode=FinalPartitioned, gby=[#7@0 as #7], aggr=[sum(p.#8)] + RepartitionExec: partitioning=Hash([#7@0], 4), input_partitions=1 + AggregateExec: mode=Partial, gby=[#7@0 as #7], aggr=[sum(p.#8)] + ProjectionExec: expr=[column1@0 as #7, CAST(column2@1 AS Int64) + scalar_subquery() as #8] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#5)@0 as #6] + AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)] + ProjectionExec: expr=[column1@0 as #5] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after LimitAggregation: + SAME TEXT AS ABOVE + + physical_plan after LimitPushPastWindows: + SAME TEXT AS ABOVE - === stage 1 === - inputs=[] - mode=Pipelined - partitions=1 - distribution=RoundRobin(channels=1) - placement=Worker - ProjectionExec: expr=[max(u.#6)@0 as #7] - AggregateExec: mode=Single, gby=[], aggr=[max(u.#6)] - ProjectionExec: expr=[column1@0 as #6] - DataSourceExec: partitions=1, partition_sizes=[1] + physical_plan after HashJoinBuffering: + SAME TEXT AS ABOVE - === stage 2 === - inputs=[StageInput(stage=0, mode=Broadcast), StageInput(stage=1, mode=Broadcast)] - mode=Pipelined - partitions=1 - distribution=RoundRobin(channels=1) - placement=Worker - ProjectionExec: expr=[#8@0 as total] - ScalarSubqueryExec: subqueries=2 - ProjectionExec: expr=[sum(t.#1 + #4 + #7)@0 as #8] - AggregateExec: mode=Single, gby=[], aggr=[sum(t.#1 + #4 + #7)] - ProjectionExec: expr=[column1@0 as #1] + physical_plan after LimitPushdown: + SAME TEXT AS ABOVE + + physical_plan after TopKRepartition: + SAME TEXT AS ABOVE + + physical_plan after ProjectionPushdown: + SAME TEXT AS ABOVE + + physical_plan after PushdownSort: + SAME TEXT AS ABOVE + + physical_plan after EnsureCooperative: + SAME TEXT AS ABOVE + + physical_plan after FilterPushdown(Post): + SAME TEXT AS ABOVE + + physical_plan after RewriteExplicitRepartition: + SAME TEXT AS ABOVE + + physical_plan after RewriteCollectLeftHashJoin: + SAME TEXT AS ABOVE + + physical_plan after EnforceBarrierPartitioning: + SAME TEXT AS ABOVE + + physical_plan after SanityCheckPlan: + SAME TEXT AS ABOVE + + physical_plan: + ProjectionExec: expr=[#9@0 as k, #10@1 as total] + ScalarSubqueryExec: subqueries=1 + ProjectionExec: expr=[#7@0 as #9, sum(p.#8)@1 as #10] + AggregateExec: mode=FinalPartitioned, gby=[#7@0 as #7], aggr=[sum(p.#8)] + RepartitionExec: partitioning=Hash([#7@0], 4), input_partitions=1 + AggregateExec: mode=Partial, gby=[#7@0 as #7], aggr=[sum(p.#8)] + ProjectionExec: expr=[column1@0 as #7, CAST(column2@1 AS Int64) + scalar_subquery() as #8] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#5)@0 as #6] + AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)] + ProjectionExec: expr=[column1@0 as #5] DataSourceExec: partitions=1, partition_sizes=[1] - StageInputExec: input=0, partitioning=UnknownPartitioning(1) - StageInputExec: input=1, partitioning=UnknownPartitioning(1) ---- -name: "test_nested_scalar_subquery" -data: - |- + + == Physical Plan == - ProjectionExec: expr=[#8@0 as v, #9@1 as shifted] + ProjectionExec: expr=[#9@0 as k, #10@1 as total] ScalarSubqueryExec: subqueries=1 - SortExec: expr=[#8@0 ASC], preserve_partitioning=[false] - ProjectionExec: expr=[column1@0 as #8, CAST(column1@0 AS Int64) + scalar_subquery() as #9] - DataSourceExec: partitions=1, partition_sizes=[1] - ScalarSubqueryExec: subqueries=1 - ProjectionExec: expr=[CAST(min(s.#3)@0 + scalar_subquery() AS Int64) as #7] - AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)] - ProjectionExec: expr=[column1@0 as #3] - DataSourceExec: partitions=1, partition_sizes=[1] - ProjectionExec: expr=[min(u.#5)@0 as #6] - AggregateExec: mode=Single, gby=[], aggr=[min(u.#5)] - ProjectionExec: expr=[column1@0 as #5] - DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[#7@0 as #9, sum(p.#8)@1 as #10] + AggregateExec: mode=FinalPartitioned, gby=[#7@0 as #7], aggr=[sum(p.#8)] + RepartitionExec: partitioning=Hash([#7@0], 4), input_partitions=1 + AggregateExec: mode=Partial, gby=[#7@0 as #7], aggr=[sum(p.#8)] + ProjectionExec: expr=[column1@0 as #7, CAST(column2@1 AS Int64) + scalar_subquery() as #8] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#5)@0 as #6] + AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)] + ProjectionExec: expr=[column1@0 as #5] + DataSourceExec: partitions=1, partition_sizes=[1] == Distributed Plan == @@ -493,8 +9192,8 @@ data: partitions=1 distribution=RoundRobin(channels=1) placement=Worker - ProjectionExec: expr=[min(u.#5)@0 as #6] - AggregateExec: mode=Single, gby=[], aggr=[min(u.#5)] + ProjectionExec: expr=[min(s.#5)@0 as #6] + AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)] ProjectionExec: expr=[column1@0 as #5] DataSourceExec: partitions=1, partition_sizes=[1] @@ -502,138 +9201,423 @@ data: inputs=[StageInput(stage=0, mode=Broadcast)] mode=Pipelined partitions=1 - distribution=RoundRobin(channels=1) + distribution=Hash(keys=[#7@0], channels=4) placement=Worker ScalarSubqueryExec: subqueries=1 - ProjectionExec: expr=[CAST(min(s.#3)@0 + scalar_subquery() AS Int64) as #7] - AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)] - ProjectionExec: expr=[column1@0 as #3] - DataSourceExec: partitions=1, partition_sizes=[1] + AggregateExec: mode=Partial, gby=[#7@0 as #7], aggr=[sum(p.#8)] + ProjectionExec: expr=[column1@0 as #7, CAST(column2@1 AS Int64) + scalar_subquery() as #8] + DataSourceExec: partitions=1, partition_sizes=[1] StageInputExec: input=0, partitioning=UnknownPartitioning(1) === stage 2 === - inputs=[StageInput(stage=1, mode=Broadcast)] + inputs=[StageInput(stage=1, mode=Shuffle)] mode=Pipelined - partitions=1 + partitions=4 distribution=RoundRobin(channels=1) placement=Worker - ProjectionExec: expr=[#8@0 as v, #9@1 as shifted] - ScalarSubqueryExec: subqueries=1 - SortExec: expr=[#8@0 ASC], preserve_partitioning=[false] - ProjectionExec: expr=[column1@0 as #8, CAST(column1@0 AS Int64) + scalar_subquery() as #9] - DataSourceExec: partitions=1, partition_sizes=[1] - StageInputExec: input=0, partitioning=UnknownPartitioning(1) + ProjectionExec: expr=[#9@0 as k, #10@1 as total] + ProjectionExec: expr=[#7@0 as #9, sum(p.#8)@1 as #10] + AggregateExec: mode=FinalPartitioned, gby=[#7@0 as #7], aggr=[sum(p.#8)] + StageInputExec: input=0, partitioning=Hash([#7@0], 4) --- -name: "test_null_scalar_subquery_result" +name: "test_scalar_subquery_in_sort_expression" data: |- - == Physical Plan == - ProjectionExec: expr=[#5@0 as v, #6@1 as maybe_shifted] - ScalarSubqueryExec: subqueries=1 - SortExec: expr=[#5@0 ASC], preserve_partitioning=[false] - ProjectionExec: expr=[column1@0 as #5, CAST(column1@0 AS Int64) + scalar_subquery() as #6] + == Codegen == + Whole-stage codegen is not supported; showing physical plan instead. + + == Plan Steps == + initial_logical_plan: + Sort: CASE WHEN () = Int32(2) THEN #2 WHEN Boolean(true) THEN negative(#2) END DESC NULLS LAST + Subquery: + Projection: min(s.#4) AS #5 + Aggregate: groupBy=[[]], aggr=[[min(s.#4)]] + SubqueryAlias: s + Projection: #3 AS #4 + Projection: column1 AS #3 + Values: (Int32(2)) + Projection: t.#1 AS #2 + SubqueryAlias: t + Projection: #0 AS #1 + Projection: column1 AS #0 + Values: (Int32(1)), (Int32(2)), (Int32(3)) + + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE + + logical_plan after resolve_grouping_function: + SAME TEXT AS ABOVE + + logical_plan after type_coercion: + SAME TEXT AS ABOVE + + analyzed_logical_plan: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_projection: + SAME TEXT AS ABOVE + + logical_plan after rewrite_set_comparison: + SAME TEXT AS ABOVE + + logical_plan after optimize_unions: + SAME TEXT AS ABOVE + + logical_plan after unions_to_filter: + SAME TEXT AS ABOVE + + logical_plan after simplify_expressions: + Sort: CASE WHEN () = Int32(2) THEN #2 ELSE negative(#2) END AS CASE WHEN #5 = Int32(2) THEN #2 WHEN Boolean(true) THEN negative(#2) END DESC NULLS LAST + Subquery: + Projection: min(s.#4) AS #5 + Aggregate: groupBy=[[]], aggr=[[min(s.#4)]] + SubqueryAlias: s + Projection: #3 AS #4 + Projection: column1 AS #3 + Values: (Int32(2)) + Projection: t.#1 AS #2 + SubqueryAlias: t + Projection: #0 AS #1 + Projection: column1 AS #0 + Values: (Int32(1)), (Int32(2)), (Int32(3)) + + logical_plan after replace_distinct_aggregate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_predicate_subquery: + SAME TEXT AS ABOVE + + logical_plan after scalar_subquery_to_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_join: + SAME TEXT AS ABOVE + + logical_plan after extract_equijoin_predicate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_duplicated_expr: + SAME TEXT AS ABOVE + + logical_plan after eliminate_filter: + SAME TEXT AS ABOVE + + logical_plan after eliminate_cross_join: + SAME TEXT AS ABOVE + + logical_plan after eliminate_limit: + SAME TEXT AS ABOVE + + logical_plan after propagate_empty_relation: + SAME TEXT AS ABOVE + + logical_plan after filter_null_join_keys: + SAME TEXT AS ABOVE + + logical_plan after eliminate_outer_join: + SAME TEXT AS ABOVE + + logical_plan after push_down_limit: + SAME TEXT AS ABOVE + + logical_plan after push_down_filter: + SAME TEXT AS ABOVE + + logical_plan after single_distinct_aggregation_to_group_by: + SAME TEXT AS ABOVE + + logical_plan after eliminate_group_by_constant: + SAME TEXT AS ABOVE + + logical_plan after common_sub_expression_eliminate: + SAME TEXT AS ABOVE + + logical_plan after extract_leaf_expressions: + SAME TEXT AS ABOVE + + logical_plan after optimize_projections: + Sort: CASE WHEN () = Int32(2) THEN #2 ELSE negative(#2) END AS CASE WHEN #5 = Int32(2) THEN #2 WHEN Boolean(true) THEN negative(#2) END DESC NULLS LAST + Subquery: + Projection: min(s.#4) AS #5 + Aggregate: groupBy=[[]], aggr=[[min(s.#4)]] + SubqueryAlias: s + Projection: column1 AS #4 + Values: (Int32(2)) + Projection: t.#1 AS #2 + SubqueryAlias: t + Projection: column1 AS #1 + Values: (Int32(1)), (Int32(2)), (Int32(3)) + + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_projection: + SAME TEXT AS ABOVE + + logical_plan after rewrite_set_comparison: + SAME TEXT AS ABOVE + + logical_plan after optimize_unions: + SAME TEXT AS ABOVE + + logical_plan after unions_to_filter: + SAME TEXT AS ABOVE + + logical_plan after simplify_expressions: + SAME TEXT AS ABOVE + + logical_plan after replace_distinct_aggregate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_predicate_subquery: + SAME TEXT AS ABOVE + + logical_plan after scalar_subquery_to_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_join: + SAME TEXT AS ABOVE + + logical_plan after extract_equijoin_predicate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_duplicated_expr: + SAME TEXT AS ABOVE + + logical_plan after eliminate_filter: + SAME TEXT AS ABOVE + + logical_plan after eliminate_cross_join: + SAME TEXT AS ABOVE + + logical_plan after eliminate_limit: + SAME TEXT AS ABOVE + + logical_plan after propagate_empty_relation: + SAME TEXT AS ABOVE + + logical_plan after filter_null_join_keys: + SAME TEXT AS ABOVE + + logical_plan after eliminate_outer_join: + SAME TEXT AS ABOVE + + logical_plan after push_down_limit: + SAME TEXT AS ABOVE + + logical_plan after push_down_filter: + SAME TEXT AS ABOVE + + logical_plan after single_distinct_aggregation_to_group_by: + SAME TEXT AS ABOVE + + logical_plan after eliminate_group_by_constant: + SAME TEXT AS ABOVE + + logical_plan after common_sub_expression_eliminate: + SAME TEXT AS ABOVE + + logical_plan after extract_leaf_expressions: + SAME TEXT AS ABOVE + + logical_plan after optimize_projections: + SAME TEXT AS ABOVE + + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE + + logical_plan: + Sort: CASE WHEN () = Int32(2) THEN #2 ELSE negative(#2) END AS CASE WHEN #5 = Int32(2) THEN #2 WHEN Boolean(true) THEN negative(#2) END DESC NULLS LAST + Subquery: + Projection: min(s.#4) AS #5 + Aggregate: groupBy=[[]], aggr=[[min(s.#4)]] + SubqueryAlias: s + Projection: column1 AS #4 + Values: (Int32(2)) + Projection: t.#1 AS #2 + SubqueryAlias: t + Projection: column1 AS #1 + Values: (Int32(1)), (Int32(2)), (Int32(3)) + + initial_physical_plan: + ScalarSubqueryExec: subqueries=1 + SortExec: expr=[CASE WHEN scalar_subquery() = 2 THEN #2@0 ELSE negative(#2@0) END DESC NULLS LAST], preserve_partitioning=[false] + ProjectionExec: expr=[#1@0 as #2] + ProjectionExec: expr=[column1@0 as #1] DataSourceExec: partitions=1, partition_sizes=[1] - ProjectionExec: expr=[min(s.#3)@0 as #4] - AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)] - ProjectionExec: expr=[column1@0 as #3] + ProjectionExec: expr=[min(s.#4)@0 as #5] + AggregateExec: mode=Final, gby=[], aggr=[min(s.#4)] + AggregateExec: mode=Partial, gby=[], aggr=[min(s.#4)] + ProjectionExec: expr=[column1@0 as #4] DataSourceExec: partitions=1, partition_sizes=[1] - == Distributed Plan == - === stage 0 === - inputs=[] - mode=Pipelined - partitions=1 - distribution=RoundRobin(channels=1) - placement=Worker - ProjectionExec: expr=[min(s.#3)@0 as #4] - AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)] - ProjectionExec: expr=[column1@0 as #3] - DataSourceExec: partitions=1, partition_sizes=[1] + initial_physical_plan_with_stats: + ScalarSubqueryExec: subqueries=1, statistics=[Rows=Exact(3), Bytes=Exact(), [(Col[0]: Null=Exact(0))]] + SortExec: expr=[CASE WHEN scalar_subquery() = 2 THEN #2@0 ELSE negative(#2@0) END DESC NULLS LAST], preserve_partitioning=[false], statistics=[Rows=Exact(3), Bytes=Exact(), [(Col[0]: Null=Exact(0))]] + ProjectionExec: expr=[#1@0 as #2], statistics=[Rows=Exact(3), Bytes=Exact(), [(Col[0]: Null=Exact(0))]] + ProjectionExec: expr=[column1@0 as #1], statistics=[Rows=Exact(3), Bytes=Exact(), [(Col[0]: Null=Exact(0))]] + DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(3), Bytes=Exact(), [(Col[0]: Null=Exact(0))]] + ProjectionExec: expr=[min(s.#4)@0 as #5], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]:)]] + AggregateExec: mode=Final, gby=[], aggr=[min(s.#4)], statistics=[Rows=Exact(1), Bytes=Absent, [(Col[0]:)]] + AggregateExec: mode=Partial, gby=[], aggr=[min(s.#4)], statistics=[Rows=Exact(0), Bytes=Inexact(), [(Col[0]:)]] + ProjectionExec: expr=[column1@0 as #4], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]: Null=Exact(0))]] + DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]: Null=Exact(0))]] - === stage 1 === - inputs=[StageInput(stage=0, mode=Broadcast)] - mode=Pipelined - partitions=1 - distribution=RoundRobin(channels=1) - placement=Worker - ProjectionExec: expr=[#5@0 as v, #6@1 as maybe_shifted] + + initial_physical_plan_with_schema: + ScalarSubqueryExec: subqueries=1, schema=[#2:Int32;N] + SortExec: expr=[CASE WHEN scalar_subquery() = 2 THEN #2@0 ELSE negative(#2@0) END DESC NULLS LAST], preserve_partitioning=[false], schema=[#2:Int32;N] + ProjectionExec: expr=[#1@0 as #2], schema=[#2:Int32;N] + ProjectionExec: expr=[column1@0 as #1], schema=[#1:Int32;N] + DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N] + ProjectionExec: expr=[min(s.#4)@0 as #5], schema=[#5:Int32;N] + AggregateExec: mode=Final, gby=[], aggr=[min(s.#4)], schema=[min(s.#4):Int32;N] + AggregateExec: mode=Partial, gby=[], aggr=[min(s.#4)], schema=[min(s.#4)[value]:Int32;N] + ProjectionExec: expr=[column1@0 as #4], schema=[#4:Int32;N] + DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N] + + + physical_plan after OutputRequirements: + OutputRequirementExec: order_by=[], dist_by=Unspecified ScalarSubqueryExec: subqueries=1 - SortExec: expr=[#5@0 ASC], preserve_partitioning=[false] - ProjectionExec: expr=[column1@0 as #5, CAST(column1@0 AS Int64) + scalar_subquery() as #6] - DataSourceExec: partitions=1, partition_sizes=[1] - StageInputExec: input=0, partitioning=UnknownPartitioning(1) ---- -name: "test_scalar_subquery_in_aggregate_filter" -data: - |- - == Physical Plan == - ProjectionExec: expr=[#7@0 as k, #8@1 as n] + SortExec: expr=[CASE WHEN scalar_subquery() = 2 THEN #2@0 ELSE negative(#2@0) END DESC NULLS LAST], preserve_partitioning=[false] + ProjectionExec: expr=[#1@0 as #2] + ProjectionExec: expr=[column1@0 as #1] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#4)@0 as #5] + AggregateExec: mode=Final, gby=[], aggr=[min(s.#4)] + AggregateExec: mode=Partial, gby=[], aggr=[min(s.#4)] + ProjectionExec: expr=[column1@0 as #4] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after aggregate_statistics: + SAME TEXT AS ABOVE + + physical_plan after join_selection: + SAME TEXT AS ABOVE + + physical_plan after LimitedDistinctAggregation: + SAME TEXT AS ABOVE + + physical_plan after FilterPushdown: + SAME TEXT AS ABOVE + + physical_plan after EnforceDistribution: + SAME TEXT AS ABOVE + + physical_plan after CombinePartialFinalAggregate: + OutputRequirementExec: order_by=[], dist_by=Unspecified ScalarSubqueryExec: subqueries=1 - ProjectionExec: expr=[#2@0 as #7, count(Int64(1)) FILTER (WHERE t.#3 > ())@1 as #8] - AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[count(Int64(1)) FILTER (WHERE t.#3 > ())] - RepartitionExec: partitioning=Hash([#2@0], 4), input_partitions=1 - AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[count(Int64(1)) FILTER (WHERE t.#3 > ())] - ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] - DataSourceExec: partitions=1, partition_sizes=[1] - ProjectionExec: expr=[min(s.#5)@0 as #6] - AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)] - ProjectionExec: expr=[column1@0 as #5] + SortExec: expr=[CASE WHEN scalar_subquery() = 2 THEN #2@0 ELSE negative(#2@0) END DESC NULLS LAST], preserve_partitioning=[false] + ProjectionExec: expr=[#1@0 as #2] + ProjectionExec: expr=[column1@0 as #1] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#4)@0 as #5] + AggregateExec: mode=Single, gby=[], aggr=[min(s.#4)] + ProjectionExec: expr=[column1@0 as #4] DataSourceExec: partitions=1, partition_sizes=[1] - == Distributed Plan == - === stage 0 === - inputs=[] - mode=Pipelined - partitions=1 - distribution=RoundRobin(channels=1) - placement=Worker - ProjectionExec: expr=[min(s.#5)@0 as #6] - AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)] - ProjectionExec: expr=[column1@0 as #5] - DataSourceExec: partitions=1, partition_sizes=[1] + physical_plan after EnforceSorting: + SAME TEXT AS ABOVE + + physical_plan after OptimizeAggregateOrder: + SAME TEXT AS ABOVE + + physical_plan after WindowTopN: + SAME TEXT AS ABOVE + + physical_plan after ProjectionPushdown: + OutputRequirementExec: order_by=[], dist_by=Unspecified + ScalarSubqueryExec: subqueries=1 + SortExec: expr=[CASE WHEN scalar_subquery() = 2 THEN #2@0 ELSE negative(#2@0) END DESC NULLS LAST], preserve_partitioning=[false] + ProjectionExec: expr=[column1@0 as #2] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#4)@0 as #5] + AggregateExec: mode=Single, gby=[], aggr=[min(s.#4)] + ProjectionExec: expr=[column1@0 as #4] + DataSourceExec: partitions=1, partition_sizes=[1] - === stage 1 === - inputs=[StageInput(stage=0, mode=Broadcast)] - mode=Pipelined - partitions=1 - distribution=Hash(keys=[#2@0], channels=4) - placement=Worker + + physical_plan after OutputRequirements: ScalarSubqueryExec: subqueries=1 - AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[count(Int64(1)) FILTER (WHERE t.#3 > ())] - ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] + SortExec: expr=[CASE WHEN scalar_subquery() = 2 THEN #2@0 ELSE negative(#2@0) END DESC NULLS LAST], preserve_partitioning=[false] + ProjectionExec: expr=[column1@0 as #2] DataSourceExec: partitions=1, partition_sizes=[1] - StageInputExec: input=0, partitioning=UnknownPartitioning(1) + ProjectionExec: expr=[min(s.#4)@0 as #5] + AggregateExec: mode=Single, gby=[], aggr=[min(s.#4)] + ProjectionExec: expr=[column1@0 as #4] + DataSourceExec: partitions=1, partition_sizes=[1] - === stage 2 === - inputs=[StageInput(stage=1, mode=Shuffle), StageInput(stage=0, mode=Broadcast)] - mode=Pipelined - partitions=4 - distribution=RoundRobin(channels=1) - placement=Worker - ProjectionExec: expr=[#7@0 as k, #8@1 as n] + + physical_plan after LimitAggregation: + SAME TEXT AS ABOVE + + physical_plan after LimitPushPastWindows: + SAME TEXT AS ABOVE + + physical_plan after HashJoinBuffering: + SAME TEXT AS ABOVE + + physical_plan after LimitPushdown: + SAME TEXT AS ABOVE + + physical_plan after TopKRepartition: + SAME TEXT AS ABOVE + + physical_plan after ProjectionPushdown: + SAME TEXT AS ABOVE + + physical_plan after PushdownSort: + SAME TEXT AS ABOVE + + physical_plan after EnsureCooperative: + SAME TEXT AS ABOVE + + physical_plan after FilterPushdown(Post): + SAME TEXT AS ABOVE + + physical_plan after RewriteExplicitRepartition: + SAME TEXT AS ABOVE + + physical_plan after RewriteCollectLeftHashJoin: + SAME TEXT AS ABOVE + + physical_plan after EnforceBarrierPartitioning: + SAME TEXT AS ABOVE + + physical_plan after SanityCheckPlan: + SAME TEXT AS ABOVE + + physical_plan: + ProjectionExec: expr=[#2@0 as v] ScalarSubqueryExec: subqueries=1 - ProjectionExec: expr=[#2@0 as #7, count(Int64(1)) FILTER (WHERE t.#3 > ())@1 as #8] - AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[count(Int64(1)) FILTER (WHERE t.#3 > ())] - StageInputExec: input=0, partitioning=Hash([#2@0], 4) - StageInputExec: input=1, partitioning=UnknownPartitioning(1) ---- -name: "test_scalar_subquery_in_aggregate_order_by" -data: - |- + SortExec: expr=[CASE WHEN scalar_subquery() = 2 THEN #2@0 ELSE negative(#2@0) END DESC NULLS LAST], preserve_partitioning=[false] + ProjectionExec: expr=[column1@0 as #2] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#4)@0 as #5] + AggregateExec: mode=Single, gby=[], aggr=[min(s.#4)] + ProjectionExec: expr=[column1@0 as #4] + DataSourceExec: partitions=1, partition_sizes=[1] + + == Physical Plan == - ProjectionExec: expr=[#7@0 as joined] + ProjectionExec: expr=[#2@0 as v] ScalarSubqueryExec: subqueries=1 - ProjectionExec: expr=[CASE WHEN __common_expr_1@0 IS NOT NULL THEN __common_expr_1@0 END as #7] - ProjectionExec: expr=[array_to_string(array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST]@0, ,) as __common_expr_1] - AggregateExec: mode=Single, gby=[], aggr=[array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 ELSE negative(t.#3) END ASC NULLS FIRST] as array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST]] - SortExec: expr=[CASE WHEN scalar_subquery() = 1 THEN #3@1 ELSE negative(#3@1) END ASC], preserve_partitioning=[false] - ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] - DataSourceExec: partitions=1, partition_sizes=[1] - ProjectionExec: expr=[min(s.#5)@0 as #6] - AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)] - ProjectionExec: expr=[column1@0 as #5] + SortExec: expr=[CASE WHEN scalar_subquery() = 2 THEN #2@0 ELSE negative(#2@0) END DESC NULLS LAST], preserve_partitioning=[false] + ProjectionExec: expr=[column1@0 as #2] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#4)@0 as #5] + AggregateExec: mode=Single, gby=[], aggr=[min(s.#4)] + ProjectionExec: expr=[column1@0 as #4] DataSourceExec: partitions=1, partition_sizes=[1] @@ -644,9 +9628,9 @@ data: partitions=1 distribution=RoundRobin(channels=1) placement=Worker - ProjectionExec: expr=[min(s.#5)@0 as #6] - AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)] - ProjectionExec: expr=[column1@0 as #5] + ProjectionExec: expr=[min(s.#4)@0 as #5] + AggregateExec: mode=Single, gby=[], aggr=[min(s.#4)] + ProjectionExec: expr=[column1@0 as #4] DataSourceExec: partitions=1, partition_sizes=[1] === stage 1 === @@ -655,92 +9639,394 @@ data: partitions=1 distribution=RoundRobin(channels=1) placement=Worker - ProjectionExec: expr=[#7@0 as joined] + ProjectionExec: expr=[#2@0 as v] ScalarSubqueryExec: subqueries=1 - ProjectionExec: expr=[CASE WHEN __common_expr_1@0 IS NOT NULL THEN __common_expr_1@0 END as #7] - ProjectionExec: expr=[array_to_string(array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST]@0, ,) as __common_expr_1] - AggregateExec: mode=Single, gby=[], aggr=[array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 ELSE negative(t.#3) END ASC NULLS FIRST] as array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST]] - SortExec: expr=[CASE WHEN scalar_subquery() = 1 THEN #3@1 ELSE negative(#3@1) END ASC], preserve_partitioning=[false] - ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] - DataSourceExec: partitions=1, partition_sizes=[1] + SortExec: expr=[CASE WHEN scalar_subquery() = 2 THEN #2@0 ELSE negative(#2@0) END DESC NULLS LAST], preserve_partitioning=[false] + ProjectionExec: expr=[column1@0 as #2] + DataSourceExec: partitions=1, partition_sizes=[1] StageInputExec: input=0, partitioning=UnknownPartitioning(1) --- -name: "test_scalar_subquery_in_filter_before_aggregate" +name: "test_scalar_subquery_in_ungrouped_aggregate_expression" data: |- - == Physical Plan == - ProjectionExec: expr=[#7@0 as k, #8@1 as total] + == Codegen == + Whole-stage codegen is not supported; showing physical plan instead. + + == Plan Steps == + initial_logical_plan: + Projection: sum(t.#1 + #4) AS total AS #5 + Aggregate: groupBy=[[]], aggr=[[sum(CAST(t.#1 AS Int64) + ())]] + Subquery: + Projection: min(s.#3) AS #4 + Aggregate: groupBy=[[]], aggr=[[min(CAST(s.#3 AS Int64))]] + SubqueryAlias: s + Projection: #2 AS #3 + Projection: column1 AS #2 + Values: (Int32(10)), (Int32(20)) + SubqueryAlias: t + Projection: #0 AS #1 + Projection: column1 AS #0 + Values: (Int32(1)), (Int32(2)), (Int32(3)) + + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE + + logical_plan after resolve_grouping_function: + SAME TEXT AS ABOVE + + logical_plan after type_coercion: + SAME TEXT AS ABOVE + + analyzed_logical_plan: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_projection: + SAME TEXT AS ABOVE + + logical_plan after rewrite_set_comparison: + SAME TEXT AS ABOVE + + logical_plan after optimize_unions: + SAME TEXT AS ABOVE + + logical_plan after unions_to_filter: + SAME TEXT AS ABOVE + + logical_plan after simplify_expressions: + SAME TEXT AS ABOVE + + logical_plan after replace_distinct_aggregate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_predicate_subquery: + SAME TEXT AS ABOVE + + logical_plan after scalar_subquery_to_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_join: + SAME TEXT AS ABOVE + + logical_plan after extract_equijoin_predicate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_duplicated_expr: + SAME TEXT AS ABOVE + + logical_plan after eliminate_filter: + SAME TEXT AS ABOVE + + logical_plan after eliminate_cross_join: + SAME TEXT AS ABOVE + + logical_plan after eliminate_limit: + SAME TEXT AS ABOVE + + logical_plan after propagate_empty_relation: + SAME TEXT AS ABOVE + + logical_plan after filter_null_join_keys: + SAME TEXT AS ABOVE + + logical_plan after eliminate_outer_join: + SAME TEXT AS ABOVE + + logical_plan after push_down_limit: + SAME TEXT AS ABOVE + + logical_plan after push_down_filter: + SAME TEXT AS ABOVE + + logical_plan after single_distinct_aggregation_to_group_by: + SAME TEXT AS ABOVE + + logical_plan after eliminate_group_by_constant: + SAME TEXT AS ABOVE + + logical_plan after common_sub_expression_eliminate: + SAME TEXT AS ABOVE + + logical_plan after extract_leaf_expressions: + SAME TEXT AS ABOVE + + logical_plan after optimize_projections: + Projection: sum(t.#1 + #4) AS total AS #5 + Aggregate: groupBy=[[]], aggr=[[sum(CAST(t.#1 AS Int64) + ())]] + Subquery: + Projection: min(s.#3) AS #4 + Aggregate: groupBy=[[]], aggr=[[min(CAST(s.#3 AS Int64))]] + SubqueryAlias: s + Projection: column1 AS #3 + Values: (Int32(10)), (Int32(20)) + SubqueryAlias: t + Projection: column1 AS #1 + Values: (Int32(1)), (Int32(2)), (Int32(3)) + + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_projection: + SAME TEXT AS ABOVE + + logical_plan after rewrite_set_comparison: + SAME TEXT AS ABOVE + + logical_plan after optimize_unions: + SAME TEXT AS ABOVE + + logical_plan after unions_to_filter: + SAME TEXT AS ABOVE + + logical_plan after simplify_expressions: + SAME TEXT AS ABOVE + + logical_plan after replace_distinct_aggregate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_predicate_subquery: + SAME TEXT AS ABOVE + + logical_plan after scalar_subquery_to_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_join: + SAME TEXT AS ABOVE + + logical_plan after extract_equijoin_predicate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_duplicated_expr: + SAME TEXT AS ABOVE + + logical_plan after eliminate_filter: + SAME TEXT AS ABOVE + + logical_plan after eliminate_cross_join: + SAME TEXT AS ABOVE + + logical_plan after eliminate_limit: + SAME TEXT AS ABOVE + + logical_plan after propagate_empty_relation: + SAME TEXT AS ABOVE + + logical_plan after filter_null_join_keys: + SAME TEXT AS ABOVE + + logical_plan after eliminate_outer_join: + SAME TEXT AS ABOVE + + logical_plan after push_down_limit: + SAME TEXT AS ABOVE + + logical_plan after push_down_filter: + SAME TEXT AS ABOVE + + logical_plan after single_distinct_aggregation_to_group_by: + SAME TEXT AS ABOVE + + logical_plan after eliminate_group_by_constant: + SAME TEXT AS ABOVE + + logical_plan after common_sub_expression_eliminate: + SAME TEXT AS ABOVE + + logical_plan after extract_leaf_expressions: + SAME TEXT AS ABOVE + + logical_plan after optimize_projections: + SAME TEXT AS ABOVE + + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE + + logical_plan: + Projection: sum(t.#1 + #4) AS total AS #5 + Aggregate: groupBy=[[]], aggr=[[sum(CAST(t.#1 AS Int64) + ())]] + Subquery: + Projection: min(s.#3) AS #4 + Aggregate: groupBy=[[]], aggr=[[min(CAST(s.#3 AS Int64))]] + SubqueryAlias: s + Projection: column1 AS #3 + Values: (Int32(10)), (Int32(20)) + SubqueryAlias: t + Projection: column1 AS #1 + Values: (Int32(1)), (Int32(2)), (Int32(3)) + + initial_physical_plan: + ScalarSubqueryExec: subqueries=1 + ProjectionExec: expr=[sum(t.#1 + #4)@0 as #5] + AggregateExec: mode=Final, gby=[], aggr=[sum(t.#1 + #4)] + AggregateExec: mode=Partial, gby=[], aggr=[sum(t.#1 + #4)] + ProjectionExec: expr=[column1@0 as #1] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#3)@0 as #4] + AggregateExec: mode=Final, gby=[], aggr=[min(s.#3)] + AggregateExec: mode=Partial, gby=[], aggr=[min(s.#3)] + ProjectionExec: expr=[column1@0 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + + + initial_physical_plan_with_stats: + ScalarSubqueryExec: subqueries=1, statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]:)]] + ProjectionExec: expr=[sum(t.#1 + #4)@0 as #5], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]:)]] + AggregateExec: mode=Final, gby=[], aggr=[sum(t.#1 + #4)], statistics=[Rows=Exact(1), Bytes=Inexact(), [(Col[0]:)]] + AggregateExec: mode=Partial, gby=[], aggr=[sum(t.#1 + #4)], statistics=[Rows=Inexact(3), Bytes=Inexact(), [(Col[0]:)]] + ProjectionExec: expr=[column1@0 as #1], statistics=[Rows=Exact(3), Bytes=Exact(), [(Col[0]: Null=Exact(0))]] + DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(3), Bytes=Exact(), [(Col[0]: Null=Exact(0))]] + ProjectionExec: expr=[min(s.#3)@0 as #4], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]:)]] + AggregateExec: mode=Final, gby=[], aggr=[min(s.#3)], statistics=[Rows=Exact(1), Bytes=Inexact(), [(Col[0]:)]] + AggregateExec: mode=Partial, gby=[], aggr=[min(s.#3)], statistics=[Rows=Inexact(2), Bytes=Inexact(), [(Col[0]:)]] + ProjectionExec: expr=[column1@0 as #3], statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0))]] + DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0))]] + + + initial_physical_plan_with_schema: + ScalarSubqueryExec: subqueries=1, schema=[#5:Int64;N] + ProjectionExec: expr=[sum(t.#1 + #4)@0 as #5], schema=[#5:Int64;N] + AggregateExec: mode=Final, gby=[], aggr=[sum(t.#1 + #4)], schema=[sum(t.#1 + #4):Int64;N] + AggregateExec: mode=Partial, gby=[], aggr=[sum(t.#1 + #4)], schema=[sum(t.#1 + #4)[sum]:Int64;N] + ProjectionExec: expr=[column1@0 as #1], schema=[#1:Int32;N] + DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N] + ProjectionExec: expr=[min(s.#3)@0 as #4], schema=[#4:Int64;N] + AggregateExec: mode=Final, gby=[], aggr=[min(s.#3)], schema=[min(s.#3):Int64;N] + AggregateExec: mode=Partial, gby=[], aggr=[min(s.#3)], schema=[min(s.#3)[value]:Int64;N] + ProjectionExec: expr=[column1@0 as #3], schema=[#3:Int32;N] + DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N] + + + physical_plan after OutputRequirements: + OutputRequirementExec: order_by=[], dist_by=Unspecified ScalarSubqueryExec: subqueries=1 - ProjectionExec: expr=[#2@0 as #7, sum(t.#3)@1 as #8] - AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[sum(t.#3)] - RepartitionExec: partitioning=Hash([#2@0], 4), input_partitions=4 - AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3)] - RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 - ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] - FilterExec: column2@1 > scalar_subquery() - DataSourceExec: partitions=1, partition_sizes=[1] - ProjectionExec: expr=[max(s.#5)@0 as #6] - AggregateExec: mode=Single, gby=[], aggr=[max(s.#5)] - ProjectionExec: expr=[column1@0 as #5] + ProjectionExec: expr=[sum(t.#1 + #4)@0 as #5] + AggregateExec: mode=Final, gby=[], aggr=[sum(t.#1 + #4)] + AggregateExec: mode=Partial, gby=[], aggr=[sum(t.#1 + #4)] + ProjectionExec: expr=[column1@0 as #1] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#3)@0 as #4] + AggregateExec: mode=Final, gby=[], aggr=[min(s.#3)] + AggregateExec: mode=Partial, gby=[], aggr=[min(s.#3)] + ProjectionExec: expr=[column1@0 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after aggregate_statistics: + SAME TEXT AS ABOVE + + physical_plan after join_selection: + SAME TEXT AS ABOVE + + physical_plan after LimitedDistinctAggregation: + SAME TEXT AS ABOVE + + physical_plan after FilterPushdown: + SAME TEXT AS ABOVE + + physical_plan after EnforceDistribution: + SAME TEXT AS ABOVE + + physical_plan after CombinePartialFinalAggregate: + OutputRequirementExec: order_by=[], dist_by=Unspecified + ScalarSubqueryExec: subqueries=1 + ProjectionExec: expr=[sum(t.#1 + #4)@0 as #5] + AggregateExec: mode=Single, gby=[], aggr=[sum(t.#1 + #4)] + ProjectionExec: expr=[column1@0 as #1] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#3)@0 as #4] + AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)] + ProjectionExec: expr=[column1@0 as #3] DataSourceExec: partitions=1, partition_sizes=[1] - == Distributed Plan == - === stage 0 === - inputs=[] - mode=Pipelined - partitions=1 - distribution=RoundRobin(channels=1) - placement=Worker - ProjectionExec: expr=[max(s.#5)@0 as #6] - AggregateExec: mode=Single, gby=[], aggr=[max(s.#5)] - ProjectionExec: expr=[column1@0 as #5] - DataSourceExec: partitions=1, partition_sizes=[1] + physical_plan after EnforceSorting: + SAME TEXT AS ABOVE - === stage 1 === - inputs=[StageInput(stage=0, mode=Broadcast)] - mode=Pipelined - partitions=1 - distribution=RoundRobin(channels=4) - placement=Worker + physical_plan after OptimizeAggregateOrder: + SAME TEXT AS ABOVE + + physical_plan after WindowTopN: + SAME TEXT AS ABOVE + + physical_plan after ProjectionPushdown: + SAME TEXT AS ABOVE + + physical_plan after OutputRequirements: ScalarSubqueryExec: subqueries=1 - ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] - FilterExec: column2@1 > scalar_subquery() - DataSourceExec: partitions=1, partition_sizes=[1] - StageInputExec: input=0, partitioning=UnknownPartitioning(1) + ProjectionExec: expr=[sum(t.#1 + #4)@0 as #5] + AggregateExec: mode=Single, gby=[], aggr=[sum(t.#1 + #4)] + ProjectionExec: expr=[column1@0 as #1] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#3)@0 as #4] + AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)] + ProjectionExec: expr=[column1@0 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after LimitAggregation: + SAME TEXT AS ABOVE + + physical_plan after LimitPushPastWindows: + SAME TEXT AS ABOVE + + physical_plan after HashJoinBuffering: + SAME TEXT AS ABOVE + + physical_plan after LimitPushdown: + SAME TEXT AS ABOVE + + physical_plan after TopKRepartition: + SAME TEXT AS ABOVE + + physical_plan after ProjectionPushdown: + SAME TEXT AS ABOVE + + physical_plan after PushdownSort: + SAME TEXT AS ABOVE + + physical_plan after EnsureCooperative: + SAME TEXT AS ABOVE + + physical_plan after FilterPushdown(Post): + SAME TEXT AS ABOVE + + physical_plan after RewriteExplicitRepartition: + SAME TEXT AS ABOVE + + physical_plan after RewriteCollectLeftHashJoin: + SAME TEXT AS ABOVE + + physical_plan after EnforceBarrierPartitioning: + SAME TEXT AS ABOVE + + physical_plan after SanityCheckPlan: + SAME TEXT AS ABOVE + + physical_plan: + ProjectionExec: expr=[#5@0 as total] + ScalarSubqueryExec: subqueries=1 + ProjectionExec: expr=[sum(t.#1 + #4)@0 as #5] + AggregateExec: mode=Single, gby=[], aggr=[sum(t.#1 + #4)] + ProjectionExec: expr=[column1@0 as #1] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#3)@0 as #4] + AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)] + ProjectionExec: expr=[column1@0 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] - === stage 2 === - inputs=[StageInput(stage=1, mode=Shuffle)] - mode=Pipelined - partitions=4 - distribution=Hash(keys=[#2@0], channels=4) - placement=Worker - AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3)] - StageInputExec: input=0, partitioning=RoundRobinBatch(4) - === stage 3 === - inputs=[StageInput(stage=2, mode=Shuffle)] - mode=Pipelined - partitions=4 - distribution=RoundRobin(channels=1) - placement=Worker - ProjectionExec: expr=[#7@0 as k, #8@1 as total] - ProjectionExec: expr=[#2@0 as #7, sum(t.#3)@1 as #8] - AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[sum(t.#3)] - StageInputExec: input=0, partitioning=Hash([#2@0], 4) ---- -name: "test_scalar_subquery_in_group_by_expression" -data: - |- == Physical Plan == - ProjectionExec: expr=[#5@0 as g, #6@1 as n] + ProjectionExec: expr=[#5@0 as total] ScalarSubqueryExec: subqueries=1 - ProjectionExec: expr=[g@0 as #5, count(Int64(1))@1 as #6] - AggregateExec: mode=FinalPartitioned, gby=[g@0 as g], aggr=[count(Int64(1))] - RepartitionExec: partitioning=Hash([g@0], 4), input_partitions=1 - AggregateExec: mode=Partial, gby=[CAST(#1@0 AS Int64) + scalar_subquery() as g], aggr=[count(Int64(1))] - ProjectionExec: expr=[column1@0 as #1] - DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[sum(t.#1 + #4)@0 as #5] + AggregateExec: mode=Single, gby=[], aggr=[sum(t.#1 + #4)] + ProjectionExec: expr=[column1@0 as #1] + DataSourceExec: partitions=1, partition_sizes=[1] ProjectionExec: expr=[min(s.#3)@0 as #4] AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)] ProjectionExec: expr=[column1@0 as #3] @@ -763,200 +10049,407 @@ data: inputs=[StageInput(stage=0, mode=Broadcast)] mode=Pipelined partitions=1 - distribution=Hash(keys=[g@0], channels=4) - placement=Worker - ScalarSubqueryExec: subqueries=1 - AggregateExec: mode=Partial, gby=[CAST(#1@0 AS Int64) + scalar_subquery() as g], aggr=[count(Int64(1))] - ProjectionExec: expr=[column1@0 as #1] - DataSourceExec: partitions=1, partition_sizes=[1] - StageInputExec: input=0, partitioning=UnknownPartitioning(1) - - === stage 2 === - inputs=[StageInput(stage=1, mode=Shuffle)] - mode=Pipelined - partitions=4 distribution=RoundRobin(channels=1) placement=Worker - ProjectionExec: expr=[#5@0 as g, #6@1 as n] - ProjectionExec: expr=[g@0 as #5, count(Int64(1))@1 as #6] - AggregateExec: mode=FinalPartitioned, gby=[g@0 as g], aggr=[count(Int64(1))] - StageInputExec: input=0, partitioning=Hash([g@0], 4) + ProjectionExec: expr=[#5@0 as total] + ScalarSubqueryExec: subqueries=1 + ProjectionExec: expr=[sum(t.#1 + #4)@0 as #5] + AggregateExec: mode=Single, gby=[], aggr=[sum(t.#1 + #4)] + ProjectionExec: expr=[column1@0 as #1] + DataSourceExec: partitions=1, partition_sizes=[1] + StageInputExec: input=0, partitioning=UnknownPartitioning(1) --- -name: "test_scalar_subquery_in_grouped_aggregate_expression" +name: "test_scalar_subquery_in_window_aggregate_argument" data: |- - == Physical Plan == - ProjectionExec: expr=[#7@0 as k, #8@1 as total] - ScalarSubqueryExec: subqueries=1 - ProjectionExec: expr=[#2@0 as #7, sum(t.#3 + #6)@1 as #8] - AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[sum(t.#3 + #6)] - RepartitionExec: partitioning=Hash([#2@0], 4), input_partitions=1 - AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3 + #6)] - ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] - DataSourceExec: partitions=1, partition_sizes=[1] - ProjectionExec: expr=[min(s.#5)@0 as #6] - AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)] - ProjectionExec: expr=[column1@0 as #5] - DataSourceExec: partitions=1, partition_sizes=[1] + == Codegen == + Whole-stage codegen is not supported; showing physical plan instead. + == Plan Steps == + initial_logical_plan: + Sort: #6 ASC NULLS FIRST + Projection: t.#1 AS #6, sum(t.#1 + #4) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING AS #5 AS total AS #7 + WindowAggr: windowExpr=[[sum(CAST(t.#1 AS Int64) + ()) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING]] + Subquery: + Projection: min(s.#3) AS #4 + Aggregate: groupBy=[[]], aggr=[[min(CAST(s.#3 AS Int64))]] + SubqueryAlias: s + Projection: #2 AS #3 + Projection: column1 AS #2 + Values: (Int32(10)), (Int32(20)) + SubqueryAlias: t + Projection: #0 AS #1 + Projection: column1 AS #0 + Values: (Int32(1)), (Int32(2)), (Int32(3)) - == Distributed Plan == - === stage 0 === - inputs=[] - mode=Pipelined - partitions=1 - distribution=RoundRobin(channels=1) - placement=Worker - ProjectionExec: expr=[min(s.#5)@0 as #6] - AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)] - ProjectionExec: expr=[column1@0 as #5] - DataSourceExec: partitions=1, partition_sizes=[1] + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE - === stage 1 === - inputs=[StageInput(stage=0, mode=Broadcast)] - mode=Pipelined - partitions=1 - distribution=Hash(keys=[#2@0], channels=4) - placement=Worker - ScalarSubqueryExec: subqueries=1 - AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3 + #6)] - ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] - DataSourceExec: partitions=1, partition_sizes=[1] - StageInputExec: input=0, partitioning=UnknownPartitioning(1) + logical_plan after resolve_grouping_function: + SAME TEXT AS ABOVE - === stage 2 === - inputs=[StageInput(stage=1, mode=Shuffle), StageInput(stage=0, mode=Broadcast)] - mode=Pipelined - partitions=4 - distribution=RoundRobin(channels=1) - placement=Worker - ProjectionExec: expr=[#7@0 as k, #8@1 as total] - ScalarSubqueryExec: subqueries=1 - ProjectionExec: expr=[#2@0 as #7, sum(t.#3 + #6)@1 as #8] - AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[sum(t.#3 + #6)] - StageInputExec: input=0, partitioning=Hash([#2@0], 4) - StageInputExec: input=1, partitioning=UnknownPartitioning(1) ---- -name: "test_scalar_subquery_in_having_filter" -data: - |- - == Physical Plan == - ProjectionExec: expr=[#7@0 as k, #8@1 as total] - ScalarSubqueryExec: subqueries=1 - ProjectionExec: expr=[#2@0 as #7, sum(t.#3)@1 as #8] - FilterExec: sum(t.#3)@1 > scalar_subquery() - AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[sum(t.#3)] - RepartitionExec: partitioning=Hash([#2@0], 4), input_partitions=1 - AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3)] - ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] - DataSourceExec: partitions=1, partition_sizes=[1] - ProjectionExec: expr=[CAST(min(s.#5)@0 AS Int64) as #6] - AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)] - ProjectionExec: expr=[column1@0 as #5] + logical_plan after type_coercion: + SAME TEXT AS ABOVE + + analyzed_logical_plan: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_projection: + SAME TEXT AS ABOVE + + logical_plan after rewrite_set_comparison: + SAME TEXT AS ABOVE + + logical_plan after optimize_unions: + SAME TEXT AS ABOVE + + logical_plan after unions_to_filter: + SAME TEXT AS ABOVE + + logical_plan after simplify_expressions: + SAME TEXT AS ABOVE + + logical_plan after replace_distinct_aggregate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_predicate_subquery: + SAME TEXT AS ABOVE + + logical_plan after scalar_subquery_to_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_join: + SAME TEXT AS ABOVE + + logical_plan after extract_equijoin_predicate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_duplicated_expr: + SAME TEXT AS ABOVE + + logical_plan after eliminate_filter: + SAME TEXT AS ABOVE + + logical_plan after eliminate_cross_join: + SAME TEXT AS ABOVE + + logical_plan after eliminate_limit: + SAME TEXT AS ABOVE + + logical_plan after propagate_empty_relation: + SAME TEXT AS ABOVE + + logical_plan after filter_null_join_keys: + SAME TEXT AS ABOVE + + logical_plan after eliminate_outer_join: + SAME TEXT AS ABOVE + + logical_plan after push_down_limit: + SAME TEXT AS ABOVE + + logical_plan after push_down_filter: + SAME TEXT AS ABOVE + + logical_plan after single_distinct_aggregation_to_group_by: + SAME TEXT AS ABOVE + + logical_plan after eliminate_group_by_constant: + SAME TEXT AS ABOVE + + logical_plan after common_sub_expression_eliminate: + SAME TEXT AS ABOVE + + logical_plan after extract_leaf_expressions: + SAME TEXT AS ABOVE + + logical_plan after optimize_projections: + Sort: #6 ASC NULLS FIRST + Projection: t.#1 AS #6, sum(t.#1 + #4) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING AS #5 AS total AS #7 + WindowAggr: windowExpr=[[sum(CAST(t.#1 AS Int64) + ()) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING]] + Subquery: + Projection: min(s.#3) AS #4 + Aggregate: groupBy=[[]], aggr=[[min(CAST(s.#3 AS Int64))]] + SubqueryAlias: s + Projection: column1 AS #3 + Values: (Int32(10)), (Int32(20)) + SubqueryAlias: t + Projection: column1 AS #1 + Values: (Int32(1)), (Int32(2)), (Int32(3)) + + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_projection: + SAME TEXT AS ABOVE + + logical_plan after rewrite_set_comparison: + SAME TEXT AS ABOVE + + logical_plan after optimize_unions: + SAME TEXT AS ABOVE + + logical_plan after unions_to_filter: + SAME TEXT AS ABOVE + + logical_plan after simplify_expressions: + SAME TEXT AS ABOVE + + logical_plan after replace_distinct_aggregate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_predicate_subquery: + SAME TEXT AS ABOVE + + logical_plan after scalar_subquery_to_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_join: + SAME TEXT AS ABOVE + + logical_plan after extract_equijoin_predicate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_duplicated_expr: + SAME TEXT AS ABOVE + + logical_plan after eliminate_filter: + SAME TEXT AS ABOVE + + logical_plan after eliminate_cross_join: + SAME TEXT AS ABOVE + + logical_plan after eliminate_limit: + SAME TEXT AS ABOVE + + logical_plan after propagate_empty_relation: + SAME TEXT AS ABOVE + + logical_plan after filter_null_join_keys: + SAME TEXT AS ABOVE + + logical_plan after eliminate_outer_join: + SAME TEXT AS ABOVE + + logical_plan after push_down_limit: + SAME TEXT AS ABOVE + + logical_plan after push_down_filter: + SAME TEXT AS ABOVE + + logical_plan after single_distinct_aggregation_to_group_by: + SAME TEXT AS ABOVE + + logical_plan after eliminate_group_by_constant: + SAME TEXT AS ABOVE + + logical_plan after common_sub_expression_eliminate: + SAME TEXT AS ABOVE + + logical_plan after extract_leaf_expressions: + SAME TEXT AS ABOVE + + logical_plan after optimize_projections: + SAME TEXT AS ABOVE + + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE + + logical_plan: + Sort: #6 ASC NULLS FIRST + Projection: t.#1 AS #6, sum(t.#1 + #4) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING AS #5 AS total AS #7 + WindowAggr: windowExpr=[[sum(CAST(t.#1 AS Int64) + ()) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING]] + Subquery: + Projection: min(s.#3) AS #4 + Aggregate: groupBy=[[]], aggr=[[min(CAST(s.#3 AS Int64))]] + SubqueryAlias: s + Projection: column1 AS #3 + Values: (Int32(10)), (Int32(20)) + SubqueryAlias: t + Projection: column1 AS #1 + Values: (Int32(1)), (Int32(2)), (Int32(3)) + + initial_physical_plan: + ScalarSubqueryExec: subqueries=1 + SortExec: expr=[#6@0 ASC], preserve_partitioning=[false] + ProjectionExec: expr=[#1@0 as #6, sum(t.#1 + #4) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@1 as #7] + WindowAggExec: wdw=[sum(t.#1 + #4) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING: Ok(Field { name: "sum(t.#1 + #4) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING", data_type: Int64, nullable: true }), frame: WindowFrame { units: Rows, start_bound: Preceding(UInt64(NULL)), end_bound: Following(UInt64(NULL)), is_causal: false }] + ProjectionExec: expr=[column1@0 as #1] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#3)@0 as #4] + AggregateExec: mode=Final, gby=[], aggr=[min(s.#3)] + AggregateExec: mode=Partial, gby=[], aggr=[min(s.#3)] + ProjectionExec: expr=[column1@0 as #3] DataSourceExec: partitions=1, partition_sizes=[1] - == Distributed Plan == - === stage 0 === - inputs=[] - mode=Pipelined - partitions=1 - distribution=RoundRobin(channels=1) - placement=Worker - ProjectionExec: expr=[CAST(min(s.#5)@0 AS Int64) as #6] - AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)] - ProjectionExec: expr=[column1@0 as #5] - DataSourceExec: partitions=1, partition_sizes=[1] + initial_physical_plan_with_stats: + ScalarSubqueryExec: subqueries=1, statistics=[Rows=Exact(3), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]:)]] + SortExec: expr=[#6@0 ASC], preserve_partitioning=[false], statistics=[Rows=Exact(3), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]:)]] + ProjectionExec: expr=[#1@0 as #6, sum(t.#1 + #4) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@1 as #7], statistics=[Rows=Exact(3), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]:)]] + WindowAggExec: wdw=[sum(t.#1 + #4) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING: Ok(Field { name: "sum(t.#1 + #4) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING", data_type: Int64, nullable: true }), frame: WindowFrame { units: Rows, start_bound: Preceding(UInt64(NULL)), end_bound: Following(UInt64(NULL)), is_causal: false }], statistics=[Rows=Exact(3), Bytes=Absent, [(Col[0]: Null=Exact(0)),(Col[1]:)]] + ProjectionExec: expr=[column1@0 as #1], statistics=[Rows=Exact(3), Bytes=Exact(), [(Col[0]: Null=Exact(0))]] + DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(3), Bytes=Exact(), [(Col[0]: Null=Exact(0))]] + ProjectionExec: expr=[min(s.#3)@0 as #4], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]:)]] + AggregateExec: mode=Final, gby=[], aggr=[min(s.#3)], statistics=[Rows=Exact(1), Bytes=Inexact(), [(Col[0]:)]] + AggregateExec: mode=Partial, gby=[], aggr=[min(s.#3)], statistics=[Rows=Inexact(2), Bytes=Inexact(), [(Col[0]:)]] + ProjectionExec: expr=[column1@0 as #3], statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0))]] + DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0))]] - === stage 1 === - inputs=[] - mode=Pipelined - partitions=1 - distribution=Hash(keys=[#2@0], channels=4) - placement=Worker - AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3)] - ProjectionExec: expr=[column1@0 as #2, column2@1 as #3] - DataSourceExec: partitions=1, partition_sizes=[1] - === stage 2 === - inputs=[StageInput(stage=1, mode=Shuffle), StageInput(stage=0, mode=Broadcast)] - mode=Pipelined - partitions=4 - distribution=RoundRobin(channels=1) - placement=Worker - ProjectionExec: expr=[#7@0 as k, #8@1 as total] - ScalarSubqueryExec: subqueries=1 - ProjectionExec: expr=[#2@0 as #7, sum(t.#3)@1 as #8] - FilterExec: sum(t.#3)@1 > scalar_subquery() - AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[sum(t.#3)] - StageInputExec: input=0, partitioning=Hash([#2@0], 4) - StageInputExec: input=1, partitioning=UnknownPartitioning(1) ---- -name: "test_scalar_subquery_in_join_condition" -data: - |- - == Physical Plan == - ProjectionExec: expr=[#7@0 as v, #8@1 as w] + initial_physical_plan_with_schema: + ScalarSubqueryExec: subqueries=1, schema=[#6:Int32;N, #7:Int64;N] + SortExec: expr=[#6@0 ASC], preserve_partitioning=[false], schema=[#6:Int32;N, #7:Int64;N] + ProjectionExec: expr=[#1@0 as #6, sum(t.#1 + #4) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@1 as #7], schema=[#6:Int32;N, #7:Int64;N] + WindowAggExec: wdw=[sum(t.#1 + #4) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING: Ok(Field { name: "sum(t.#1 + #4) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING", data_type: Int64, nullable: true }), frame: WindowFrame { units: Rows, start_bound: Preceding(UInt64(NULL)), end_bound: Following(UInt64(NULL)), is_causal: false }], schema=[#1:Int32;N, sum(t.#1 + #4) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING:Int64;N] + ProjectionExec: expr=[column1@0 as #1], schema=[#1:Int32;N] + DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N] + ProjectionExec: expr=[min(s.#3)@0 as #4], schema=[#4:Int64;N] + AggregateExec: mode=Final, gby=[], aggr=[min(s.#3)], schema=[min(s.#3):Int64;N] + AggregateExec: mode=Partial, gby=[], aggr=[min(s.#3)], schema=[min(s.#3)[value]:Int64;N] + ProjectionExec: expr=[column1@0 as #3], schema=[#3:Int32;N] + DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N] + + + physical_plan after OutputRequirements: + OutputRequirementExec: order_by=[], dist_by=Unspecified ScalarSubqueryExec: subqueries=1 - SortExec: expr=[#7@0 ASC], preserve_partitioning=[false] - ProjectionExec: expr=[#1@0 as #7, #3@1 as #8] - HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(l.#1 + #6@1, CAST(r.#3 AS Int64)@1)], projection=[#1@0, #3@2] - ProjectionExec: expr=[column1@0 as #1, CAST(column1@0 AS Int64) + scalar_subquery() as l.#1 + #6] + SortExec: expr=[#6@0 ASC], preserve_partitioning=[false] + ProjectionExec: expr=[#1@0 as #6, sum(t.#1 + #4) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@1 as #7] + WindowAggExec: wdw=[sum(t.#1 + #4) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING: Ok(Field { name: "sum(t.#1 + #4) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING", data_type: Int64, nullable: true }), frame: WindowFrame { units: Rows, start_bound: Preceding(UInt64(NULL)), end_bound: Following(UInt64(NULL)), is_causal: false }] + ProjectionExec: expr=[column1@0 as #1] DataSourceExec: partitions=1, partition_sizes=[1] - ProjectionExec: expr=[column1@0 as #3, CAST(column1@0 AS Int64) as CAST(r.#3 AS Int64)] + ProjectionExec: expr=[min(s.#3)@0 as #4] + AggregateExec: mode=Final, gby=[], aggr=[min(s.#3)] + AggregateExec: mode=Partial, gby=[], aggr=[min(s.#3)] + ProjectionExec: expr=[column1@0 as #3] DataSourceExec: partitions=1, partition_sizes=[1] - ProjectionExec: expr=[min(s.#5)@0 as #6] - AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)] - ProjectionExec: expr=[column1@0 as #5] + + + physical_plan after aggregate_statistics: + SAME TEXT AS ABOVE + + physical_plan after join_selection: + SAME TEXT AS ABOVE + + physical_plan after LimitedDistinctAggregation: + SAME TEXT AS ABOVE + + physical_plan after FilterPushdown: + SAME TEXT AS ABOVE + + physical_plan after EnforceDistribution: + SAME TEXT AS ABOVE + + physical_plan after CombinePartialFinalAggregate: + OutputRequirementExec: order_by=[], dist_by=Unspecified + ScalarSubqueryExec: subqueries=1 + SortExec: expr=[#6@0 ASC], preserve_partitioning=[false] + ProjectionExec: expr=[#1@0 as #6, sum(t.#1 + #4) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@1 as #7] + WindowAggExec: wdw=[sum(t.#1 + #4) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING: Ok(Field { name: "sum(t.#1 + #4) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING", data_type: Int64, nullable: true }), frame: WindowFrame { units: Rows, start_bound: Preceding(UInt64(NULL)), end_bound: Following(UInt64(NULL)), is_causal: false }] + ProjectionExec: expr=[column1@0 as #1] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#3)@0 as #4] + AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)] + ProjectionExec: expr=[column1@0 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after EnforceSorting: + SAME TEXT AS ABOVE + + physical_plan after OptimizeAggregateOrder: + SAME TEXT AS ABOVE + + physical_plan after WindowTopN: + SAME TEXT AS ABOVE + + physical_plan after ProjectionPushdown: + SAME TEXT AS ABOVE + + physical_plan after OutputRequirements: + ScalarSubqueryExec: subqueries=1 + SortExec: expr=[#6@0 ASC], preserve_partitioning=[false] + ProjectionExec: expr=[#1@0 as #6, sum(t.#1 + #4) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@1 as #7] + WindowAggExec: wdw=[sum(t.#1 + #4) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING: Ok(Field { name: "sum(t.#1 + #4) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING", data_type: Int64, nullable: true }), frame: WindowFrame { units: Rows, start_bound: Preceding(UInt64(NULL)), end_bound: Following(UInt64(NULL)), is_causal: false }] + ProjectionExec: expr=[column1@0 as #1] DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#3)@0 as #4] + AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)] + ProjectionExec: expr=[column1@0 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] - == Distributed Plan == - === stage 0 === - inputs=[] - mode=Pipelined - partitions=1 - distribution=RoundRobin(channels=1) - placement=Worker - ProjectionExec: expr=[min(s.#5)@0 as #6] - AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)] - ProjectionExec: expr=[column1@0 as #5] - DataSourceExec: partitions=1, partition_sizes=[1] + physical_plan after LimitAggregation: + SAME TEXT AS ABOVE - === stage 1 === - inputs=[StageInput(stage=0, mode=Broadcast)] - mode=Pipelined - partitions=1 - distribution=RoundRobin(channels=1) - placement=Worker - ProjectionExec: expr=[#7@0 as v, #8@1 as w] + physical_plan after LimitPushPastWindows: + SAME TEXT AS ABOVE + + physical_plan after HashJoinBuffering: + SAME TEXT AS ABOVE + + physical_plan after LimitPushdown: + SAME TEXT AS ABOVE + + physical_plan after TopKRepartition: + SAME TEXT AS ABOVE + + physical_plan after ProjectionPushdown: + SAME TEXT AS ABOVE + + physical_plan after PushdownSort: + SAME TEXT AS ABOVE + + physical_plan after EnsureCooperative: + SAME TEXT AS ABOVE + + physical_plan after FilterPushdown(Post): + SAME TEXT AS ABOVE + + physical_plan after RewriteExplicitRepartition: + SAME TEXT AS ABOVE + + physical_plan after RewriteCollectLeftHashJoin: + SAME TEXT AS ABOVE + + physical_plan after EnforceBarrierPartitioning: + SAME TEXT AS ABOVE + + physical_plan after SanityCheckPlan: + SAME TEXT AS ABOVE + + physical_plan: + ProjectionExec: expr=[#6@0 as v, #7@1 as total] ScalarSubqueryExec: subqueries=1 - SortExec: expr=[#7@0 ASC], preserve_partitioning=[false] - ProjectionExec: expr=[#1@0 as #7, #3@1 as #8] - HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(l.#1 + #6@1, CAST(r.#3 AS Int64)@1)], projection=[#1@0, #3@2] - ProjectionExec: expr=[column1@0 as #1, CAST(column1@0 AS Int64) + scalar_subquery() as l.#1 + #6] - DataSourceExec: partitions=1, partition_sizes=[1] - ProjectionExec: expr=[column1@0 as #3, CAST(column1@0 AS Int64) as CAST(r.#3 AS Int64)] + SortExec: expr=[#6@0 ASC], preserve_partitioning=[false] + ProjectionExec: expr=[#1@0 as #6, sum(t.#1 + #4) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@1 as #7] + WindowAggExec: wdw=[sum(t.#1 + #4) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING: Ok(Field { name: "sum(t.#1 + #4) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING", data_type: Int64, nullable: true }), frame: WindowFrame { units: Rows, start_bound: Preceding(UInt64(NULL)), end_bound: Following(UInt64(NULL)), is_causal: false }] + ProjectionExec: expr=[column1@0 as #1] DataSourceExec: partitions=1, partition_sizes=[1] - StageInputExec: input=0, partitioning=UnknownPartitioning(1) ---- -name: "test_scalar_subquery_in_nonequi_join_condition" -data: - |- + ProjectionExec: expr=[min(s.#3)@0 as #4] + AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)] + ProjectionExec: expr=[column1@0 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + + == Physical Plan == - ProjectionExec: expr=[#7@0 as v, #8@1 as w] + ProjectionExec: expr=[#6@0 as v, #7@1 as total] ScalarSubqueryExec: subqueries=1 - SortExec: expr=[#7@0 ASC, #8@1 ASC], preserve_partitioning=[false] - ProjectionExec: expr=[#1@0 as #7, #3@1 as #8] - FilterExec: CAST(#1@0 AS Int64) + scalar_subquery() < CAST(#3@1 AS Int64) - CrossJoinExec - ProjectionExec: expr=[column1@0 as #1] - DataSourceExec: partitions=1, partition_sizes=[1] - ProjectionExec: expr=[column1@0 as #3] - DataSourceExec: partitions=1, partition_sizes=[1] - ProjectionExec: expr=[min(s.#5)@0 as #6] - AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)] - ProjectionExec: expr=[column1@0 as #5] + SortExec: expr=[#6@0 ASC], preserve_partitioning=[false] + ProjectionExec: expr=[#1@0 as #6, sum(t.#1 + #4) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@1 as #7] + WindowAggExec: wdw=[sum(t.#1 + #4) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING: Ok(Field { name: "sum(t.#1 + #4) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING", data_type: Int64, nullable: true }), frame: WindowFrame { units: Rows, start_bound: Preceding(UInt64(NULL)), end_bound: Following(UInt64(NULL)), is_causal: false }] + ProjectionExec: expr=[column1@0 as #1] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#3)@0 as #4] + AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)] + ProjectionExec: expr=[column1@0 as #3] DataSourceExec: partitions=1, partition_sizes=[1] @@ -967,9 +10460,9 @@ data: partitions=1 distribution=RoundRobin(channels=1) placement=Worker - ProjectionExec: expr=[min(s.#5)@0 as #6] - AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)] - ProjectionExec: expr=[column1@0 as #5] + ProjectionExec: expr=[min(s.#3)@0 as #4] + AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)] + ProjectionExec: expr=[column1@0 as #3] DataSourceExec: partitions=1, partition_sizes=[1] === stage 1 === @@ -978,162 +10471,336 @@ data: partitions=1 distribution=RoundRobin(channels=1) placement=Worker - ProjectionExec: expr=[#7@0 as v, #8@1 as w] + ProjectionExec: expr=[#6@0 as v, #7@1 as total] ScalarSubqueryExec: subqueries=1 - SortExec: expr=[#7@0 ASC, #8@1 ASC], preserve_partitioning=[false] - ProjectionExec: expr=[#1@0 as #7, #3@1 as #8] - FilterExec: CAST(#1@0 AS Int64) + scalar_subquery() < CAST(#3@1 AS Int64) - CrossJoinExec - ProjectionExec: expr=[column1@0 as #1] - DataSourceExec: partitions=1, partition_sizes=[1] - ProjectionExec: expr=[column1@0 as #3] - DataSourceExec: partitions=1, partition_sizes=[1] + SortExec: expr=[#6@0 ASC], preserve_partitioning=[false] + ProjectionExec: expr=[#1@0 as #6, sum(t.#1 + #4) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@1 as #7] + WindowAggExec: wdw=[sum(t.#1 + #4) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING: Ok(Field { name: "sum(t.#1 + #4) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING", data_type: Int64, nullable: true }), frame: WindowFrame { units: Rows, start_bound: Preceding(UInt64(NULL)), end_bound: Following(UInt64(NULL)), is_causal: false }] + ProjectionExec: expr=[column1@0 as #1] + DataSourceExec: partitions=1, partition_sizes=[1] StageInputExec: input=0, partitioning=UnknownPartitioning(1) --- -name: "test_scalar_subquery_in_projection_before_aggregate" +name: "test_scalar_subquery_in_window_order_expression" data: |- - == Physical Plan == - ProjectionExec: expr=[#9@0 as k, #10@1 as total] - ScalarSubqueryExec: subqueries=1 - ProjectionExec: expr=[#7@0 as #9, sum(p.#8)@1 as #10] - AggregateExec: mode=FinalPartitioned, gby=[#7@0 as #7], aggr=[sum(p.#8)] - RepartitionExec: partitioning=Hash([#7@0], 4), input_partitions=1 - AggregateExec: mode=Partial, gby=[#7@0 as #7], aggr=[sum(p.#8)] - ProjectionExec: expr=[column1@0 as #7, CAST(column2@1 AS Int64) + scalar_subquery() as #8] - DataSourceExec: partitions=1, partition_sizes=[1] - ProjectionExec: expr=[min(s.#5)@0 as #6] - AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)] - ProjectionExec: expr=[column1@0 as #5] - DataSourceExec: partitions=1, partition_sizes=[1] + == Codegen == + Whole-stage codegen is not supported; showing physical plan instead. + == Plan Steps == + initial_logical_plan: + Sort: #7 ASC NULLS FIRST + Projection: t.#1 AS #6, CAST(row_number() ORDER BY [CASE WHEN () = Int32(2) THEN t.#1 WHEN Boolean(true) THEN negative(t.#1) END ASC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW AS #5 AS Int32) AS rn AS #7 + WindowAggr: windowExpr=[[row_number() ORDER BY [CASE WHEN () = Int32(2) THEN t.#1 WHEN Boolean(true) THEN negative(t.#1) END ASC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW]] + Subquery: + Projection: min(s.#3) AS #4 + Aggregate: groupBy=[[]], aggr=[[min(s.#3)]] + SubqueryAlias: s + Projection: #2 AS #3 + Projection: column1 AS #2 + Values: (Int32(2)) + SubqueryAlias: t + Projection: #0 AS #1 + Projection: column1 AS #0 + Values: (Int32(3)), (Int32(1)), (Int32(2)) - == Distributed Plan == - === stage 0 === - inputs=[] - mode=Pipelined - partitions=1 - distribution=RoundRobin(channels=1) - placement=Worker - ProjectionExec: expr=[min(s.#5)@0 as #6] - AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)] - ProjectionExec: expr=[column1@0 as #5] - DataSourceExec: partitions=1, partition_sizes=[1] + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE - === stage 1 === - inputs=[StageInput(stage=0, mode=Broadcast)] - mode=Pipelined - partitions=1 - distribution=Hash(keys=[#7@0], channels=4) - placement=Worker - ScalarSubqueryExec: subqueries=1 - AggregateExec: mode=Partial, gby=[#7@0 as #7], aggr=[sum(p.#8)] - ProjectionExec: expr=[column1@0 as #7, CAST(column2@1 AS Int64) + scalar_subquery() as #8] - DataSourceExec: partitions=1, partition_sizes=[1] - StageInputExec: input=0, partitioning=UnknownPartitioning(1) + logical_plan after resolve_grouping_function: + SAME TEXT AS ABOVE - === stage 2 === - inputs=[StageInput(stage=1, mode=Shuffle)] - mode=Pipelined - partitions=4 - distribution=RoundRobin(channels=1) - placement=Worker - ProjectionExec: expr=[#9@0 as k, #10@1 as total] - ProjectionExec: expr=[#7@0 as #9, sum(p.#8)@1 as #10] - AggregateExec: mode=FinalPartitioned, gby=[#7@0 as #7], aggr=[sum(p.#8)] - StageInputExec: input=0, partitioning=Hash([#7@0], 4) ---- -name: "test_scalar_subquery_in_sort_expression" -data: - |- - == Physical Plan == - ProjectionExec: expr=[#2@0 as v] - ScalarSubqueryExec: subqueries=1 - SortExec: expr=[CASE WHEN scalar_subquery() = 2 THEN #2@0 ELSE negative(#2@0) END DESC NULLS LAST], preserve_partitioning=[false] - ProjectionExec: expr=[column1@0 as #2] - DataSourceExec: partitions=1, partition_sizes=[1] - ProjectionExec: expr=[min(s.#4)@0 as #5] - AggregateExec: mode=Single, gby=[], aggr=[min(s.#4)] - ProjectionExec: expr=[column1@0 as #4] + logical_plan after type_coercion: + SAME TEXT AS ABOVE + + analyzed_logical_plan: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_projection: + SAME TEXT AS ABOVE + + logical_plan after rewrite_set_comparison: + SAME TEXT AS ABOVE + + logical_plan after optimize_unions: + SAME TEXT AS ABOVE + + logical_plan after unions_to_filter: + SAME TEXT AS ABOVE + + logical_plan after simplify_expressions: + Sort: #7 ASC NULLS FIRST + Projection: t.#1 AS #6, CAST(row_number() ORDER BY [CASE WHEN () = Int32(2) THEN t.#1 WHEN Boolean(true) THEN negative(t.#1) END ASC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW AS #5 AS Int32) AS rn AS #7 + WindowAggr: windowExpr=[[row_number() ORDER BY [CASE WHEN () = Int32(2) THEN t.#1 ELSE negative(t.#1) END ASC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW AS row_number() ORDER BY [CASE WHEN () = Int32(2) THEN t.#1 WHEN Boolean(true) THEN negative(t.#1) END ASC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW]] + Subquery: + Projection: min(s.#3) AS #4 + Aggregate: groupBy=[[]], aggr=[[min(s.#3)]] + SubqueryAlias: s + Projection: #2 AS #3 + Projection: column1 AS #2 + Values: (Int32(2)) + SubqueryAlias: t + Projection: #0 AS #1 + Projection: column1 AS #0 + Values: (Int32(3)), (Int32(1)), (Int32(2)) + + logical_plan after replace_distinct_aggregate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_predicate_subquery: + SAME TEXT AS ABOVE + + logical_plan after scalar_subquery_to_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_join: + SAME TEXT AS ABOVE + + logical_plan after extract_equijoin_predicate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_duplicated_expr: + SAME TEXT AS ABOVE + + logical_plan after eliminate_filter: + SAME TEXT AS ABOVE + + logical_plan after eliminate_cross_join: + SAME TEXT AS ABOVE + + logical_plan after eliminate_limit: + SAME TEXT AS ABOVE + + logical_plan after propagate_empty_relation: + SAME TEXT AS ABOVE + + logical_plan after filter_null_join_keys: + SAME TEXT AS ABOVE + + logical_plan after eliminate_outer_join: + SAME TEXT AS ABOVE + + logical_plan after push_down_limit: + SAME TEXT AS ABOVE + + logical_plan after push_down_filter: + SAME TEXT AS ABOVE + + logical_plan after single_distinct_aggregation_to_group_by: + SAME TEXT AS ABOVE + + logical_plan after eliminate_group_by_constant: + SAME TEXT AS ABOVE + + logical_plan after common_sub_expression_eliminate: + SAME TEXT AS ABOVE + + logical_plan after extract_leaf_expressions: + SAME TEXT AS ABOVE + + logical_plan after optimize_projections: + Sort: #7 ASC NULLS FIRST + Projection: t.#1 AS #6, CAST(row_number() ORDER BY [CASE WHEN () = Int32(2) THEN t.#1 WHEN Boolean(true) THEN negative(t.#1) END ASC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW AS #5 AS Int32) AS rn AS #7 + WindowAggr: windowExpr=[[row_number() ORDER BY [CASE WHEN () = Int32(2) THEN t.#1 ELSE negative(t.#1) END ASC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW AS row_number() ORDER BY [CASE WHEN () = Int32(2) THEN t.#1 WHEN Boolean(true) THEN negative(t.#1) END ASC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW]] + Subquery: + Projection: min(s.#3) AS #4 + Aggregate: groupBy=[[]], aggr=[[min(s.#3)]] + SubqueryAlias: s + Projection: column1 AS #3 + Values: (Int32(2)) + SubqueryAlias: t + Projection: column1 AS #1 + Values: (Int32(3)), (Int32(1)), (Int32(2)) + + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_projection: + SAME TEXT AS ABOVE + + logical_plan after rewrite_set_comparison: + SAME TEXT AS ABOVE + + logical_plan after optimize_unions: + SAME TEXT AS ABOVE + + logical_plan after unions_to_filter: + SAME TEXT AS ABOVE + + logical_plan after simplify_expressions: + SAME TEXT AS ABOVE + + logical_plan after replace_distinct_aggregate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_predicate_subquery: + SAME TEXT AS ABOVE + + logical_plan after scalar_subquery_to_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_join: + SAME TEXT AS ABOVE + + logical_plan after extract_equijoin_predicate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_duplicated_expr: + SAME TEXT AS ABOVE + + logical_plan after eliminate_filter: + SAME TEXT AS ABOVE + + logical_plan after eliminate_cross_join: + SAME TEXT AS ABOVE + + logical_plan after eliminate_limit: + SAME TEXT AS ABOVE + + logical_plan after propagate_empty_relation: + SAME TEXT AS ABOVE + + logical_plan after filter_null_join_keys: + SAME TEXT AS ABOVE + + logical_plan after eliminate_outer_join: + SAME TEXT AS ABOVE + + logical_plan after push_down_limit: + SAME TEXT AS ABOVE + + logical_plan after push_down_filter: + SAME TEXT AS ABOVE + + logical_plan after single_distinct_aggregation_to_group_by: + SAME TEXT AS ABOVE + + logical_plan after eliminate_group_by_constant: + SAME TEXT AS ABOVE + + logical_plan after common_sub_expression_eliminate: + SAME TEXT AS ABOVE + + logical_plan after extract_leaf_expressions: + SAME TEXT AS ABOVE + + logical_plan after optimize_projections: + SAME TEXT AS ABOVE + + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE + + logical_plan: + Sort: #7 ASC NULLS FIRST + Projection: t.#1 AS #6, CAST(row_number() ORDER BY [CASE WHEN () = Int32(2) THEN t.#1 WHEN Boolean(true) THEN negative(t.#1) END ASC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW AS #5 AS Int32) AS rn AS #7 + WindowAggr: windowExpr=[[row_number() ORDER BY [CASE WHEN () = Int32(2) THEN t.#1 ELSE negative(t.#1) END ASC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW AS row_number() ORDER BY [CASE WHEN () = Int32(2) THEN t.#1 WHEN Boolean(true) THEN negative(t.#1) END ASC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW]] + Subquery: + Projection: min(s.#3) AS #4 + Aggregate: groupBy=[[]], aggr=[[min(s.#3)]] + SubqueryAlias: s + Projection: column1 AS #3 + Values: (Int32(2)) + SubqueryAlias: t + Projection: column1 AS #1 + Values: (Int32(3)), (Int32(1)), (Int32(2)) + + initial_physical_plan: + ScalarSubqueryExec: subqueries=1 + SortExec: expr=[#7@1 ASC], preserve_partitioning=[false] + ProjectionExec: expr=[#1@0 as #6, CAST(row_number() ORDER BY [CASE WHEN () = Int32(2) THEN t.#1 WHEN Boolean(true) THEN negative(t.#1) END ASC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@1 AS Int32) as #7] + BoundedWindowAggExec: wdw=[row_number() ORDER BY [CASE WHEN () = Int32(2) THEN t.#1 WHEN Boolean(true) THEN negative(t.#1) END ASC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "row_number() ORDER BY [CASE WHEN () = Int32(2) THEN t.#1 WHEN Boolean(true) THEN negative(t.#1) END ASC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] + ProjectionExec: expr=[column1@0 as #1] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#3)@0 as #4] + AggregateExec: mode=Final, gby=[], aggr=[min(s.#3)] + AggregateExec: mode=Partial, gby=[], aggr=[min(s.#3)] + ProjectionExec: expr=[column1@0 as #3] DataSourceExec: partitions=1, partition_sizes=[1] - == Distributed Plan == - === stage 0 === - inputs=[] - mode=Pipelined - partitions=1 - distribution=RoundRobin(channels=1) - placement=Worker - ProjectionExec: expr=[min(s.#4)@0 as #5] - AggregateExec: mode=Single, gby=[], aggr=[min(s.#4)] - ProjectionExec: expr=[column1@0 as #4] - DataSourceExec: partitions=1, partition_sizes=[1] + initial_physical_plan_with_stats: + ScalarSubqueryExec: subqueries=1, statistics=[Rows=Exact(3), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]:)]] + SortExec: expr=[#7@1 ASC], preserve_partitioning=[false], statistics=[Rows=Exact(3), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]:)]] + ProjectionExec: expr=[#1@0 as #6, CAST(row_number() ORDER BY [CASE WHEN () = Int32(2) THEN t.#1 WHEN Boolean(true) THEN negative(t.#1) END ASC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@1 AS Int32) as #7], statistics=[Rows=Exact(3), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]:)]] + BoundedWindowAggExec: wdw=[row_number() ORDER BY [CASE WHEN () = Int32(2) THEN t.#1 WHEN Boolean(true) THEN negative(t.#1) END ASC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "row_number() ORDER BY [CASE WHEN () = Int32(2) THEN t.#1 WHEN Boolean(true) THEN negative(t.#1) END ASC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted], statistics=[Rows=Exact(3), Bytes=Absent, [(Col[0]: Null=Exact(0)),(Col[1]:)]] + ProjectionExec: expr=[column1@0 as #1], statistics=[Rows=Exact(3), Bytes=Exact(), [(Col[0]: Null=Exact(0))]] + DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(3), Bytes=Exact(), [(Col[0]: Null=Exact(0))]] + ProjectionExec: expr=[min(s.#3)@0 as #4], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]:)]] + AggregateExec: mode=Final, gby=[], aggr=[min(s.#3)], statistics=[Rows=Exact(1), Bytes=Absent, [(Col[0]:)]] + AggregateExec: mode=Partial, gby=[], aggr=[min(s.#3)], statistics=[Rows=Exact(0), Bytes=Inexact(), [(Col[0]:)]] + ProjectionExec: expr=[column1@0 as #3], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]: Null=Exact(0))]] + DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]: Null=Exact(0))]] - === stage 1 === - inputs=[StageInput(stage=0, mode=Broadcast)] - mode=Pipelined - partitions=1 - distribution=RoundRobin(channels=1) - placement=Worker - ProjectionExec: expr=[#2@0 as v] + + initial_physical_plan_with_schema: + ScalarSubqueryExec: subqueries=1, schema=[#6:Int32;N, #7:Int32] + SortExec: expr=[#7@1 ASC], preserve_partitioning=[false], schema=[#6:Int32;N, #7:Int32] + ProjectionExec: expr=[#1@0 as #6, CAST(row_number() ORDER BY [CASE WHEN () = Int32(2) THEN t.#1 WHEN Boolean(true) THEN negative(t.#1) END ASC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@1 AS Int32) as #7], schema=[#6:Int32;N, #7:Int32] + BoundedWindowAggExec: wdw=[row_number() ORDER BY [CASE WHEN () = Int32(2) THEN t.#1 WHEN Boolean(true) THEN negative(t.#1) END ASC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "row_number() ORDER BY [CASE WHEN () = Int32(2) THEN t.#1 WHEN Boolean(true) THEN negative(t.#1) END ASC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted], schema=[#1:Int32;N, row_number() ORDER BY [CASE WHEN () = Int32(2) THEN t.#1 WHEN Boolean(true) THEN negative(t.#1) END ASC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW:UInt64] + ProjectionExec: expr=[column1@0 as #1], schema=[#1:Int32;N] + DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N] + ProjectionExec: expr=[min(s.#3)@0 as #4], schema=[#4:Int32;N] + AggregateExec: mode=Final, gby=[], aggr=[min(s.#3)], schema=[min(s.#3):Int32;N] + AggregateExec: mode=Partial, gby=[], aggr=[min(s.#3)], schema=[min(s.#3)[value]:Int32;N] + ProjectionExec: expr=[column1@0 as #3], schema=[#3:Int32;N] + DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N] + + + physical_plan after OutputRequirements: + OutputRequirementExec: order_by=[], dist_by=Unspecified ScalarSubqueryExec: subqueries=1 - SortExec: expr=[CASE WHEN scalar_subquery() = 2 THEN #2@0 ELSE negative(#2@0) END DESC NULLS LAST], preserve_partitioning=[false] - ProjectionExec: expr=[column1@0 as #2] - DataSourceExec: partitions=1, partition_sizes=[1] - StageInputExec: input=0, partitioning=UnknownPartitioning(1) ---- -name: "test_scalar_subquery_in_ungrouped_aggregate_expression" -data: - |- - == Physical Plan == - ProjectionExec: expr=[#5@0 as total] + SortExec: expr=[#7@1 ASC], preserve_partitioning=[false] + ProjectionExec: expr=[#1@0 as #6, CAST(row_number() ORDER BY [CASE WHEN () = Int32(2) THEN t.#1 WHEN Boolean(true) THEN negative(t.#1) END ASC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@1 AS Int32) as #7] + BoundedWindowAggExec: wdw=[row_number() ORDER BY [CASE WHEN () = Int32(2) THEN t.#1 WHEN Boolean(true) THEN negative(t.#1) END ASC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "row_number() ORDER BY [CASE WHEN () = Int32(2) THEN t.#1 WHEN Boolean(true) THEN negative(t.#1) END ASC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] + ProjectionExec: expr=[column1@0 as #1] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#3)@0 as #4] + AggregateExec: mode=Final, gby=[], aggr=[min(s.#3)] + AggregateExec: mode=Partial, gby=[], aggr=[min(s.#3)] + ProjectionExec: expr=[column1@0 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after aggregate_statistics: + SAME TEXT AS ABOVE + + physical_plan after join_selection: + SAME TEXT AS ABOVE + + physical_plan after LimitedDistinctAggregation: + SAME TEXT AS ABOVE + + physical_plan after FilterPushdown: + SAME TEXT AS ABOVE + + physical_plan after EnforceDistribution: + SAME TEXT AS ABOVE + + physical_plan after CombinePartialFinalAggregate: + OutputRequirementExec: order_by=[], dist_by=Unspecified ScalarSubqueryExec: subqueries=1 - ProjectionExec: expr=[sum(t.#1 + #4)@0 as #5] - AggregateExec: mode=Single, gby=[], aggr=[sum(t.#1 + #4)] - ProjectionExec: expr=[column1@0 as #1] - DataSourceExec: partitions=1, partition_sizes=[1] + SortExec: expr=[#7@1 ASC], preserve_partitioning=[false] + ProjectionExec: expr=[#1@0 as #6, CAST(row_number() ORDER BY [CASE WHEN () = Int32(2) THEN t.#1 WHEN Boolean(true) THEN negative(t.#1) END ASC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@1 AS Int32) as #7] + BoundedWindowAggExec: wdw=[row_number() ORDER BY [CASE WHEN () = Int32(2) THEN t.#1 WHEN Boolean(true) THEN negative(t.#1) END ASC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "row_number() ORDER BY [CASE WHEN () = Int32(2) THEN t.#1 WHEN Boolean(true) THEN negative(t.#1) END ASC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] + ProjectionExec: expr=[column1@0 as #1] + DataSourceExec: partitions=1, partition_sizes=[1] ProjectionExec: expr=[min(s.#3)@0 as #4] AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)] ProjectionExec: expr=[column1@0 as #3] DataSourceExec: partitions=1, partition_sizes=[1] - == Distributed Plan == - === stage 0 === - inputs=[] - mode=Pipelined - partitions=1 - distribution=RoundRobin(channels=1) - placement=Worker - ProjectionExec: expr=[min(s.#3)@0 as #4] - AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)] - ProjectionExec: expr=[column1@0 as #3] - DataSourceExec: partitions=1, partition_sizes=[1] - - === stage 1 === - inputs=[StageInput(stage=0, mode=Broadcast)] - mode=Pipelined - partitions=1 - distribution=RoundRobin(channels=1) - placement=Worker - ProjectionExec: expr=[#5@0 as total] - ScalarSubqueryExec: subqueries=1 - ProjectionExec: expr=[sum(t.#1 + #4)@0 as #5] - AggregateExec: mode=Single, gby=[], aggr=[sum(t.#1 + #4)] - ProjectionExec: expr=[column1@0 as #1] - DataSourceExec: partitions=1, partition_sizes=[1] - StageInputExec: input=0, partitioning=UnknownPartitioning(1) ---- -name: "test_scalar_subquery_in_window_aggregate_argument" -data: - |- - == Physical Plan == - ProjectionExec: expr=[#6@0 as v, #7@1 as total] + physical_plan after EnforceSorting: + OutputRequirementExec: order_by=[], dist_by=Unspecified ScalarSubqueryExec: subqueries=1 - SortExec: expr=[#6@0 ASC], preserve_partitioning=[false] - ProjectionExec: expr=[#1@0 as #6, sum(t.#1 + #4) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@1 as #7] - WindowAggExec: wdw=[sum(t.#1 + #4) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING: Ok(Field { name: "sum(t.#1 + #4) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING", data_type: Int64, nullable: true }), frame: WindowFrame { units: Rows, start_bound: Preceding(UInt64(NULL)), end_bound: Following(UInt64(NULL)), is_causal: false }] + ProjectionExec: expr=[#1@0 as #6, CAST(row_number() ORDER BY [CASE WHEN () = Int32(2) THEN t.#1 WHEN Boolean(true) THEN negative(t.#1) END ASC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@1 AS Int32) as #7] + BoundedWindowAggExec: wdw=[row_number() ORDER BY [CASE WHEN () = Int32(2) THEN t.#1 WHEN Boolean(true) THEN negative(t.#1) END ASC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "row_number() ORDER BY [CASE WHEN () = Int32(2) THEN t.#1 WHEN Boolean(true) THEN negative(t.#1) END ASC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] + SortExec: expr=[CASE WHEN scalar_subquery() = 2 THEN #1@0 ELSE negative(#1@0) END ASC], preserve_partitioning=[false] ProjectionExec: expr=[column1@0 as #1] DataSourceExec: partitions=1, partition_sizes=[1] ProjectionExec: expr=[min(s.#3)@0 as #4] @@ -1142,36 +10809,81 @@ data: DataSourceExec: partitions=1, partition_sizes=[1] - == Distributed Plan == - === stage 0 === - inputs=[] - mode=Pipelined - partitions=1 - distribution=RoundRobin(channels=1) - placement=Worker - ProjectionExec: expr=[min(s.#3)@0 as #4] - AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)] - ProjectionExec: expr=[column1@0 as #3] - DataSourceExec: partitions=1, partition_sizes=[1] + physical_plan after OptimizeAggregateOrder: + SAME TEXT AS ABOVE - === stage 1 === - inputs=[StageInput(stage=0, mode=Broadcast)] - mode=Pipelined - partitions=1 - distribution=RoundRobin(channels=1) - placement=Worker - ProjectionExec: expr=[#6@0 as v, #7@1 as total] + physical_plan after WindowTopN: + SAME TEXT AS ABOVE + + physical_plan after ProjectionPushdown: + SAME TEXT AS ABOVE + + physical_plan after OutputRequirements: + ScalarSubqueryExec: subqueries=1 + ProjectionExec: expr=[#1@0 as #6, CAST(row_number() ORDER BY [CASE WHEN () = Int32(2) THEN t.#1 WHEN Boolean(true) THEN negative(t.#1) END ASC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@1 AS Int32) as #7] + BoundedWindowAggExec: wdw=[row_number() ORDER BY [CASE WHEN () = Int32(2) THEN t.#1 WHEN Boolean(true) THEN negative(t.#1) END ASC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "row_number() ORDER BY [CASE WHEN () = Int32(2) THEN t.#1 WHEN Boolean(true) THEN negative(t.#1) END ASC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] + SortExec: expr=[CASE WHEN scalar_subquery() = 2 THEN #1@0 ELSE negative(#1@0) END ASC], preserve_partitioning=[false] + ProjectionExec: expr=[column1@0 as #1] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#3)@0 as #4] + AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)] + ProjectionExec: expr=[column1@0 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after LimitAggregation: + SAME TEXT AS ABOVE + + physical_plan after LimitPushPastWindows: + SAME TEXT AS ABOVE + + physical_plan after HashJoinBuffering: + SAME TEXT AS ABOVE + + physical_plan after LimitPushdown: + SAME TEXT AS ABOVE + + physical_plan after TopKRepartition: + SAME TEXT AS ABOVE + + physical_plan after ProjectionPushdown: + SAME TEXT AS ABOVE + + physical_plan after PushdownSort: + SAME TEXT AS ABOVE + + physical_plan after EnsureCooperative: + SAME TEXT AS ABOVE + + physical_plan after FilterPushdown(Post): + SAME TEXT AS ABOVE + + physical_plan after RewriteExplicitRepartition: + SAME TEXT AS ABOVE + + physical_plan after RewriteCollectLeftHashJoin: + SAME TEXT AS ABOVE + + physical_plan after EnforceBarrierPartitioning: + SAME TEXT AS ABOVE + + physical_plan after SanityCheckPlan: + SAME TEXT AS ABOVE + + physical_plan: + ProjectionExec: expr=[#6@0 as v, #7@1 as rn] ScalarSubqueryExec: subqueries=1 - SortExec: expr=[#6@0 ASC], preserve_partitioning=[false] - ProjectionExec: expr=[#1@0 as #6, sum(t.#1 + #4) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@1 as #7] - WindowAggExec: wdw=[sum(t.#1 + #4) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING: Ok(Field { name: "sum(t.#1 + #4) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING", data_type: Int64, nullable: true }), frame: WindowFrame { units: Rows, start_bound: Preceding(UInt64(NULL)), end_bound: Following(UInt64(NULL)), is_causal: false }] + ProjectionExec: expr=[#1@0 as #6, CAST(row_number() ORDER BY [CASE WHEN () = Int32(2) THEN t.#1 WHEN Boolean(true) THEN negative(t.#1) END ASC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@1 AS Int32) as #7] + BoundedWindowAggExec: wdw=[row_number() ORDER BY [CASE WHEN () = Int32(2) THEN t.#1 WHEN Boolean(true) THEN negative(t.#1) END ASC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "row_number() ORDER BY [CASE WHEN () = Int32(2) THEN t.#1 WHEN Boolean(true) THEN negative(t.#1) END ASC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] + SortExec: expr=[CASE WHEN scalar_subquery() = 2 THEN #1@0 ELSE negative(#1@0) END ASC], preserve_partitioning=[false] ProjectionExec: expr=[column1@0 as #1] DataSourceExec: partitions=1, partition_sizes=[1] - StageInputExec: input=0, partitioning=UnknownPartitioning(1) ---- -name: "test_scalar_subquery_in_window_order_expression" -data: - |- + ProjectionExec: expr=[min(s.#3)@0 as #4] + AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)] + ProjectionExec: expr=[column1@0 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + + == Physical Plan == ProjectionExec: expr=[#6@0 as v, #7@1 as rn] ScalarSubqueryExec: subqueries=1 @@ -1216,6 +10928,423 @@ data: name: "test_scalar_subquery_in_window_partition_expression" data: |- + == Codegen == + Whole-stage codegen is not supported; showing physical plan instead. + + == Plan Steps == + initial_logical_plan: + Sort: #6 ASC NULLS FIRST + Projection: t.#1 AS #6, count(Int64(1)) PARTITION BY [CASE WHEN #4 = Int32(1) THEN t.#1 % CASE WHEN Int32(2) = Int32(0) THEN raise_error(Utf8("Remainder by zero")) ELSE Int32(2) END WHEN Boolean(true) THEN t.#1 END] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING AS #5 AS n AS #7 + WindowAggr: windowExpr=[[count(Int64(1)) PARTITION BY [CASE WHEN () = Int32(1) THEN t.#1 % CASE WHEN Int32(2) = Int32(0) THEN raise_error(Utf8("Remainder by zero")) ELSE Int32(2) END WHEN Boolean(true) THEN t.#1 END] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING]] + Subquery: + Projection: min(s.#3) AS #4 + Aggregate: groupBy=[[]], aggr=[[min(s.#3)]] + SubqueryAlias: s + Projection: #2 AS #3 + Projection: column1 AS #2 + Values: (Int32(1)) + SubqueryAlias: t + Projection: #0 AS #1 + Projection: column1 AS #0 + Values: (Int32(1)), (Int32(2)), (Int32(3)), (Int32(4)) + + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE + + logical_plan after resolve_grouping_function: + SAME TEXT AS ABOVE + + logical_plan after type_coercion: + Sort: #6 ASC NULLS FIRST + Projection: t.#1 AS #6, count(Int64(1)) PARTITION BY [CASE WHEN #4 = Int32(1) THEN t.#1 % CASE WHEN Int32(2) = Int32(0) THEN raise_error(Utf8("Remainder by zero")) ELSE Int32(2) END WHEN Boolean(true) THEN t.#1 END] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING AS #5 AS n AS #7 + WindowAggr: windowExpr=[[count(Int64(1)) PARTITION BY [CASE WHEN () = Int32(1) THEN t.#1 % CASE WHEN Int32(2) = Int32(0) THEN CAST(raise_error(Utf8("Remainder by zero")) AS Int32) ELSE Int32(2) END WHEN Boolean(true) THEN t.#1 END] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING]] + Subquery: + Projection: min(s.#3) AS #4 + Aggregate: groupBy=[[]], aggr=[[min(s.#3)]] + SubqueryAlias: s + Projection: #2 AS #3 + Projection: column1 AS #2 + Values: (Int32(1)) + SubqueryAlias: t + Projection: #0 AS #1 + Projection: column1 AS #0 + Values: (Int32(1)), (Int32(2)), (Int32(3)), (Int32(4)) + + analyzed_logical_plan: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_projection: + SAME TEXT AS ABOVE + + logical_plan after rewrite_set_comparison: + SAME TEXT AS ABOVE + + logical_plan after optimize_unions: + SAME TEXT AS ABOVE + + logical_plan after unions_to_filter: + SAME TEXT AS ABOVE + + logical_plan after simplify_expressions: + Sort: #6 ASC NULLS FIRST + Projection: t.#1 AS #6, count(Int64(1)) PARTITION BY [CASE WHEN #4 = Int32(1) THEN t.#1 % CASE WHEN Int32(2) = Int32(0) THEN raise_error(Utf8("Remainder by zero")) ELSE Int32(2) END WHEN Boolean(true) THEN t.#1 END] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING AS #5 AS n AS #7 + WindowAggr: windowExpr=[[count(Int64(1)) PARTITION BY [CASE WHEN () = Int32(1) THEN t.#1 % Int32(2) ELSE t.#1 END] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING AS count(Int64(1)) PARTITION BY [CASE WHEN #4 = Int32(1) THEN t.#1 % CASE WHEN Int32(2) = Int32(0) THEN raise_error(Utf8("Remainder by zero")) ELSE Int32(2) END WHEN Boolean(true) THEN t.#1 END] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING]] + Subquery: + Projection: min(s.#3) AS #4 + Aggregate: groupBy=[[]], aggr=[[min(s.#3)]] + SubqueryAlias: s + Projection: #2 AS #3 + Projection: column1 AS #2 + Values: (Int32(1)) + SubqueryAlias: t + Projection: #0 AS #1 + Projection: column1 AS #0 + Values: (Int32(1)), (Int32(2)), (Int32(3)), (Int32(4)) + + logical_plan after replace_distinct_aggregate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_predicate_subquery: + SAME TEXT AS ABOVE + + logical_plan after scalar_subquery_to_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_join: + SAME TEXT AS ABOVE + + logical_plan after extract_equijoin_predicate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_duplicated_expr: + SAME TEXT AS ABOVE + + logical_plan after eliminate_filter: + SAME TEXT AS ABOVE + + logical_plan after eliminate_cross_join: + SAME TEXT AS ABOVE + + logical_plan after eliminate_limit: + SAME TEXT AS ABOVE + + logical_plan after propagate_empty_relation: + SAME TEXT AS ABOVE + + logical_plan after filter_null_join_keys: + SAME TEXT AS ABOVE + + logical_plan after eliminate_outer_join: + SAME TEXT AS ABOVE + + logical_plan after push_down_limit: + SAME TEXT AS ABOVE + + logical_plan after push_down_filter: + SAME TEXT AS ABOVE + + logical_plan after single_distinct_aggregation_to_group_by: + SAME TEXT AS ABOVE + + logical_plan after eliminate_group_by_constant: + SAME TEXT AS ABOVE + + logical_plan after common_sub_expression_eliminate: + SAME TEXT AS ABOVE + + logical_plan after extract_leaf_expressions: + SAME TEXT AS ABOVE + + logical_plan after optimize_projections: + Sort: #6 ASC NULLS FIRST + Projection: t.#1 AS #6, count(Int64(1)) PARTITION BY [CASE WHEN #4 = Int32(1) THEN t.#1 % CASE WHEN Int32(2) = Int32(0) THEN raise_error(Utf8("Remainder by zero")) ELSE Int32(2) END WHEN Boolean(true) THEN t.#1 END] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING AS #5 AS n AS #7 + WindowAggr: windowExpr=[[count(Int64(1)) PARTITION BY [CASE WHEN () = Int32(1) THEN t.#1 % Int32(2) ELSE t.#1 END] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING AS count(Int64(1)) PARTITION BY [CASE WHEN #4 = Int32(1) THEN t.#1 % CASE WHEN Int32(2) = Int32(0) THEN raise_error(Utf8("Remainder by zero")) ELSE Int32(2) END WHEN Boolean(true) THEN t.#1 END] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING]] + Subquery: + Projection: min(s.#3) AS #4 + Aggregate: groupBy=[[]], aggr=[[min(s.#3)]] + SubqueryAlias: s + Projection: column1 AS #3 + Values: (Int32(1)) + SubqueryAlias: t + Projection: column1 AS #1 + Values: (Int32(1)), (Int32(2)), (Int32(3)), (Int32(4)) + + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_projection: + SAME TEXT AS ABOVE + + logical_plan after rewrite_set_comparison: + SAME TEXT AS ABOVE + + logical_plan after optimize_unions: + SAME TEXT AS ABOVE + + logical_plan after unions_to_filter: + SAME TEXT AS ABOVE + + logical_plan after simplify_expressions: + SAME TEXT AS ABOVE + + logical_plan after replace_distinct_aggregate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_predicate_subquery: + SAME TEXT AS ABOVE + + logical_plan after scalar_subquery_to_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_join: + SAME TEXT AS ABOVE + + logical_plan after extract_equijoin_predicate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_duplicated_expr: + SAME TEXT AS ABOVE + + logical_plan after eliminate_filter: + SAME TEXT AS ABOVE + + logical_plan after eliminate_cross_join: + SAME TEXT AS ABOVE + + logical_plan after eliminate_limit: + SAME TEXT AS ABOVE + + logical_plan after propagate_empty_relation: + SAME TEXT AS ABOVE + + logical_plan after filter_null_join_keys: + SAME TEXT AS ABOVE + + logical_plan after eliminate_outer_join: + SAME TEXT AS ABOVE + + logical_plan after push_down_limit: + SAME TEXT AS ABOVE + + logical_plan after push_down_filter: + SAME TEXT AS ABOVE + + logical_plan after single_distinct_aggregation_to_group_by: + SAME TEXT AS ABOVE + + logical_plan after eliminate_group_by_constant: + SAME TEXT AS ABOVE + + logical_plan after common_sub_expression_eliminate: + SAME TEXT AS ABOVE + + logical_plan after extract_leaf_expressions: + SAME TEXT AS ABOVE + + logical_plan after optimize_projections: + SAME TEXT AS ABOVE + + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE + + logical_plan: + Sort: #6 ASC NULLS FIRST + Projection: t.#1 AS #6, count(Int64(1)) PARTITION BY [CASE WHEN #4 = Int32(1) THEN t.#1 % CASE WHEN Int32(2) = Int32(0) THEN raise_error(Utf8("Remainder by zero")) ELSE Int32(2) END WHEN Boolean(true) THEN t.#1 END] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING AS #5 AS n AS #7 + WindowAggr: windowExpr=[[count(Int64(1)) PARTITION BY [CASE WHEN () = Int32(1) THEN t.#1 % Int32(2) ELSE t.#1 END] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING AS count(Int64(1)) PARTITION BY [CASE WHEN #4 = Int32(1) THEN t.#1 % CASE WHEN Int32(2) = Int32(0) THEN raise_error(Utf8("Remainder by zero")) ELSE Int32(2) END WHEN Boolean(true) THEN t.#1 END] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING]] + Subquery: + Projection: min(s.#3) AS #4 + Aggregate: groupBy=[[]], aggr=[[min(s.#3)]] + SubqueryAlias: s + Projection: column1 AS #3 + Values: (Int32(1)) + SubqueryAlias: t + Projection: column1 AS #1 + Values: (Int32(1)), (Int32(2)), (Int32(3)), (Int32(4)) + + initial_physical_plan: + ScalarSubqueryExec: subqueries=1 + SortExec: expr=[#6@0 ASC], preserve_partitioning=[false] + ProjectionExec: expr=[#1@0 as #6, count(Int64(1)) PARTITION BY [CASE WHEN #4 = Int32(1) THEN t.#1 % CASE WHEN Int32(2) = Int32(0) THEN raise_error(Utf8("Remainder by zero")) ELSE Int32(2) END WHEN Boolean(true) THEN t.#1 END] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@1 as #7] + WindowAggExec: wdw=[count(Int64(1)) PARTITION BY [CASE WHEN #4 = Int32(1) THEN t.#1 % CASE WHEN Int32(2) = Int32(0) THEN raise_error(Utf8("Remainder by zero")) ELSE Int32(2) END WHEN Boolean(true) THEN t.#1 END] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING: Ok(Field { name: "count(Int64(1)) PARTITION BY [CASE WHEN #4 = Int32(1) THEN t.#1 % CASE WHEN Int32(2) = Int32(0) THEN raise_error(Utf8(/"Remainder by zero/")) ELSE Int32(2) END WHEN Boolean(true) THEN t.#1 END] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING", data_type: Int64 }), frame: WindowFrame { units: Rows, start_bound: Preceding(UInt64(NULL)), end_bound: Following(UInt64(NULL)), is_causal: false }] + ProjectionExec: expr=[column1@0 as #1] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#3)@0 as #4] + AggregateExec: mode=Final, gby=[], aggr=[min(s.#3)] + AggregateExec: mode=Partial, gby=[], aggr=[min(s.#3)] + ProjectionExec: expr=[column1@0 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + + + initial_physical_plan_with_stats: + ScalarSubqueryExec: subqueries=1, statistics=[Rows=Exact(4), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]:)]] + SortExec: expr=[#6@0 ASC], preserve_partitioning=[false], statistics=[Rows=Exact(4), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]:)]] + ProjectionExec: expr=[#1@0 as #6, count(Int64(1)) PARTITION BY [CASE WHEN #4 = Int32(1) THEN t.#1 % CASE WHEN Int32(2) = Int32(0) THEN raise_error(Utf8("Remainder by zero")) ELSE Int32(2) END WHEN Boolean(true) THEN t.#1 END] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@1 as #7], statistics=[Rows=Exact(4), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]:)]] + WindowAggExec: wdw=[count(Int64(1)) PARTITION BY [CASE WHEN #4 = Int32(1) THEN t.#1 % CASE WHEN Int32(2) = Int32(0) THEN raise_error(Utf8("Remainder by zero")) ELSE Int32(2) END WHEN Boolean(true) THEN t.#1 END] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING: Ok(Field { name: "count(Int64(1)) PARTITION BY [CASE WHEN #4 = Int32(1) THEN t.#1 % CASE WHEN Int32(2) = Int32(0) THEN raise_error(Utf8(/"Remainder by zero/")) ELSE Int32(2) END WHEN Boolean(true) THEN t.#1 END] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING", data_type: Int64 }), frame: WindowFrame { units: Rows, start_bound: Preceding(UInt64(NULL)), end_bound: Following(UInt64(NULL)), is_causal: false }], statistics=[Rows=Exact(4), Bytes=Absent, [(Col[0]: Null=Exact(0)),(Col[1]:)]] + ProjectionExec: expr=[column1@0 as #1], statistics=[Rows=Exact(4), Bytes=Exact(), [(Col[0]: Null=Exact(0))]] + DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(4), Bytes=Exact(), [(Col[0]: Null=Exact(0))]] + ProjectionExec: expr=[min(s.#3)@0 as #4], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]:)]] + AggregateExec: mode=Final, gby=[], aggr=[min(s.#3)], statistics=[Rows=Exact(1), Bytes=Absent, [(Col[0]:)]] + AggregateExec: mode=Partial, gby=[], aggr=[min(s.#3)], statistics=[Rows=Exact(0), Bytes=Inexact(), [(Col[0]:)]] + ProjectionExec: expr=[column1@0 as #3], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]: Null=Exact(0))]] + DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]: Null=Exact(0))]] + + + initial_physical_plan_with_schema: + ScalarSubqueryExec: subqueries=1, schema=[#6:Int32;N, #7:Int64] + SortExec: expr=[#6@0 ASC], preserve_partitioning=[false], schema=[#6:Int32;N, #7:Int64] + ProjectionExec: expr=[#1@0 as #6, count(Int64(1)) PARTITION BY [CASE WHEN #4 = Int32(1) THEN t.#1 % CASE WHEN Int32(2) = Int32(0) THEN raise_error(Utf8("Remainder by zero")) ELSE Int32(2) END WHEN Boolean(true) THEN t.#1 END] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@1 as #7], schema=[#6:Int32;N, #7:Int64] + WindowAggExec: wdw=[count(Int64(1)) PARTITION BY [CASE WHEN #4 = Int32(1) THEN t.#1 % CASE WHEN Int32(2) = Int32(0) THEN raise_error(Utf8("Remainder by zero")) ELSE Int32(2) END WHEN Boolean(true) THEN t.#1 END] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING: Ok(Field { name: "count(Int64(1)) PARTITION BY [CASE WHEN #4 = Int32(1) THEN t.#1 % CASE WHEN Int32(2) = Int32(0) THEN raise_error(Utf8(/"Remainder by zero/")) ELSE Int32(2) END WHEN Boolean(true) THEN t.#1 END] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING", data_type: Int64 }), frame: WindowFrame { units: Rows, start_bound: Preceding(UInt64(NULL)), end_bound: Following(UInt64(NULL)), is_causal: false }], schema=[#1:Int32;N, count(Int64(1)) PARTITION BY [CASE WHEN #4 = Int32(1) THEN t.#1 % CASE WHEN Int32(2) = Int32(0) THEN raise_error(Utf8("Remainder by zero")) ELSE Int32(2) END WHEN Boolean(true) THEN t.#1 END] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING:Int64] + ProjectionExec: expr=[column1@0 as #1], schema=[#1:Int32;N] + DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N] + ProjectionExec: expr=[min(s.#3)@0 as #4], schema=[#4:Int32;N] + AggregateExec: mode=Final, gby=[], aggr=[min(s.#3)], schema=[min(s.#3):Int32;N] + AggregateExec: mode=Partial, gby=[], aggr=[min(s.#3)], schema=[min(s.#3)[value]:Int32;N] + ProjectionExec: expr=[column1@0 as #3], schema=[#3:Int32;N] + DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N] + + + physical_plan after OutputRequirements: + OutputRequirementExec: order_by=[], dist_by=Unspecified + ScalarSubqueryExec: subqueries=1 + SortExec: expr=[#6@0 ASC], preserve_partitioning=[false] + ProjectionExec: expr=[#1@0 as #6, count(Int64(1)) PARTITION BY [CASE WHEN #4 = Int32(1) THEN t.#1 % CASE WHEN Int32(2) = Int32(0) THEN raise_error(Utf8("Remainder by zero")) ELSE Int32(2) END WHEN Boolean(true) THEN t.#1 END] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@1 as #7] + WindowAggExec: wdw=[count(Int64(1)) PARTITION BY [CASE WHEN #4 = Int32(1) THEN t.#1 % CASE WHEN Int32(2) = Int32(0) THEN raise_error(Utf8("Remainder by zero")) ELSE Int32(2) END WHEN Boolean(true) THEN t.#1 END] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING: Ok(Field { name: "count(Int64(1)) PARTITION BY [CASE WHEN #4 = Int32(1) THEN t.#1 % CASE WHEN Int32(2) = Int32(0) THEN raise_error(Utf8(/"Remainder by zero/")) ELSE Int32(2) END WHEN Boolean(true) THEN t.#1 END] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING", data_type: Int64 }), frame: WindowFrame { units: Rows, start_bound: Preceding(UInt64(NULL)), end_bound: Following(UInt64(NULL)), is_causal: false }] + ProjectionExec: expr=[column1@0 as #1] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#3)@0 as #4] + AggregateExec: mode=Final, gby=[], aggr=[min(s.#3)] + AggregateExec: mode=Partial, gby=[], aggr=[min(s.#3)] + ProjectionExec: expr=[column1@0 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after aggregate_statistics: + SAME TEXT AS ABOVE + + physical_plan after join_selection: + SAME TEXT AS ABOVE + + physical_plan after LimitedDistinctAggregation: + SAME TEXT AS ABOVE + + physical_plan after FilterPushdown: + SAME TEXT AS ABOVE + + physical_plan after EnforceDistribution: + SAME TEXT AS ABOVE + + physical_plan after CombinePartialFinalAggregate: + OutputRequirementExec: order_by=[], dist_by=Unspecified + ScalarSubqueryExec: subqueries=1 + SortExec: expr=[#6@0 ASC], preserve_partitioning=[false] + ProjectionExec: expr=[#1@0 as #6, count(Int64(1)) PARTITION BY [CASE WHEN #4 = Int32(1) THEN t.#1 % CASE WHEN Int32(2) = Int32(0) THEN raise_error(Utf8("Remainder by zero")) ELSE Int32(2) END WHEN Boolean(true) THEN t.#1 END] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@1 as #7] + WindowAggExec: wdw=[count(Int64(1)) PARTITION BY [CASE WHEN #4 = Int32(1) THEN t.#1 % CASE WHEN Int32(2) = Int32(0) THEN raise_error(Utf8("Remainder by zero")) ELSE Int32(2) END WHEN Boolean(true) THEN t.#1 END] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING: Ok(Field { name: "count(Int64(1)) PARTITION BY [CASE WHEN #4 = Int32(1) THEN t.#1 % CASE WHEN Int32(2) = Int32(0) THEN raise_error(Utf8(/"Remainder by zero/")) ELSE Int32(2) END WHEN Boolean(true) THEN t.#1 END] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING", data_type: Int64 }), frame: WindowFrame { units: Rows, start_bound: Preceding(UInt64(NULL)), end_bound: Following(UInt64(NULL)), is_causal: false }] + ProjectionExec: expr=[column1@0 as #1] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#3)@0 as #4] + AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)] + ProjectionExec: expr=[column1@0 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after EnforceSorting: + OutputRequirementExec: order_by=[], dist_by=Unspecified + ScalarSubqueryExec: subqueries=1 + SortExec: expr=[#6@0 ASC], preserve_partitioning=[false] + ProjectionExec: expr=[#1@0 as #6, count(Int64(1)) PARTITION BY [CASE WHEN #4 = Int32(1) THEN t.#1 % CASE WHEN Int32(2) = Int32(0) THEN raise_error(Utf8("Remainder by zero")) ELSE Int32(2) END WHEN Boolean(true) THEN t.#1 END] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@1 as #7] + WindowAggExec: wdw=[count(Int64(1)) PARTITION BY [CASE WHEN #4 = Int32(1) THEN t.#1 % CASE WHEN Int32(2) = Int32(0) THEN raise_error(Utf8("Remainder by zero")) ELSE Int32(2) END WHEN Boolean(true) THEN t.#1 END] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING: Ok(Field { name: "count(Int64(1)) PARTITION BY [CASE WHEN #4 = Int32(1) THEN t.#1 % CASE WHEN Int32(2) = Int32(0) THEN raise_error(Utf8(/"Remainder by zero/")) ELSE Int32(2) END WHEN Boolean(true) THEN t.#1 END] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING", data_type: Int64 }), frame: WindowFrame { units: Rows, start_bound: Preceding(UInt64(NULL)), end_bound: Following(UInt64(NULL)), is_causal: false }] + SortExec: expr=[CASE WHEN scalar_subquery() = 1 THEN #1@0 % 2 ELSE #1@0 END ASC NULLS LAST], preserve_partitioning=[false] + ProjectionExec: expr=[column1@0 as #1] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#3)@0 as #4] + AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)] + ProjectionExec: expr=[column1@0 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after OptimizeAggregateOrder: + SAME TEXT AS ABOVE + + physical_plan after WindowTopN: + SAME TEXT AS ABOVE + + physical_plan after ProjectionPushdown: + SAME TEXT AS ABOVE + + physical_plan after OutputRequirements: + ScalarSubqueryExec: subqueries=1 + SortExec: expr=[#6@0 ASC], preserve_partitioning=[false] + ProjectionExec: expr=[#1@0 as #6, count(Int64(1)) PARTITION BY [CASE WHEN #4 = Int32(1) THEN t.#1 % CASE WHEN Int32(2) = Int32(0) THEN raise_error(Utf8("Remainder by zero")) ELSE Int32(2) END WHEN Boolean(true) THEN t.#1 END] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@1 as #7] + WindowAggExec: wdw=[count(Int64(1)) PARTITION BY [CASE WHEN #4 = Int32(1) THEN t.#1 % CASE WHEN Int32(2) = Int32(0) THEN raise_error(Utf8("Remainder by zero")) ELSE Int32(2) END WHEN Boolean(true) THEN t.#1 END] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING: Ok(Field { name: "count(Int64(1)) PARTITION BY [CASE WHEN #4 = Int32(1) THEN t.#1 % CASE WHEN Int32(2) = Int32(0) THEN raise_error(Utf8(/"Remainder by zero/")) ELSE Int32(2) END WHEN Boolean(true) THEN t.#1 END] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING", data_type: Int64 }), frame: WindowFrame { units: Rows, start_bound: Preceding(UInt64(NULL)), end_bound: Following(UInt64(NULL)), is_causal: false }] + SortExec: expr=[CASE WHEN scalar_subquery() = 1 THEN #1@0 % 2 ELSE #1@0 END ASC NULLS LAST], preserve_partitioning=[false] + ProjectionExec: expr=[column1@0 as #1] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#3)@0 as #4] + AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)] + ProjectionExec: expr=[column1@0 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + + + physical_plan after LimitAggregation: + SAME TEXT AS ABOVE + + physical_plan after LimitPushPastWindows: + SAME TEXT AS ABOVE + + physical_plan after HashJoinBuffering: + SAME TEXT AS ABOVE + + physical_plan after LimitPushdown: + SAME TEXT AS ABOVE + + physical_plan after TopKRepartition: + SAME TEXT AS ABOVE + + physical_plan after ProjectionPushdown: + SAME TEXT AS ABOVE + + physical_plan after PushdownSort: + SAME TEXT AS ABOVE + + physical_plan after EnsureCooperative: + SAME TEXT AS ABOVE + + physical_plan after FilterPushdown(Post): + SAME TEXT AS ABOVE + + physical_plan after RewriteExplicitRepartition: + SAME TEXT AS ABOVE + + physical_plan after RewriteCollectLeftHashJoin: + SAME TEXT AS ABOVE + + physical_plan after EnforceBarrierPartitioning: + SAME TEXT AS ABOVE + + physical_plan after SanityCheckPlan: + SAME TEXT AS ABOVE + + physical_plan: + ProjectionExec: expr=[#6@0 as v, #7@1 as n] + ScalarSubqueryExec: subqueries=1 + SortExec: expr=[#6@0 ASC], preserve_partitioning=[false] + ProjectionExec: expr=[#1@0 as #6, count(Int64(1)) PARTITION BY [CASE WHEN #4 = Int32(1) THEN t.#1 % CASE WHEN Int32(2) = Int32(0) THEN raise_error(Utf8("Remainder by zero")) ELSE Int32(2) END WHEN Boolean(true) THEN t.#1 END] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@1 as #7] + WindowAggExec: wdw=[count(Int64(1)) PARTITION BY [CASE WHEN #4 = Int32(1) THEN t.#1 % CASE WHEN Int32(2) = Int32(0) THEN raise_error(Utf8("Remainder by zero")) ELSE Int32(2) END WHEN Boolean(true) THEN t.#1 END] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING: Ok(Field { name: "count(Int64(1)) PARTITION BY [CASE WHEN #4 = Int32(1) THEN t.#1 % CASE WHEN Int32(2) = Int32(0) THEN raise_error(Utf8(/"Remainder by zero/")) ELSE Int32(2) END WHEN Boolean(true) THEN t.#1 END] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING", data_type: Int64 }), frame: WindowFrame { units: Rows, start_bound: Preceding(UInt64(NULL)), end_bound: Following(UInt64(NULL)), is_causal: false }] + SortExec: expr=[CASE WHEN scalar_subquery() = 1 THEN #1@0 % 2 ELSE #1@0 END ASC NULLS LAST], preserve_partitioning=[false] + ProjectionExec: expr=[column1@0 as #1] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[min(s.#3)@0 as #4] + AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)] + ProjectionExec: expr=[column1@0 as #3] + DataSourceExec: partitions=1, partition_sizes=[1] + + == Physical Plan == ProjectionExec: expr=[#6@0 as v, #7@1 as n] ScalarSubqueryExec: subqueries=1 diff --git a/python/pysail/tests/spark/execution/__snapshots__/test_shuffle_storage.plan.yaml b/python/pysail/tests/spark/execution/__snapshots__/test_shuffle_storage.plan.yaml index 697670fe5..db354c5f3 100644 --- a/python/pysail/tests/spark/execution/__snapshots__/test_shuffle_storage.plan.yaml +++ b/python/pysail/tests/spark/execution/__snapshots__/test_shuffle_storage.plan.yaml @@ -3,6 +3,566 @@ name: "test_query_execution_with_storage_shuffle" data: |- + == Codegen == + Whole-stage codegen is not supported; showing physical plan instead. + + == Plan Steps == + initial_logical_plan: + Sort: #13 ASC NULLS FIRST + Projection: #12 AS #13, count(Int64(1)) AS count AS #14, sum(#10) AS s1 AS #15, sum(#11) AS s2 AS #16 + Aggregate: groupBy=[[#12]], aggr=[[count(Int64(1)), sum(#10), sum(#11)]] + Projection: #8 AS #9, #2 AS #10, #5 AS #11, #8 % CASE WHEN Int32(4) = Int32(0) THEN raise_error(Utf8("Remainder by zero")) ELSE Int32(4) END AS #12 + Projection: #8, #2, #5 + Projection: coalesce(#1, #4) AS #8, #1 AS #6, #2, #4 AS #7, #5 + Inner Join: #1 = #4 + ExplicitRepartition: kind=hash, n=Some(8), expr=[#1] + Projection: #0 AS k AS #1, #0 * Int32(2) AS v1 AS #2 + Range: start=0, end=64, step=1, num_partitions=4 + ExplicitRepartition: kind=hash, n=Some(8), expr=[#4] + Projection: #3 AS k AS #4, #3 + Int32(1) AS v2 AS #5 + Range: start=0, end=64, step=1, num_partitions=4 + + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE + + logical_plan after resolve_grouping_function: + SAME TEXT AS ABOVE + + logical_plan after type_coercion: + Sort: #13 ASC NULLS FIRST + Projection: #12 AS #13, count(Int64(1)) AS count AS #14, sum(#10) AS s1 AS #15, sum(#11) AS s2 AS #16 + Aggregate: groupBy=[[#12]], aggr=[[count(Int64(1)), sum(#10), sum(#11)]] + Projection: #8 AS #9, #2 AS #10, #5 AS #11, #8 % CAST(CASE WHEN Int32(4) = Int32(0) THEN CAST(raise_error(Utf8("Remainder by zero")) AS Int32) ELSE Int32(4) END AS Int64) AS #12 + Projection: #8, #2, #5 + Projection: coalesce(#1, #4) AS #8, #1 AS #6, #2, #4 AS #7, #5 + Inner Join: #1 = #4 + ExplicitRepartition: kind=hash, n=Some(8), expr=[#1] + Projection: #0 AS k AS #1, #0 * CAST(Int32(2) AS Int64) AS v1 AS #2 + Range: start=0, end=64, step=1, num_partitions=4 + ExplicitRepartition: kind=hash, n=Some(8), expr=[#4] + Projection: #3 AS k AS #4, #3 + CAST(Int32(1) AS Int64) AS v2 AS #5 + Range: start=0, end=64, step=1, num_partitions=4 + + analyzed_logical_plan: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_projection: + SAME TEXT AS ABOVE + + logical_plan after rewrite_set_comparison: + SAME TEXT AS ABOVE + + logical_plan after optimize_unions: + SAME TEXT AS ABOVE + + logical_plan after unions_to_filter: + SAME TEXT AS ABOVE + + logical_plan after simplify_expressions: + Sort: #13 ASC NULLS FIRST + Projection: #12 AS #13, count(Int64(1)) AS count AS #14, sum(#10) AS s1 AS #15, sum(#11) AS s2 AS #16 + Aggregate: groupBy=[[#12]], aggr=[[count(Int64(1)), sum(#10), sum(#11)]] + Projection: #8 AS #9, #2 AS #10, #5 AS #11, #8 % Int64(4) AS #12 + Projection: #8, #2, #5 + Projection: #1 AS #8, #1 AS #6, #2, #4 AS #7, #5 + Inner Join: #1 = #4 + ExplicitRepartition: kind=hash, n=Some(8), expr=[#1] + Projection: #0 AS k AS #1, #0 * Int64(2) AS v1 AS #2 + Range: start=0, end=64, step=1, num_partitions=4 + ExplicitRepartition: kind=hash, n=Some(8), expr=[#4] + Projection: #3 AS k AS #4, #3 + Int64(1) AS v2 AS #5 + Range: start=0, end=64, step=1, num_partitions=4 + + logical_plan after replace_distinct_aggregate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_predicate_subquery: + SAME TEXT AS ABOVE + + logical_plan after scalar_subquery_to_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_join: + SAME TEXT AS ABOVE + + logical_plan after extract_equijoin_predicate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_duplicated_expr: + SAME TEXT AS ABOVE + + logical_plan after eliminate_filter: + SAME TEXT AS ABOVE + + logical_plan after eliminate_cross_join: + SAME TEXT AS ABOVE + + logical_plan after eliminate_limit: + SAME TEXT AS ABOVE + + logical_plan after propagate_empty_relation: + SAME TEXT AS ABOVE + + logical_plan after filter_null_join_keys: + SAME TEXT AS ABOVE + + logical_plan after eliminate_outer_join: + SAME TEXT AS ABOVE + + logical_plan after push_down_limit: + SAME TEXT AS ABOVE + + logical_plan after push_down_filter: + SAME TEXT AS ABOVE + + logical_plan after single_distinct_aggregation_to_group_by: + SAME TEXT AS ABOVE + + logical_plan after eliminate_group_by_constant: + SAME TEXT AS ABOVE + + logical_plan after common_sub_expression_eliminate: + SAME TEXT AS ABOVE + + logical_plan after extract_leaf_expressions: + SAME TEXT AS ABOVE + + logical_plan after optimize_projections: + Sort: #13 ASC NULLS FIRST + Projection: #12 AS #13, count(Int64(1)) AS count AS #14, sum(#10) AS s1 AS #15, sum(#11) AS s2 AS #16 + Aggregate: groupBy=[[#12]], aggr=[[count(Int64(1)), sum(#10), sum(#11)]] + Projection: #2 AS #10, #5 AS #11, #8 % Int64(4) AS #12 + Projection: #1 AS #8, #2, #5 + Inner Join: #1 = #4 + ExplicitRepartition: kind=hash, n=Some(8), expr=[#1] + Projection: #0 AS k AS #1, #0 * Int64(2) AS v1 AS #2 + Range: start=0, end=64, step=1, num_partitions=4 + ExplicitRepartition: kind=hash, n=Some(8), expr=[#4] + Projection: #3 AS k AS #4, #3 + Int64(1) AS v2 AS #5 + Range: start=0, end=64, step=1, num_partitions=4 + + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_projection: + SAME TEXT AS ABOVE + + logical_plan after rewrite_set_comparison: + SAME TEXT AS ABOVE + + logical_plan after optimize_unions: + SAME TEXT AS ABOVE + + logical_plan after unions_to_filter: + SAME TEXT AS ABOVE + + logical_plan after simplify_expressions: + SAME TEXT AS ABOVE + + logical_plan after replace_distinct_aggregate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_predicate_subquery: + SAME TEXT AS ABOVE + + logical_plan after scalar_subquery_to_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_join: + SAME TEXT AS ABOVE + + logical_plan after extract_equijoin_predicate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_duplicated_expr: + SAME TEXT AS ABOVE + + logical_plan after eliminate_filter: + SAME TEXT AS ABOVE + + logical_plan after eliminate_cross_join: + SAME TEXT AS ABOVE + + logical_plan after eliminate_limit: + SAME TEXT AS ABOVE + + logical_plan after propagate_empty_relation: + SAME TEXT AS ABOVE + + logical_plan after filter_null_join_keys: + SAME TEXT AS ABOVE + + logical_plan after eliminate_outer_join: + SAME TEXT AS ABOVE + + logical_plan after push_down_limit: + SAME TEXT AS ABOVE + + logical_plan after push_down_filter: + SAME TEXT AS ABOVE + + logical_plan after single_distinct_aggregation_to_group_by: + SAME TEXT AS ABOVE + + logical_plan after eliminate_group_by_constant: + SAME TEXT AS ABOVE + + logical_plan after common_sub_expression_eliminate: + SAME TEXT AS ABOVE + + logical_plan after extract_leaf_expressions: + SAME TEXT AS ABOVE + + logical_plan after optimize_projections: + Sort: #13 ASC NULLS FIRST + Projection: #12 AS #13, count(Int64(1)) AS count AS #14, sum(#10) AS s1 AS #15, sum(#11) AS s2 AS #16 + Aggregate: groupBy=[[#12]], aggr=[[count(Int64(1)), sum(#10), sum(#11)]] + Projection: #2 AS #10, #5 AS #11, #1 % Int64(4) AS #12 + Inner Join: #1 = #4 + ExplicitRepartition: kind=hash, n=Some(8), expr=[#1] + Projection: #0 AS k AS #1, #0 * Int64(2) AS v1 AS #2 + Range: start=0, end=64, step=1, num_partitions=4 + ExplicitRepartition: kind=hash, n=Some(8), expr=[#4] + Projection: #3 AS k AS #4, #3 + Int64(1) AS v2 AS #5 + Range: start=0, end=64, step=1, num_partitions=4 + + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_projection: + SAME TEXT AS ABOVE + + logical_plan after rewrite_set_comparison: + SAME TEXT AS ABOVE + + logical_plan after optimize_unions: + SAME TEXT AS ABOVE + + logical_plan after unions_to_filter: + SAME TEXT AS ABOVE + + logical_plan after simplify_expressions: + SAME TEXT AS ABOVE + + logical_plan after replace_distinct_aggregate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_predicate_subquery: + SAME TEXT AS ABOVE + + logical_plan after scalar_subquery_to_join: + SAME TEXT AS ABOVE + + logical_plan after decorrelate_lateral_join: + SAME TEXT AS ABOVE + + logical_plan after extract_equijoin_predicate: + SAME TEXT AS ABOVE + + logical_plan after eliminate_duplicated_expr: + SAME TEXT AS ABOVE + + logical_plan after eliminate_filter: + SAME TEXT AS ABOVE + + logical_plan after eliminate_cross_join: + SAME TEXT AS ABOVE + + logical_plan after eliminate_limit: + SAME TEXT AS ABOVE + + logical_plan after propagate_empty_relation: + SAME TEXT AS ABOVE + + logical_plan after filter_null_join_keys: + SAME TEXT AS ABOVE + + logical_plan after eliminate_outer_join: + SAME TEXT AS ABOVE + + logical_plan after push_down_limit: + SAME TEXT AS ABOVE + + logical_plan after push_down_filter: + SAME TEXT AS ABOVE + + logical_plan after single_distinct_aggregation_to_group_by: + SAME TEXT AS ABOVE + + logical_plan after eliminate_group_by_constant: + SAME TEXT AS ABOVE + + logical_plan after common_sub_expression_eliminate: + SAME TEXT AS ABOVE + + logical_plan after extract_leaf_expressions: + SAME TEXT AS ABOVE + + logical_plan after optimize_projections: + SAME TEXT AS ABOVE + + logical_plan after resolve_lambda_variables: + SAME TEXT AS ABOVE + + logical_plan: + Sort: #13 ASC NULLS FIRST + Projection: #12 AS #13, count(Int64(1)) AS count AS #14, sum(#10) AS s1 AS #15, sum(#11) AS s2 AS #16 + Aggregate: groupBy=[[#12]], aggr=[[count(Int64(1)), sum(#10), sum(#11)]] + Projection: #2 AS #10, #5 AS #11, #1 % Int64(4) AS #12 + Inner Join: #1 = #4 + ExplicitRepartition: kind=hash, n=Some(8), expr=[#1] + Projection: #0 AS k AS #1, #0 * Int64(2) AS v1 AS #2 + Range: start=0, end=64, step=1, num_partitions=4 + ExplicitRepartition: kind=hash, n=Some(8), expr=[#4] + Projection: #3 AS k AS #4, #3 + Int64(1) AS v2 AS #5 + Range: start=0, end=64, step=1, num_partitions=4 + + initial_physical_plan: + SortExec: expr=[#13@0 ASC], preserve_partitioning=[false] + ProjectionExec: expr=[#12@0 as #13, count(Int64(1))@1 as #14, sum(#10)@2 as #15, sum(#11)@3 as #16] + AggregateExec: mode=FinalPartitioned, gby=[#12@0 as #12], aggr=[count(Int64(1)), sum(#10), sum(#11)] + AggregateExec: mode=Partial, gby=[#12@2 as #12], aggr=[count(Int64(1)), sum(#10), sum(#11)] + ProjectionExec: expr=[#2@1 as #10, #5@3 as #11, #1@0 % 4 as #12] + HashJoinExec: mode=Auto, join_type=Inner, on=[(#1@0, #4@0)] + ExplicitRepartitionExec + ProjectionExec: expr=[#0@0 as #1, #0@0 * 2 as #2] + RangeExec + ExplicitRepartitionExec + ProjectionExec: expr=[#3@0 as #4, #3@0 + 1 as #5] + RangeExec + + + initial_physical_plan_with_stats: + SortExec: expr=[#13@0 ASC], preserve_partitioning=[false], statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:),(Col[3]:)]] + ProjectionExec: expr=[#12@0 as #13, count(Int64(1))@1 as #14, sum(#10)@2 as #15, sum(#11)@3 as #16], statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:),(Col[3]:)]] + AggregateExec: mode=FinalPartitioned, gby=[#12@0 as #12], aggr=[count(Int64(1)), sum(#10), sum(#11)], statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:),(Col[3]:)]] + AggregateExec: mode=Partial, gby=[#12@2 as #12], aggr=[count(Int64(1)), sum(#10), sum(#11)], statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:),(Col[3]:)]] + ProjectionExec: expr=[#2@1 as #10, #5@3 as #11, #1@0 % 4 as #12], statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:)]] + HashJoinExec: mode=Auto, join_type=Inner, on=[(#1@0, #4@0)], statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:),(Col[3]:)]] + ExplicitRepartitionExec, statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:),(Col[1]:)]] + ProjectionExec: expr=[#0@0 as #1, #0@0 * 2 as #2], statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:),(Col[1]:)]] + RangeExec, statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:)]] + ExplicitRepartitionExec, statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:),(Col[1]:)]] + ProjectionExec: expr=[#3@0 as #4, #3@0 + 1 as #5], statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:),(Col[1]:)]] + RangeExec, statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:)]] + + + initial_physical_plan_with_schema: + SortExec: expr=[#13@0 ASC], preserve_partitioning=[false], schema=[#13:Int64, #14:Int64, #15:Int64;N, #16:Int64;N] + ProjectionExec: expr=[#12@0 as #13, count(Int64(1))@1 as #14, sum(#10)@2 as #15, sum(#11)@3 as #16], schema=[#13:Int64, #14:Int64, #15:Int64;N, #16:Int64;N] + AggregateExec: mode=FinalPartitioned, gby=[#12@0 as #12], aggr=[count(Int64(1)), sum(#10), sum(#11)], schema=[#12:Int64, count(Int64(1)):Int64, sum(#10):Int64;N, sum(#11):Int64;N] + AggregateExec: mode=Partial, gby=[#12@2 as #12], aggr=[count(Int64(1)), sum(#10), sum(#11)], schema=[#12:Int64, count(Int64(1))[count]:Int64, sum(#10)[sum]:Int64;N, sum(#11)[sum]:Int64;N] + ProjectionExec: expr=[#2@1 as #10, #5@3 as #11, #1@0 % 4 as #12], schema=[#10:Int64, #11:Int64, #12:Int64] + HashJoinExec: mode=Auto, join_type=Inner, on=[(#1@0, #4@0)], schema=[#1:Int64, #2:Int64, #4:Int64, #5:Int64] + ExplicitRepartitionExec, schema=[#1:Int64, #2:Int64] + ProjectionExec: expr=[#0@0 as #1, #0@0 * 2 as #2], schema=[#1:Int64, #2:Int64] + RangeExec, schema=[#0:Int64] + ExplicitRepartitionExec, schema=[#4:Int64, #5:Int64] + ProjectionExec: expr=[#3@0 as #4, #3@0 + 1 as #5], schema=[#4:Int64, #5:Int64] + RangeExec, schema=[#3:Int64] + + + physical_plan after OutputRequirements: + OutputRequirementExec: order_by=[(#13@0, asc)], dist_by=SinglePartition + SortExec: expr=[#13@0 ASC], preserve_partitioning=[false] + ProjectionExec: expr=[#12@0 as #13, count(Int64(1))@1 as #14, sum(#10)@2 as #15, sum(#11)@3 as #16] + AggregateExec: mode=FinalPartitioned, gby=[#12@0 as #12], aggr=[count(Int64(1)), sum(#10), sum(#11)] + AggregateExec: mode=Partial, gby=[#12@2 as #12], aggr=[count(Int64(1)), sum(#10), sum(#11)] + ProjectionExec: expr=[#2@1 as #10, #5@3 as #11, #1@0 % 4 as #12] + HashJoinExec: mode=Auto, join_type=Inner, on=[(#1@0, #4@0)] + ExplicitRepartitionExec + ProjectionExec: expr=[#0@0 as #1, #0@0 * 2 as #2] + RangeExec + ExplicitRepartitionExec + ProjectionExec: expr=[#3@0 as #4, #3@0 + 1 as #5] + RangeExec + + + physical_plan after aggregate_statistics: + SAME TEXT AS ABOVE + + physical_plan after join_selection: + OutputRequirementExec: order_by=[(#13@0, asc)], dist_by=SinglePartition + SortExec: expr=[#13@0 ASC], preserve_partitioning=[false] + ProjectionExec: expr=[#12@0 as #13, count(Int64(1))@1 as #14, sum(#10)@2 as #15, sum(#11)@3 as #16] + AggregateExec: mode=FinalPartitioned, gby=[#12@0 as #12], aggr=[count(Int64(1)), sum(#10), sum(#11)] + AggregateExec: mode=Partial, gby=[#12@2 as #12], aggr=[count(Int64(1)), sum(#10), sum(#11)] + ProjectionExec: expr=[#2@1 as #10, #5@3 as #11, #1@0 % 4 as #12] + HashJoinExec: mode=Partitioned, join_type=Inner, on=[(#1@0, #4@0)] + ExplicitRepartitionExec + ProjectionExec: expr=[#0@0 as #1, #0@0 * 2 as #2] + RangeExec + ExplicitRepartitionExec + ProjectionExec: expr=[#3@0 as #4, #3@0 + 1 as #5] + RangeExec + + + physical_plan after LimitedDistinctAggregation: + SAME TEXT AS ABOVE + + physical_plan after FilterPushdown: + SAME TEXT AS ABOVE + + physical_plan after EnforceDistribution: + OutputRequirementExec: order_by=[(#13@0, asc)], dist_by=SinglePartition + SortExec: expr=[#13@0 ASC], preserve_partitioning=[false] + CoalescePartitionsExec + ProjectionExec: expr=[#12@0 as #13, count(Int64(1))@1 as #14, sum(#10)@2 as #15, sum(#11)@3 as #16] + AggregateExec: mode=FinalPartitioned, gby=[#12@0 as #12], aggr=[count(Int64(1)), sum(#10), sum(#11)] + RepartitionExec: partitioning=Hash([#12@0], 4), input_partitions=8 + AggregateExec: mode=Partial, gby=[#12@2 as #12], aggr=[count(Int64(1)), sum(#10), sum(#11)] + ProjectionExec: expr=[#2@1 as #10, #5@3 as #11, #1@0 % 4 as #12] + HashJoinExec: mode=Partitioned, join_type=Inner, on=[(#1@0, #4@0)] + ExplicitRepartitionExec + ProjectionExec: expr=[#0@0 as #1, #0@0 * 2 as #2] + RangeExec + ExplicitRepartitionExec + ProjectionExec: expr=[#3@0 as #4, #3@0 + 1 as #5] + RangeExec + + + physical_plan after CombinePartialFinalAggregate: + SAME TEXT AS ABOVE + + physical_plan after EnforceSorting: + OutputRequirementExec: order_by=[(#13@0, asc)], dist_by=SinglePartition + SortPreservingMergeExec: [#13@0 ASC] + SortExec: expr=[#13@0 ASC], preserve_partitioning=[true] + ProjectionExec: expr=[#12@0 as #13, count(Int64(1))@1 as #14, sum(#10)@2 as #15, sum(#11)@3 as #16] + AggregateExec: mode=FinalPartitioned, gby=[#12@0 as #12], aggr=[count(Int64(1)), sum(#10), sum(#11)] + RepartitionExec: partitioning=Hash([#12@0], 4), input_partitions=8 + AggregateExec: mode=Partial, gby=[#12@2 as #12], aggr=[count(Int64(1)), sum(#10), sum(#11)] + ProjectionExec: expr=[#2@1 as #10, #5@3 as #11, #1@0 % 4 as #12] + HashJoinExec: mode=Partitioned, join_type=Inner, on=[(#1@0, #4@0)] + ExplicitRepartitionExec + ProjectionExec: expr=[#0@0 as #1, #0@0 * 2 as #2] + RangeExec + ExplicitRepartitionExec + ProjectionExec: expr=[#3@0 as #4, #3@0 + 1 as #5] + RangeExec + + + physical_plan after OptimizeAggregateOrder: + SAME TEXT AS ABOVE + + physical_plan after WindowTopN: + SAME TEXT AS ABOVE + + physical_plan after ProjectionPushdown: + OutputRequirementExec: order_by=[(#13@0, asc)], dist_by=SinglePartition + SortPreservingMergeExec: [#13@0 ASC] + SortExec: expr=[#13@0 ASC], preserve_partitioning=[true] + ProjectionExec: expr=[#12@0 as #13, count(Int64(1))@1 as #14, sum(#10)@2 as #15, sum(#11)@3 as #16] + AggregateExec: mode=FinalPartitioned, gby=[#12@0 as #12], aggr=[count(Int64(1)), sum(#10), sum(#11)] + RepartitionExec: partitioning=Hash([#12@0], 4), input_partitions=8 + AggregateExec: mode=Partial, gby=[#12@2 as #12], aggr=[count(Int64(1)), sum(#10), sum(#11)] + ProjectionExec: expr=[#2@0 as #10, #5@1 as #11, #1@2 % 4 as #12] + HashJoinExec: mode=Partitioned, join_type=Inner, on=[(#1@0, #4@0)], projection=[#2@1, #5@3, #1@0] + ExplicitRepartitionExec + ProjectionExec: expr=[#0@0 as #1, #0@0 * 2 as #2] + RangeExec + ExplicitRepartitionExec + ProjectionExec: expr=[#3@0 as #4, #3@0 + 1 as #5] + RangeExec + + + physical_plan after OutputRequirements: + SortPreservingMergeExec: [#13@0 ASC] + SortExec: expr=[#13@0 ASC], preserve_partitioning=[true] + ProjectionExec: expr=[#12@0 as #13, count(Int64(1))@1 as #14, sum(#10)@2 as #15, sum(#11)@3 as #16] + AggregateExec: mode=FinalPartitioned, gby=[#12@0 as #12], aggr=[count(Int64(1)), sum(#10), sum(#11)] + RepartitionExec: partitioning=Hash([#12@0], 4), input_partitions=8 + AggregateExec: mode=Partial, gby=[#12@2 as #12], aggr=[count(Int64(1)), sum(#10), sum(#11)] + ProjectionExec: expr=[#2@0 as #10, #5@1 as #11, #1@2 % 4 as #12] + HashJoinExec: mode=Partitioned, join_type=Inner, on=[(#1@0, #4@0)], projection=[#2@1, #5@3, #1@0] + ExplicitRepartitionExec + ProjectionExec: expr=[#0@0 as #1, #0@0 * 2 as #2] + RangeExec + ExplicitRepartitionExec + ProjectionExec: expr=[#3@0 as #4, #3@0 + 1 as #5] + RangeExec + + + physical_plan after LimitAggregation: + SAME TEXT AS ABOVE + + physical_plan after LimitPushPastWindows: + SAME TEXT AS ABOVE + + physical_plan after HashJoinBuffering: + SAME TEXT AS ABOVE + + physical_plan after LimitPushdown: + SAME TEXT AS ABOVE + + physical_plan after TopKRepartition: + SAME TEXT AS ABOVE + + physical_plan after ProjectionPushdown: + SAME TEXT AS ABOVE + + physical_plan after PushdownSort: + SAME TEXT AS ABOVE + + physical_plan after EnsureCooperative: + SAME TEXT AS ABOVE + + physical_plan after FilterPushdown(Post): + SAME TEXT AS ABOVE + + physical_plan after RewriteExplicitRepartition: + SortPreservingMergeExec: [#13@0 ASC] + SortExec: expr=[#13@0 ASC], preserve_partitioning=[true] + ProjectionExec: expr=[#12@0 as #13, count(Int64(1))@1 as #14, sum(#10)@2 as #15, sum(#11)@3 as #16] + AggregateExec: mode=FinalPartitioned, gby=[#12@0 as #12], aggr=[count(Int64(1)), sum(#10), sum(#11)] + RepartitionExec: partitioning=Hash([#12@0], 4), input_partitions=8 + AggregateExec: mode=Partial, gby=[#12@2 as #12], aggr=[count(Int64(1)), sum(#10), sum(#11)] + ProjectionExec: expr=[#2@0 as #10, #5@1 as #11, #1@2 % 4 as #12] + HashJoinExec: mode=Partitioned, join_type=Inner, on=[(#1@0, #4@0)], projection=[#2@1, #5@3, #1@0] + RepartitionExec: partitioning=Hash([#1@0], 8), input_partitions=4 + ProjectionExec: expr=[#0@0 as #1, #0@0 * 2 as #2] + RangeExec + RepartitionExec: partitioning=Hash([#4@0], 8), input_partitions=4 + ProjectionExec: expr=[#3@0 as #4, #3@0 + 1 as #5] + RangeExec + + + physical_plan after RewriteCollectLeftHashJoin: + SAME TEXT AS ABOVE + + physical_plan after EnforceBarrierPartitioning: + SAME TEXT AS ABOVE + + physical_plan after SanityCheckPlan: + SAME TEXT AS ABOVE + + physical_plan: + ProjectionExec: expr=[#13@0 as g, #14@1 as count, #15@2 as s1, #16@3 as s2] + SortPreservingMergeExec: [#13@0 ASC] + SortExec: expr=[#13@0 ASC], preserve_partitioning=[true] + ProjectionExec: expr=[#12@0 as #13, count(Int64(1))@1 as #14, sum(#10)@2 as #15, sum(#11)@3 as #16] + AggregateExec: mode=FinalPartitioned, gby=[#12@0 as #12], aggr=[count(Int64(1)), sum(#10), sum(#11)] + RepartitionExec: partitioning=Hash([#12@0], 4), input_partitions=8 + AggregateExec: mode=Partial, gby=[#12@2 as #12], aggr=[count(Int64(1)), sum(#10), sum(#11)] + ProjectionExec: expr=[#2@0 as #10, #5@1 as #11, #1@2 % 4 as #12] + HashJoinExec: mode=Partitioned, join_type=Inner, on=[(#1@0, #4@0)], projection=[#2@1, #5@3, #1@0] + RepartitionExec: partitioning=Hash([#1@0], 8), input_partitions=4 + ProjectionExec: expr=[#0@0 as #1, #0@0 * 2 as #2] + RangeExec + RepartitionExec: partitioning=Hash([#4@0], 8), input_partitions=4 + ProjectionExec: expr=[#3@0 as #4, #3@0 + 1 as #5] + RangeExec + + == Physical Plan == ProjectionExec: expr=[#13@0 as g, #14@1 as count, #15@2 as s1, #16@3 as s2] SortPreservingMergeExec: [#13@0 ASC] diff --git a/python/pysail/tests/spark/execution/features/explain.feature b/python/pysail/tests/spark/execution/features/explain.feature index af588adc6..f11924950 100644 --- a/python/pysail/tests/spark/execution/features/explain.feature +++ b/python/pysail/tests/spark/execution/features/explain.feature @@ -1,8 +1,8 @@ -Feature: EXPLAIN in distributed execution - Scenario: EXPLAIN includes the distributed execution plan in local-cluster mode +Feature: EXPLAIN CODEGEN in distributed execution + Scenario: EXPLAIN CODEGEN includes the distributed execution plan When query """ - EXPLAIN + EXPLAIN CODEGEN SELECT k, SUM(v) AS total FROM VALUES (1, 2), (1, 3) t(k, v) GROUP BY k diff --git a/python/pysail/tests/spark/execution/features/lakehouse_commit.feature b/python/pysail/tests/spark/execution/features/lakehouse_commit.feature index 4203e05c5..6c1a00524 100644 --- a/python/pysail/tests/spark/execution/features/lakehouse_commit.feature +++ b/python/pysail/tests/spark/execution/features/lakehouse_commit.feature @@ -17,7 +17,7 @@ Feature: Lakehouse commits in distributed execution Scenario: Delta commit runs on the driver after parallel file writing When query """ - EXPLAIN + EXPLAIN CODEGEN INSERT INTO distributed_delta_commit SELECT id FROM range(0, 400, 1, 4) """ @@ -55,7 +55,7 @@ Feature: Lakehouse commits in distributed execution Scenario: Iceberg commit runs on the driver after parallel file writing When query """ - EXPLAIN + EXPLAIN CODEGEN INSERT INTO distributed_iceberg_commit SELECT id FROM range(0, 400, 1, 4) """ diff --git a/python/pysail/tests/spark/execution/features/scalar_subquery.feature b/python/pysail/tests/spark/execution/features/scalar_subquery.feature index 2bd474053..193c84f0c 100644 --- a/python/pysail/tests/spark/execution/features/scalar_subquery.feature +++ b/python/pysail/tests/spark/execution/features/scalar_subquery.feature @@ -13,7 +13,7 @@ Feature: Scalar subqueries in distributed execution | 2 | 10 | When query """ - EXPLAIN + EXPLAIN CODEGEN SELECT k, SUM(v) AS total FROM VALUES (1, 2), (1, 3), (2, 10), (3, 1) AS t(k, v) WHERE v > (SELECT MAX(x) FROM VALUES (1), (2) AS s(x)) @@ -35,7 +35,7 @@ Feature: Scalar subqueries in distributed execution | 36 | When query """ - EXPLAIN + EXPLAIN CODEGEN SELECT SUM(CAST(v AS BIGINT) + ( SELECT MIN(CAST(x AS BIGINT)) FROM VALUES (10), (20) AS s(x) @@ -60,7 +60,7 @@ Feature: Scalar subqueries in distributed execution | 2 | 13 | When query """ - EXPLAIN + EXPLAIN CODEGEN SELECT k, SUM(CAST(v AS BIGINT) + ( SELECT MIN(CAST(x AS BIGINT)) FROM VALUES (10), (20) AS s(x) @@ -88,7 +88,7 @@ Feature: Scalar subqueries in distributed execution | 2 | 3 | When query """ - EXPLAIN + EXPLAIN CODEGEN SELECT outer_t.k, outer_t.v FROM VALUES (1, 2), (1, 4), (2, 1), (2, 3) AS outer_t(k, v) WHERE outer_t.v = ( @@ -119,7 +119,7 @@ Feature: Scalar subqueries in distributed execution | 2 | 13 | When query """ - EXPLAIN + EXPLAIN CODEGEN SELECT k, SUM(shifted) AS total FROM ( SELECT k, CAST(v AS BIGINT) + ( @@ -148,7 +148,7 @@ Feature: Scalar subqueries in distributed execution | 1 | When query """ - EXPLAIN + EXPLAIN CODEGEN SELECT v FROM VALUES (1), (2), (3) AS t(v) ORDER BY CASE WHEN ( @@ -173,7 +173,7 @@ Feature: Scalar subqueries in distributed execution | 12 | 1 | When query """ - EXPLAIN + EXPLAIN CODEGEN SELECT CAST(v AS BIGINT) + ( SELECT MIN(CAST(x AS BIGINT)) FROM VALUES (10), (20) AS s(x) @@ -202,7 +202,7 @@ Feature: Scalar subqueries in distributed execution | 3 | 13 | When query """ - EXPLAIN + EXPLAIN CODEGEN SELECT l.v, r.w FROM VALUES (1), (2), (3) AS l(v) JOIN VALUES (11), (12), (13) AS r(w) @@ -233,7 +233,7 @@ Feature: Scalar subqueries in distributed execution | 2 | 13 | When query """ - EXPLAIN + EXPLAIN CODEGEN SELECT l.v, r.w FROM VALUES (1), (2), (3) AS l(v) JOIN VALUES (11), (12), (13) AS r(w) @@ -258,7 +258,7 @@ Feature: Scalar subqueries in distributed execution | 2 | 7 | When query """ - EXPLAIN + EXPLAIN CODEGEN SELECT k, SUM(v) AS total FROM VALUES (1, 1), (1, 2), (2, 3), (2, 4) AS t(k, v) GROUP BY k @@ -280,7 +280,7 @@ Feature: Scalar subqueries in distributed execution | 42 | When query """ - EXPLAIN + EXPLAIN CODEGEN SELECT SUM(CAST(v AS BIGINT) + (SELECT MIN(CAST(x AS BIGINT)) FROM VALUES (10), (20) AS s(x)) + (SELECT MAX(CAST(y AS BIGINT)) FROM VALUES (1), (2) AS u(y)) @@ -305,7 +305,7 @@ Feature: Scalar subqueries in distributed execution | 3 | 3 | When query """ - EXPLAIN + EXPLAIN CODEGEN SELECT v, ROW_NUMBER() OVER (ORDER BY CASE WHEN ( SELECT MIN(x) FROM VALUES (2) AS s(x) ) = 2 THEN v ELSE -v END) AS rn @@ -331,7 +331,7 @@ Feature: Scalar subqueries in distributed execution | 4 | 2 | When query """ - EXPLAIN + EXPLAIN CODEGEN SELECT v, COUNT(*) OVER (PARTITION BY CASE WHEN ( SELECT MIN(x) FROM VALUES (1) AS s(x) ) = 1 THEN v % 2 ELSE v END) AS n @@ -357,7 +357,7 @@ Feature: Scalar subqueries in distributed execution | 3 | 36 | When query """ - EXPLAIN + EXPLAIN CODEGEN SELECT v, SUM(CAST(v AS BIGINT) + ( SELECT MIN(CAST(x AS BIGINT)) FROM VALUES (10), (20) AS s(x) @@ -382,7 +382,7 @@ Feature: Scalar subqueries in distributed execution | 2 | 1 | When query """ - EXPLAIN + EXPLAIN CODEGEN SELECT k, COUNT(*) FILTER (WHERE v > ( SELECT MIN(x) FROM VALUES (1) AS s(x) )) AS n @@ -407,7 +407,7 @@ Feature: Scalar subqueries in distributed execution | b,a,c | When query """ - EXPLAIN + EXPLAIN CODEGEN SELECT listagg(v, ',') WITHIN GROUP ( ORDER BY CASE WHEN ( SELECT MIN(x) FROM VALUES (1) AS s(x) @@ -433,7 +433,7 @@ Feature: Scalar subqueries in distributed execution | 2 | 13 | When query """ - EXPLAIN + EXPLAIN CODEGEN SELECT v, CAST(v AS BIGINT) + ( SELECT MIN(x) + (SELECT MIN(y) FROM VALUES (1) AS u(y)) FROM VALUES (10), (20) AS s(x) @@ -460,7 +460,7 @@ Feature: Scalar subqueries in distributed execution | 2 | NULL | When query """ - EXPLAIN + EXPLAIN CODEGEN SELECT v, CAST(v AS BIGINT) + ( SELECT CAST(x AS BIGINT) FROM VALUES (10) AS s(x) @@ -487,7 +487,7 @@ Feature: Scalar subqueries in distributed execution | 2 | NULL | When query """ - EXPLAIN + EXPLAIN CODEGEN SELECT v, CAST(v AS BIGINT) + ( SELECT MIN(CAST(x AS BIGINT)) FROM VALUES (CAST(NULL AS INT)) AS s(x) @@ -545,7 +545,7 @@ Feature: Scalar subqueries in distributed execution | 2 | 3 | 3 | When query """ - EXPLAIN + EXPLAIN CODEGEN SELECT outer_t.k, outer_t.v, ( SELECT MAX(inner_t.x) FROM VALUES (1, 4), (1, 2), (2, 3), (2, 1) AS inner_t(k, x) @@ -573,7 +573,7 @@ Feature: Scalar subqueries in distributed execution | 2 | 3 | NULL | When query """ - EXPLAIN + EXPLAIN CODEGEN SELECT outer_t.k, outer_t.v, ( SELECT MAX(inner_t.x) FROM VALUES (1, 4), (1, 2) AS inner_t(k, x) @@ -603,7 +603,7 @@ Feature: Scalar subqueries in distributed execution | 2 | 3 | 4 | When query """ - EXPLAIN + EXPLAIN CODEGEN SELECT outer_t.k, outer_t.v, ( SELECT MAX(inner_t.x) + (SELECT MIN(y) FROM VALUES (1) AS u(y)) FROM VALUES (1, 4), (1, 2), (2, 3), (2, 1) AS inner_t(k, x) @@ -632,7 +632,7 @@ Feature: Scalar subqueries in distributed execution | 2 | 4 | When query """ - EXPLAIN + EXPLAIN CODEGEN SELECT outer_t.k, outer_t.v FROM VALUES (1, 2), (1, 5), (2, 1), (2, 4) AS outer_t(k, v) WHERE outer_t.v = ( diff --git a/python/pysail/tests/spark/execution/test_shuffle_storage.py b/python/pysail/tests/spark/execution/test_shuffle_storage.py index 165568788..df3a0a470 100644 --- a/python/pysail/tests/spark/execution/test_shuffle_storage.py +++ b/python/pysail/tests/spark/execution/test_shuffle_storage.py @@ -51,7 +51,7 @@ def test_query_execution_with_storage_shuffle(spark, snapshot): .orderBy("g") ) - plan = normalize_plan_text(df._explain_string()) # noqa: SLF001 + plan = normalize_plan_text(df._explain_string(mode="codegen")) # noqa: SLF001 assert plan == snapshot actual = df.toPandas() From 6fdd8154a41fd22b9e8231493bff6f1f2080210a Mon Sep 17 00:00:00 2001 From: Xiaolong Liang Date: Thu, 30 Jul 2026 18:11:27 +0800 Subject: [PATCH 099/131] fix: improve Spark datetime compatibility (#2322) --- .../proto/sail/plan/physical.proto | 1 + crates/sail-execution/src/proto/codec.rs | 23 +- .../src/scalar/csv/schema_of_csv.rs | 8 +- .../src/scalar/csv/spark_from_csv.rs | 12 +- .../src/scalar/csv/spark_to_csv.rs | 12 +- .../src/scalar/datetime/format/formatting.rs | 302 ++-------- .../src/scalar/datetime/format/parser.rs | 126 +++-- .../src/scalar/datetime/format/parsing.rs | 335 ++++++----- .../src/scalar/datetime/format/pattern.rs | 112 +--- .../sail-function/src/scalar/datetime/mod.rs | 1 - .../src/scalar/datetime/spark_date.rs | 41 +- .../src/scalar/datetime/spark_date_format.rs | 6 +- .../src/scalar/datetime/spark_time.rs | 42 +- .../src/scalar/datetime/spark_timestamp.rs | 27 +- .../scalar/datetime/spark_to_chrono_fmt.rs | 86 --- .../scalar/datetime/spark_unix_timestamp.rs | 343 ++++++++---- .../src/scalar/datetime/utils.rs | 152 ----- .../src/scalar/json/from_json.rs | 12 +- .../sail-function/src/scalar/json/to_json.rs | 12 +- .../sail-function/src/scalar/xml/from_xml.rs | 112 ++-- crates/sail-function/src/scalar/xml/to_xml.rs | 20 +- .../sail-plan/src/function/scalar/datetime.rs | 90 ++- .../datetime/datetime_edge_cases.feature | 91 ++- .../features/datetime/datetime_format.feature | 305 ++++------ .../features/datetime/datetime_parse.feature | 525 ++++++++---------- .../features/datetime/to_date.feature | 157 ++++++ .../features/datetime/to_time.feature | 46 +- .../datetime/to_unix_timestamp.feature | 223 +++++++- .../features/datetime/try_to_time.feature | 68 ++- .../features/datetime/unix_timestamp.feature | 222 +++++++- .../function/features/xml/from_xml.feature | 51 ++ 31 files changed, 2001 insertions(+), 1562 deletions(-) delete mode 100644 crates/sail-function/src/scalar/datetime/spark_to_chrono_fmt.rs diff --git a/crates/sail-execution/proto/sail/plan/physical.proto b/crates/sail-execution/proto/sail/plan/physical.proto index 077c5f325..a83847618 100644 --- a/crates/sail-execution/proto/sail/plan/physical.proto +++ b/crates/sail-execution/proto/sail/plan/physical.proto @@ -284,6 +284,7 @@ message SparkFromXmlUdf { message SparkUnixTimestampUdf { string session_timezone = 1; + bool ansi_mode = 2; } message SparkDateFormatUdf { diff --git a/crates/sail-execution/src/proto/codec.rs b/crates/sail-execution/src/proto/codec.rs index e1a395482..3a0a67611 100644 --- a/crates/sail-execution/src/proto/codec.rs +++ b/crates/sail-execution/src/proto/codec.rs @@ -162,7 +162,6 @@ use sail_function::scalar::datetime::spark_time::SparkTime; use sail_function::scalar::datetime::spark_time_diff::SparkTimeDiff; use sail_function::scalar::datetime::spark_time_trunc::SparkTimeTrunc; use sail_function::scalar::datetime::spark_timestamp::SparkTimestamp; -use sail_function::scalar::datetime::spark_to_chrono_fmt::SparkToChronoFmt; use sail_function::scalar::datetime::spark_try_to_timestamp::SparkTryToTimestamp; use sail_function::scalar::datetime::spark_unix_timestamp::SparkUnixTimestamp; use sail_function::scalar::datetime::spark_window_buckets::SparkWindowBuckets; @@ -2458,8 +2457,11 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { let udf = SparkFromXml::new(Arc::from(session_timezone)); return Ok(Arc::new(ScalarUDF::from(udf))); } - UdfKind::SparkUnixTimestamp(r#gen::SparkUnixTimestampUdf { session_timezone }) => { - let udf = SparkUnixTimestamp::new(Arc::from(session_timezone)); + UdfKind::SparkUnixTimestamp(r#gen::SparkUnixTimestampUdf { + session_timezone, + ansi_mode, + }) => { + let udf = SparkUnixTimestamp::new(Arc::from(session_timezone), ansi_mode); return Ok(Arc::new(ScalarUDF::from(udf))); } UdfKind::SparkDateFormat(r#gen::SparkDateFormatUdf { session_timezone }) => { @@ -2737,7 +2739,6 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { "spark_try_to_timestamp" | "try_to_timestamp" => { Ok(Arc::new(ScalarUDF::from(SparkTryToTimestamp::new()))) } - "spark_to_chrono_fmt" => Ok(Arc::new(ScalarUDF::from(SparkToChronoFmt::new()))), "spark_expm1" | "expm1" => Ok(Arc::new(ScalarUDF::from(SparkExpm1::new()))), "spark_to_utf8" => Ok(Arc::new(ScalarUDF::from(SparkToUtf8::new()))), "spark_to_large_utf8" => Ok(Arc::new(ScalarUDF::from(SparkToLargeUtf8::new()))), @@ -2955,7 +2956,11 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { UdfKind::SparkFromXml(r#gen::SparkFromXmlUdf { session_timezone }) } else if let Some(func) = node.inner().downcast_ref::() { let session_timezone = func.session_timezone().to_string(); - UdfKind::SparkUnixTimestamp(r#gen::SparkUnixTimestampUdf { session_timezone }) + let ansi_mode = func.ansi_mode(); + UdfKind::SparkUnixTimestamp(r#gen::SparkUnixTimestampUdf { + session_timezone, + ansi_mode, + }) } else if let Some(func) = node.inner().downcast_ref::() { let session_timezone = func.session_timezone().to_string(); UdfKind::SparkDateFormat(r#gen::SparkDateFormatUdf { session_timezone }) @@ -5633,12 +5638,14 @@ mod tests { #[test] fn test_round_trip_spark_unix_timestamp_preserves_options() -> Result<()> { - let decoded = round_trip_udf(ScalarUDF::from(SparkUnixTimestamp::new(Arc::from( - "America/Los_Angeles", - ))))?; + let decoded = round_trip_udf(ScalarUDF::from(SparkUnixTimestamp::new( + Arc::from("America/Los_Angeles"), + true, + )))?; let decoded = downcast_udf::(&decoded, "SparkUnixTimestamp")?; assert_eq!(decoded.session_timezone(), "America/Los_Angeles"); + assert!(decoded.ansi_mode()); Ok(()) } diff --git a/crates/sail-function/src/scalar/csv/schema_of_csv.rs b/crates/sail-function/src/scalar/csv/schema_of_csv.rs index 1a55c6f9a..50f89ebaf 100644 --- a/crates/sail-function/src/scalar/csv/schema_of_csv.rs +++ b/crates/sail-function/src/scalar/csv/schema_of_csv.rs @@ -163,9 +163,9 @@ impl Default for SparkSchemaOfCsvOptions { fn default() -> Self { Self { sep: ",".to_string(), - timestamp_format: DateTimeFormat::parse("yyyy-MM-dd HH:mm:ss") + timestamp_format: DateTimeFormat::for_parsing("yyyy-MM-dd HH:mm:ss") .expect("default timestamp format should be valid"), - date_format: DateTimeFormat::parse("yyyy-MM-dd") + date_format: DateTimeFormat::for_parsing("yyyy-MM-dd") .expect("default date format should be valid"), timezone: None, } @@ -210,10 +210,10 @@ impl SparkSchemaOfCsvOptions { match key { "sep" | "delimiter" => options.sep = value.to_string(), "timestampFormat" => { - options.timestamp_format = DateTimeFormat::parse(value)?; + options.timestamp_format = DateTimeFormat::for_parsing(value)?; } "dateFormat" => { - options.date_format = DateTimeFormat::parse(value)?; + options.date_format = DateTimeFormat::for_parsing(value)?; } "timeZone" => { options.timezone = Some(value.to_string()); diff --git a/crates/sail-function/src/scalar/csv/spark_from_csv.rs b/crates/sail-function/src/scalar/csv/spark_from_csv.rs index 41ac8f20a..5ca4f15f8 100644 --- a/crates/sail-function/src/scalar/csv/spark_from_csv.rs +++ b/crates/sail-function/src/scalar/csv/spark_from_csv.rs @@ -69,21 +69,21 @@ impl SparkFromCSVOptions { let timestamp_format = find_key_value(map, Self::TIMESTAMP_FORMAT_OPTION) .as_deref() - .map(DateTimeFormat::parse) + .map(DateTimeFormat::for_parsing) .transpose()? .unwrap_or_else(|| { #[expect(clippy::expect_used)] - DateTimeFormat::parse(Self::TIMESTAMP_FORMAT_DEFAULT) + DateTimeFormat::for_parsing(Self::TIMESTAMP_FORMAT_DEFAULT) .expect("default timestamp format should be valid") }); let date_format = find_key_value(map, Self::DATE_FORMAT_OPTION) .as_deref() - .map(DateTimeFormat::parse) + .map(DateTimeFormat::for_parsing) .transpose()? .unwrap_or_else(|| { #[expect(clippy::expect_used)] - DateTimeFormat::parse(Self::DATE_FORMAT_DEFAULT) + DateTimeFormat::for_parsing(Self::DATE_FORMAT_DEFAULT) .expect("default date format should be valid") }); @@ -104,9 +104,9 @@ impl Default for SparkFromCSVOptions { fn default() -> Self { Self { sep: Self::SEP_DEFAULT.to_string(), - timestamp_format: DateTimeFormat::parse(Self::TIMESTAMP_FORMAT_DEFAULT) + timestamp_format: DateTimeFormat::for_parsing(Self::TIMESTAMP_FORMAT_DEFAULT) .expect("default timestamp format should be valid"), - date_format: DateTimeFormat::parse(Self::DATE_FORMAT_DEFAULT) + date_format: DateTimeFormat::for_parsing(Self::DATE_FORMAT_DEFAULT) .expect("default date format should be valid"), timezone: None, } diff --git a/crates/sail-function/src/scalar/csv/spark_to_csv.rs b/crates/sail-function/src/scalar/csv/spark_to_csv.rs index 2d74d5a4f..fbe13f49a 100644 --- a/crates/sail-function/src/scalar/csv/spark_to_csv.rs +++ b/crates/sail-function/src/scalar/csv/spark_to_csv.rs @@ -18,19 +18,19 @@ use crate::scalar::datetime::format::{ fn default_timestamp_format() -> DateTimeFormat { #[expect(clippy::expect_used)] - DateTimeFormat::parse(SparkToCsvOptions::TIMESTAMP_FORMAT_DEFAULT) + DateTimeFormat::for_formatting(SparkToCsvOptions::TIMESTAMP_FORMAT_DEFAULT) .expect("default timestamp format should be valid") } fn default_date_format() -> DateTimeFormat { #[expect(clippy::expect_used)] - DateTimeFormat::parse(SparkToCsvOptions::DATE_FORMAT_DEFAULT) + DateTimeFormat::for_formatting(SparkToCsvOptions::DATE_FORMAT_DEFAULT) .expect("default date format should be valid") } fn default_ltz_format() -> DateTimeFormat { #[expect(clippy::expect_used)] - DateTimeFormat::parse("yyyy-MM-dd'T'HH:mm:ss.SSSXXX") + DateTimeFormat::for_formatting("yyyy-MM-dd'T'HH:mm:ss.SSSXXX") .expect("default timestamp LTZ format should be valid") } @@ -138,13 +138,13 @@ impl SparkToCsvOptions { let timestamp_format = find_key_value(map, Self::TIMESTAMP_FORMAT_OPTION) .as_deref() - .map(DateTimeFormat::parse) + .map(DateTimeFormat::for_formatting) .transpose()? .unwrap_or_else(|| DEFAULT_TIMESTAMP_FORMAT.clone()); let date_format = find_key_value(map, Self::DATE_FORMAT_OPTION) .as_deref() - .map(DateTimeFormat::parse) + .map(DateTimeFormat::for_formatting) .transpose()? .unwrap_or_else(|| DEFAULT_DATE_FORMAT.clone()); @@ -1486,7 +1486,7 @@ mod tests { // Manually construct options with custom format let options = SparkToCsvOptions { - timestamp_format: DateTimeFormat::parse("dd/MM/yyyy")?, + timestamp_format: DateTimeFormat::for_formatting("dd/MM/yyyy")?, ..SparkToCsvOptions::default() }; diff --git a/crates/sail-function/src/scalar/datetime/format/formatting.rs b/crates/sail-function/src/scalar/datetime/format/formatting.rs index 9afa6ed28..3fcbc0b7e 100644 --- a/crates/sail-function/src/scalar/datetime/format/formatting.rs +++ b/crates/sail-function/src/scalar/datetime/format/formatting.rs @@ -1,10 +1,10 @@ -use chrono::{Datelike, Duration, FixedOffset, NaiveDateTime, Timelike}; +use chrono::{Datelike, FixedOffset, NaiveDateTime, Timelike}; use datafusion_common::Result; -use super::locale::{EN_US, LocaleData}; +use super::locale::LocaleData; use super::pattern::{ DateTimeField, DateTimeFieldSpec, DateTimeFormat, DateTimeItem, FieldStyle, FractionField, - FractionSpec, PredefinedFormatter, ZoneField, ZoneSpec, + FractionSpec, ZoneField, ZoneSpec, }; #[derive(Debug, Clone, Copy)] @@ -39,286 +39,38 @@ pub struct DateTimeFormatInput<'a> { impl DateTimeFormat { pub fn format(&self, input: DateTimeFormatInput<'_>) -> Result { - if let Some(predefined) = self.predefined { - return Ok(format_predefined(predefined, input)); - } let mut output = String::new(); format_items(&self.items, input, self.locale.data(), &mut output)?; Ok(output) } } -fn format_predefined(predefined: PredefinedFormatter, input: DateTimeFormatInput<'_>) -> String { - match predefined { - PredefinedFormatter::BasicIsoDate => { - let mut output = input.datetime.format("%Y%m%d").to_string(); - // Only include offset if timezone is present and not UTC - if let Some(tz) = input.timezone - && tz.offset.local_minus_utc() != 0 - { - push_basic_offset(offset_seconds(input), &mut output); - } - output - } - PredefinedFormatter::IsoLocalDate => input.datetime.format("%Y-%m-%d").to_string(), - PredefinedFormatter::IsoOffsetDate => { - let mut output = input.datetime.format("%Y-%m-%d").to_string(); - push_iso_offset_id(offset_seconds(input), &mut output); - output - } - PredefinedFormatter::IsoDate => { - let mut output = input.datetime.format("%Y-%m-%d").to_string(); - if input.timezone.is_some() { - push_iso_offset_id(offset_seconds(input), &mut output); - } - output - } - PredefinedFormatter::IsoLocalTime => format_iso_local_time(input.datetime), - PredefinedFormatter::IsoOffsetTime => { - let mut output = format_iso_local_time(input.datetime); - push_iso_offset_id(offset_seconds(input), &mut output); - output - } - PredefinedFormatter::IsoTime => { - let mut output = format_iso_local_time(input.datetime); - if input.timezone.is_some() { - push_iso_offset_id(offset_seconds(input), &mut output); - } - output - } - PredefinedFormatter::IsoLocalDateTime => format_iso_local_date_time(input.datetime), - PredefinedFormatter::IsoOffsetDateTime => { - let mut output = format_iso_local_date_time(input.datetime); - push_iso_offset_id(offset_seconds(input), &mut output); - output - } - PredefinedFormatter::IsoZonedDateTime => { - let mut output = format_iso_local_date_time(input.datetime); - push_iso_offset_id(offset_seconds(input), &mut output); - if let Some(name) = input.timezone.and_then(|timezone| timezone.name) { - output.push('['); - output.push_str(name); - output.push(']'); - } - output - } - PredefinedFormatter::IsoDateTime => { - let mut output = format_iso_local_date_time(input.datetime); - if input.timezone.is_some() { - push_iso_offset_id(offset_seconds(input), &mut output); - if let Some(name) = input.timezone.and_then(|timezone| timezone.name) { - output.push('['); - output.push_str(name); - output.push(']'); - } - } - output - } - PredefinedFormatter::IsoOrdinalDate => { - let mut output = format!( - "{:04}-{:03}", - input.datetime.year(), - input.datetime.ordinal() - ); - if input.timezone.is_some() { - push_iso_offset_id(offset_seconds(input), &mut output); - } - output - } - PredefinedFormatter::IsoWeekDate => { - let week = input.datetime.iso_week(); - let mut output = format!( - "{:04}-W{:02}-{}", - week.year(), - week.week(), - input.datetime.weekday().number_from_monday() - ); - if input.timezone.is_some() { - push_iso_offset_id(offset_seconds(input), &mut output); - } - output - } - PredefinedFormatter::IsoInstant => { - let utc_datetime = input - .datetime - .checked_sub_signed(Duration::seconds(offset_seconds(input) as i64)) - .unwrap_or(input.datetime); - let mut output = format_iso_local_date_time(utc_datetime); - output.push('Z'); - output - } - PredefinedFormatter::Rfc1123DateTime => { - let mut output = format!( - "{}, {} {} {} {:02}:{:02}:{:02} ", - EN_US.weekdays_short[input.datetime.weekday().num_days_from_monday() as usize], - input.datetime.day(), - EN_US.months_short[(input.datetime.month() - 1) as usize], - input.datetime.year(), - input.datetime.hour(), - input.datetime.minute(), - input.datetime.second() - ); - let seconds = offset_seconds(input); - if seconds == 0 { - output.push_str("GMT"); - } else { - push_basic_offset(seconds, &mut output); - } - output - } - } -} - -fn format_iso_local_date_time(datetime: NaiveDateTime) -> String { - let mut output = datetime.format("%Y-%m-%dT%H:%M:%S").to_string(); - push_iso_fraction(datetime.nanosecond(), &mut output); - output -} - -fn format_iso_local_time(datetime: NaiveDateTime) -> String { - let mut output = datetime.format("%H:%M:%S").to_string(); - push_iso_fraction(datetime.nanosecond(), &mut output); - output -} - -fn push_iso_fraction(nanos: u32, output: &mut String) { - if nanos == 0 { - return; - } - let mut digits = format!("{nanos:09}"); - while digits.ends_with('0') { - digits.pop(); - } - output.push('.'); - output.push_str(&digits); -} - -fn offset_seconds(input: DateTimeFormatInput<'_>) -> i32 { - input - .timezone - .map(|timezone| timezone.offset.local_minus_utc()) - .unwrap_or(0) -} - -fn push_iso_offset_id(seconds: i32, output: &mut String) { - if seconds == 0 { - output.push('Z'); - } else { - push_offset(seconds, true, seconds % 60 != 0, output); - } -} - -fn push_basic_offset(seconds: i32, output: &mut String) { - push_offset(seconds, false, seconds % 60 != 0, output); -} - fn format_items( items: &[DateTimeItem], input: DateTimeFormatInput<'_>, locale: &LocaleData, output: &mut String, ) -> Result<()> { - let mut pad_next: Option<(usize, char)> = None; - for item in items { match item { - DateTimeItem::Literal(value) => { - if let Some((width, pad_char)) = pad_next.take() { - let padded = pad_string(value, width, pad_char); - output.push_str(&padded); - } else { - output.push_str(value); - } - } + DateTimeItem::Literal(value) => output.push_str(value), DateTimeItem::Field(field_spec) => { - let start_len = output.len(); format_field_spec(field_spec, input, locale, output); - if let Some((width, pad_char)) = pad_next.take() { - let field_len = output.len() - start_len; - if field_len < width { - let padding = width - field_len; - let padded_value = format!( - "{}{}", - pad_char.to_string().repeat(padding), - &output[start_len..] - ); - output.truncate(start_len); - output.push_str(&padded_value); - } - } } DateTimeItem::Fraction(fraction_spec) => { - let start_len = output.len(); format_fraction_spec(fraction_spec, input, output); - if let Some((width, pad_char)) = pad_next.take() { - let field_len = output.len() - start_len; - if field_len < width { - let padding = width - field_len; - let padded_value = format!( - "{}{}", - pad_char.to_string().repeat(padding), - &output[start_len..] - ); - output.truncate(start_len); - output.push_str(&padded_value); - } - } } DateTimeItem::Zone(zone_spec) => { - let start_len = output.len(); format_zone_spec(zone_spec, input, output); - if let Some((width, pad_char)) = pad_next.take() { - let field_len = output.len() - start_len; - if field_len < width { - let padding = width - field_len; - let padded_value = format!( - "{}{}", - pad_char.to_string().repeat(padding), - &output[start_len..] - ); - output.truncate(start_len); - output.push_str(&padded_value); - } - } } DateTimeItem::Optional(items) => { - let start_len = output.len(); format_items(items, input, locale, output)?; - let optional_content = &output[start_len..]; - if is_all_zeros(optional_content) { - output.truncate(start_len); - } - } - DateTimeItem::PadNext { width, pad_char } => { - pad_next = Some((*width, *pad_char)); } } } Ok(()) } -fn is_all_zeros(s: &str) -> bool { - let mut has_digits = false; - for c in s.chars() { - if c.is_ascii_digit() { - has_digits = true; - if c != '0' { - return false; - } - } - } - has_digits -} - -fn pad_string(value: &str, width: usize, pad_char: char) -> String { - if value.len() >= width { - value.to_string() - } else { - let padding = width - value.len(); - format!("{}{}", pad_char.to_string().repeat(padding), value) - } -} - fn format_year(year: i32, count: usize, output: &mut String) { if count == 2 { push_exact_padded((year % 100).abs() as i64, 2, output); @@ -390,9 +142,16 @@ fn format_offset( output.push('Z'); return; } - let include_colon = count >= 3; - let include_seconds = count >= 4; - push_offset(seconds, include_colon, include_seconds, output); + let minutes = seconds.abs() % 3600 / 60; + let offset_seconds = seconds.abs() % 60; + match count { + 1 => push_offset(seconds, false, minutes != 0, false, output), + 2 => push_offset(seconds, false, true, false, output), + 3 => push_offset(seconds, true, true, false, output), + 4 => push_offset(seconds, false, true, offset_seconds != 0, output), + 5 => push_offset(seconds, true, true, offset_seconds != 0, output), + _ => unreachable!("offset pattern width is validated when the pattern is parsed"), + } } fn format_localized_offset( @@ -405,10 +164,22 @@ fn format_localized_offset( if seconds == 0 { return; } - push_offset(seconds, count == 4, count == 4 && seconds % 60 != 0, output); + push_offset( + seconds, + count == 4, + true, + count == 4 && seconds % 60 != 0, + output, + ); } -fn push_offset(seconds: i32, colon: bool, include_seconds: bool, output: &mut String) { +fn push_offset( + seconds: i32, + colon: bool, + include_minutes: bool, + include_seconds: bool, + output: &mut String, +) { let sign = if seconds < 0 { '-' } else { '+' }; let seconds = seconds.abs(); let hours = seconds / 3600; @@ -416,10 +187,12 @@ fn push_offset(seconds: i32, colon: bool, include_seconds: bool, output: &mut St let seconds = seconds % 60; output.push(sign); push_exact_padded(hours as i64, 2, output); - if colon { - output.push(':'); + if include_minutes { + if colon { + output.push(':'); + } + push_exact_padded(minutes as i64, 2, output); } - push_exact_padded(minutes as i64, 2, output); if include_seconds { if colon { output.push(':'); @@ -645,9 +418,16 @@ fn format_fraction_spec(spec: &FractionSpec, input: DateTimeFormatInput<'_>, out fn format_zone_spec(spec: &ZoneSpec, input: DateTimeFormatInput<'_>, output: &mut String) { match spec.kind { - ZoneField::Offset => { + ZoneField::IsoOffset => { format_offset(input.timezone, spec.width, spec.zero_as_z, output); } + ZoneField::Rfc822Offset => { + let seconds = input + .timezone + .map(|tz| tz.offset.local_minus_utc()) + .unwrap_or(0); + push_offset(seconds, false, true, false, output); + } ZoneField::LocalizedOffset => { format_localized_offset(input.timezone, spec.width, output); } diff --git a/crates/sail-function/src/scalar/datetime/format/parser.rs b/crates/sail-function/src/scalar/datetime/format/parser.rs index 2f85429e6..25ae5fb32 100644 --- a/crates/sail-function/src/scalar/datetime/format/parser.rs +++ b/crates/sail-function/src/scalar/datetime/format/parser.rs @@ -2,25 +2,32 @@ use datafusion_common::{Result, exec_err}; use super::pattern::{ DateTimeField, DateTimeFieldSpec, DateTimeFormat, DateTimeItem, FieldStyle, FractionField, - FractionSpec, ResolverStyle, SignStyle, ZoneField, ZoneSpec, ZoneStyle, + FractionSpec, PatternUse, ResolverStyle, SignStyle, ZoneField, ZoneSpec, ZoneStyle, }; -pub(crate) fn parse_datetime_pattern(pattern: &str) -> Result { +pub(crate) fn parse_datetime_pattern( + pattern: &str, + pattern_use: PatternUse, +) -> Result { let chars: Vec = pattern.chars().collect(); let mut position = 0; - let items = parse_items(&chars, &mut position, false)?; + let items = parse_items(&chars, &mut position, false, pattern_use)?; if position != chars.len() { return exec_err!("invalid datetime pattern: unexpected closing optional section"); } Ok(DateTimeFormat { items, - predefined: None, locale: Default::default(), - resolver_style: ResolverStyle::Smart, + resolver_style: ResolverStyle::Strict, }) } -fn parse_items(chars: &[char], position: &mut usize, optional: bool) -> Result> { +fn parse_items( + chars: &[char], + position: &mut usize, + optional: bool, + pattern_use: PatternUse, +) -> Result> { let mut items = Vec::new(); let mut literal = String::new(); @@ -31,14 +38,15 @@ fn parse_items(chars: &[char], position: &mut usize, optional: bool) -> Result { - literal.push(ch); - *position += 1; - } '[' => { flush_literal(&mut items, &mut literal); *position += 1; - items.push(DateTimeItem::Optional(parse_items(chars, position, true)?)); + items.push(DateTimeItem::Optional(parse_items( + chars, + position, + true, + pattern_use, + )?)); } ']' => { if optional { @@ -48,21 +56,7 @@ fn parse_items(chars: &[char], position: &mut usize, optional: bool) -> Result { - flush_literal(&mut items, &mut literal); - let token_start = *position; - *position += 1; - while *position < chars.len() && chars[*position] == 'p' { - *position += 1; - } - let count = *position - token_start; - items.push(DateTimeItem::PadNext { - width: count, - pad_char: ' ', - }); - } ch if ch.is_ascii_alphabetic() => { - // Try to parse as a pattern field let symbol = ch; let token_start = *position; *position += 1; @@ -71,18 +65,9 @@ fn parse_items(chars: &[char], position: &mut usize, optional: bool) -> Result { literal.push(ch); @@ -132,12 +117,40 @@ fn flush_literal(items: &mut Vec, literal: &mut String) { } } -fn validate_width(symbol: char, count: usize) -> Result<()> { +fn validate_pattern_field(symbol: char, count: usize, pattern_use: PatternUse) -> Result<()> { + if matches!(symbol, 'Y' | 'W' | 'w' | 'u' | 'e' | 'c') { + return exec_err!("invalid datetime pattern: week-based pattern letter '{symbol}'"); + } + if matches!(symbol, 'A' | 'B' | 'n' | 'N' | 'p') { + return exec_err!("invalid datetime pattern: unsupported pattern letter '{symbol}'"); + } + if pattern_use == PatternUse::Parsing && matches!(symbol, 'E' | 'F' | 'q' | 'Q') { + return exec_err!( + "invalid datetime pattern: pattern letter '{symbol}' is unsupported for parsing" + ); + } + match symbol { - 'E' if count > 5 => exec_err!("invalid datetime pattern: 'E' width must be 1 through 5")?, - 'G' if count > 5 => exec_err!("invalid datetime pattern: 'G' width must be 1 through 5")?, - 'M' | 'L' | 'Q' | 'q' if count > 5 => { - exec_err!("invalid datetime pattern: text field width must be 1 through 5")? + 'G' | 'M' | 'L' | 'E' | 'Q' | 'q' if count > 4 => { + exec_err!("invalid datetime pattern: text field width must be 1 through 4") + } + 'y' if count > 6 => { + exec_err!("invalid datetime pattern: 'y' width must be 1 through 6") + } + 'F' | 'a' if count != 1 => { + exec_err!("invalid datetime pattern: '{symbol}' requires width 1") + } + 'd' if count > 2 => { + exec_err!("invalid datetime pattern: 'd' width must be 1 or 2") + } + 'D' if count > 3 => { + exec_err!("invalid datetime pattern: 'D' width must be 1 through 3") + } + 'H' | 'h' | 'k' | 'K' | 'm' | 's' if count > 2 => { + exec_err!("invalid datetime pattern: '{symbol}' width must be 1 or 2") + } + 'S' if count > 9 => { + exec_err!("invalid datetime pattern: 'S' width must be 1 through 9") } 'V' if count != 2 => exec_err!("invalid datetime pattern: 'V' requires width 2"), 'O' if count != 1 && count != 4 => { @@ -146,16 +159,11 @@ fn validate_width(symbol: char, count: usize) -> Result<()> { 'X' | 'x' | 'Z' if count > 5 => { exec_err!("invalid datetime pattern: offset width must be 1 through 5") } - // Add width validation for previously unchecked fields - 'w' if count > 2 => exec_err!("invalid datetime pattern: 'w' width must be 1 or 2")?, - 'W' if count > 1 => exec_err!("invalid datetime pattern: 'W' width must be 1")?, - 'F' if count > 1 => exec_err!("invalid datetime pattern: 'F' width must be 1")?, - 'A' if count > 9 => exec_err!("invalid datetime pattern: 'A' width must be 1 through 9")?, - 'n' if count > 9 => exec_err!("invalid datetime pattern: 'n' width must be 1 through 9")?, - 'N' if count > 19 => exec_err!("invalid datetime pattern: 'N' width must be 1 through 19")?, - 'A' | 'D' | 'E' | 'F' | 'G' | 'H' | 'K' | 'L' | 'M' | 'N' | 'O' | 'Q' | 'S' | 'V' | 'W' - | 'X' | 'Y' | 'Z' | 'a' | 'c' | 'd' | 'e' | 'h' | 'k' | 'm' | 'n' | 'q' | 's' | 'u' - | 'w' | 'x' | 'y' | 'z' => Ok(()), + 'z' if count > 4 => { + exec_err!("invalid datetime pattern: 'z' width must be 1 through 4") + } + 'D' | 'E' | 'F' | 'G' | 'H' | 'K' | 'L' | 'M' | 'O' | 'Q' | 'S' | 'V' | 'X' | 'Z' | 'a' + | 'd' | 'h' | 'k' | 'm' | 'q' | 's' | 'x' | 'y' | 'z' => Ok(()), _ if symbol.is_ascii_alphabetic() => { exec_err!("invalid datetime pattern: unsupported pattern letter '{symbol}'") } @@ -163,7 +171,7 @@ fn validate_width(symbol: char, count: usize) -> Result<()> { } } -fn build_field_item(symbol: char, count: usize) -> Result { +fn build_field_item(symbol: char, count: usize, pattern_use: PatternUse) -> Result { match symbol { 'G' => Ok(DateTimeItem::Field(DateTimeFieldSpec { kind: DateTimeField::Era, @@ -351,7 +359,11 @@ fn build_field_item(symbol: char, count: usize) -> Result { })), 'S' => Ok(DateTimeItem::Fraction(FractionSpec { field: FractionField::NanoOfSecond, - min_width: count, + min_width: if pattern_use == PatternUse::Parsing { + 1 + } else { + count + }, max_width: count.min(9), decimal_point: false, })), @@ -374,7 +386,7 @@ fn build_field_item(symbol: char, count: usize) -> Result { sign_style: SignStyle::NotNegative, })), 'X' | 'x' => Ok(DateTimeItem::Zone(ZoneSpec { - kind: ZoneField::Offset, + kind: ZoneField::IsoOffset, width: count, zero_as_z: symbol == 'X', style: match count { @@ -386,8 +398,10 @@ fn build_field_item(symbol: char, count: usize) -> Result { 'Z' => Ok(DateTimeItem::Zone(ZoneSpec { kind: if count == 4 { ZoneField::LocalizedOffset + } else if count == 5 { + ZoneField::IsoOffset } else { - ZoneField::Offset + ZoneField::Rfc822Offset }, width: count, zero_as_z: count == 5, diff --git a/crates/sail-function/src/scalar/datetime/format/parsing.rs b/crates/sail-function/src/scalar/datetime/format/parsing.rs index 3416be227..d9016c613 100644 --- a/crates/sail-function/src/scalar/datetime/format/parsing.rs +++ b/crates/sail-function/src/scalar/datetime/format/parsing.rs @@ -1,5 +1,5 @@ use chrono::format::Parsed; -use chrono::{Datelike, Duration, FixedOffset, NaiveDate, NaiveDateTime, Timelike, Weekday}; +use chrono::{Datelike, FixedOffset, NaiveDate, NaiveDateTime, Weekday}; use datafusion_common::{Result, exec_datafusion_err}; use super::locale::LocaleData; @@ -28,17 +28,48 @@ struct ParseState { milli_of_day: Option, nano_of_day: Option, timezone: Option, - /// Flag indicating hour was 24 (midnight of next day) - hour_24: Option, - leap_second: bool, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +enum AdjacentNumericRun { + #[default] + None, + FixedWidth, + HasVariableWidth, + HasVariableWidthAcrossOptional, +} + +impl AdjacentNumericRun { + fn include(self, variable_width: bool) -> Self { + match self { + Self::HasVariableWidthAcrossOptional => Self::HasVariableWidthAcrossOptional, + Self::HasVariableWidth if !variable_width => Self::HasVariableWidth, + _ if variable_width => Self::HasVariableWidth, + _ => Self::FixedWidth, + } + } + + fn cross_optional(self) -> Self { + match self { + Self::HasVariableWidth | Self::HasVariableWidthAcrossOptional => { + Self::HasVariableWidthAcrossOptional + } + other => other, + } + } } impl DateTimeFormat { pub fn parse_datetime_value(&self, value: &str) -> Result { let mut state = ParseState::default(); - let value = value.trim(); - let mut position = parse_items(&self.items, value, 0, self.locale.data(), &mut state)?; - position = parse_optional_zone_region(value, position); + let (position, _) = parse_items( + &self.items, + value, + 0, + self.locale.data(), + &mut state, + AdjacentNumericRun::None, + )?; if position != value.len() { return Err(exec_datafusion_err!( "datetime value does not match format at byte offset {position}: {value}" @@ -58,93 +89,71 @@ fn parse_items( mut position: usize, locale: &LocaleData, state: &mut ParseState, -) -> Result { - let mut pad_next = false; + mut adjacent_numeric_run: AdjacentNumericRun, +) -> Result<(usize, AdjacentNumericRun)> { for item in items { match item { DateTimeItem::Literal(literal) => { position = consume_literal(value, position, literal)?; + adjacent_numeric_run = AdjacentNumericRun::None; } DateTimeItem::Field(field_spec) => { - if pad_next { - position = skip_padding(value, position); - pad_next = false; - } position = parse_field_spec(field_spec, value, position, locale, state)?; + if matches!( + field_spec.style, + FieldStyle::Numeric | FieldStyle::LocalizedNumeric + ) { + adjacent_numeric_run = adjacent_numeric_run + .include(field_has_variable_width_in_numeric_run(field_spec)); + } else { + adjacent_numeric_run = AdjacentNumericRun::None; + } } DateTimeItem::Fraction(fraction_spec) => { - if pad_next { - position = skip_padding(value, position); - pad_next = false; + if adjacent_numeric_run == AdjacentNumericRun::HasVariableWidthAcrossOptional + || (adjacent_numeric_run == AdjacentNumericRun::HasVariableWidth + && fraction_spec.min_width < fraction_spec.max_width) + { + return Err(exec_datafusion_err!( + "seconds fraction is ambiguous after a variable-width numeric datetime field" + )); } position = parse_fraction_spec(fraction_spec, value, position, state)?; + adjacent_numeric_run = AdjacentNumericRun::None; } DateTimeItem::Zone(zone_spec) => { - if pad_next { - position = skip_padding(value, position); - pad_next = false; - } position = parse_zone_spec(zone_spec, value, position, state)?; + adjacent_numeric_run = AdjacentNumericRun::None; } DateTimeItem::Optional(items) => { let snapshot = state.clone(); - match parse_items(items, value, position, locale, state) { - Ok(next) => position = next, + match parse_items(items, value, position, locale, state, adjacent_numeric_run) { + Ok((next, next_numeric_run)) => { + position = next; + adjacent_numeric_run = next_numeric_run.cross_optional(); + } Err(_) => { - *state = snapshot.clone(); - // Try partial literal matching for the first item - if let Some(DateTimeItem::Literal(literal)) = items.first() { - // Try to find a suffix of the literal that matches - let mut matched = false; - for i in 0..literal.len() { - let suffix = &literal[i..]; - if value[position..].starts_with(suffix) { - // Found matching suffix, parse rest of optional items - let new_position = position + suffix.len(); - match parse_items( - &items[1..], - value, - new_position, - locale, - state, - ) { - Ok(next) => { - position = next; - matched = true; - break; - } - Err(_) => { - *state = snapshot.clone(); - } - } - } - } - // Also try the original special case for single space - if !matched && literal == " " { - match parse_items(&items[1..], value, position, locale, state) { - Ok(next) => position = next, - Err(_) => *state = snapshot, - } - } - } + *state = snapshot; + adjacent_numeric_run = adjacent_numeric_run.cross_optional(); } } } - DateTimeItem::PadNext { .. } => { - pad_next = true; - } } } - Ok(position) + Ok((position, adjacent_numeric_run)) } -fn consume_literal(value: &str, position: usize, literal: &str) -> Result { - if literal == " " { - let next = skip_padding(value, position); - if next > position { - return Ok(next); +fn field_has_variable_width_in_numeric_run(spec: &DateTimeFieldSpec) -> bool { + match spec.kind { + DateTimeField::YearOfEra | DateTimeField::ProlepticYear | DateTimeField::WeekBasedYear => { + spec.width != 2 } + DateTimeField::DayOfYear => spec.width < 3, + _ => spec.width == 1, } +} + +fn consume_literal(value: &str, position: usize, literal: &str) -> Result { if value[position..].starts_with(literal) { Ok(position + literal.len()) } else { @@ -154,13 +163,6 @@ fn consume_literal(value: &str, position: usize, literal: &str) -> Result } } -fn skip_padding(value: &str, mut position: usize) -> usize { - while value[position..].starts_with(' ') { - position += 1; - } - position -} - fn parse_quarter( value: &str, position: usize, @@ -258,21 +260,23 @@ fn parse_fraction( fn parse_era( value: &str, position: usize, + style: FieldStyle, locale: &LocaleData, state: &mut ParseState, ) -> Result { - if starts_with_ignore_case(&value[position..], "CE") { - state.era_bc = Some(false); - return Ok(position + 2); - } - let choices = [ - (locale.eras_full[0], true), - (locale.eras_full[1], false), - (locale.eras_short[0], true), - (locale.eras_short[1], false), - (locale.eras_narrow[0], true), - (locale.eras_narrow[1], false), - ]; + let choices = match style { + FieldStyle::TextShort => [(locale.eras_short[0], true), (locale.eras_short[1], false)], + FieldStyle::TextFull => [(locale.eras_full[0], true), (locale.eras_full[1], false)], + FieldStyle::TextNarrow => [ + (locale.eras_narrow[0], true), + (locale.eras_narrow[1], false), + ], + _ => { + return Err(exec_datafusion_err!( + "unsupported datetime era field style: {style:?}" + )); + } + }; choices .iter() .filter(|(text, _)| starts_with_ignore_case(&value[position..], text)) @@ -369,10 +373,47 @@ fn parse_offset( .ok_or_else(|| exec_datafusion_err!("invalid UTC offset seconds: 0"))?; return Ok((position + 1, offset)); } - let colon = count >= 3; - let include_seconds = count >= 4; - let require_minutes = count != 1; - parse_numeric_offset(value, position, colon, include_seconds, require_minutes) + parse_iso_offset(value, position, count) +} + +fn parse_iso_offset(value: &str, position: usize, count: usize) -> Result<(usize, FixedOffset)> { + let sign = parse_offset_sign(value, position)?; + let (mut next, hours) = parse_number(value, position + 1, (2, 2))?; + let mut minutes = 0; + let mut seconds = 0; + + match count { + 1 => { + if matches!(value.as_bytes().get(next), Some(b'0'..=b'9')) { + (next, minutes) = parse_number(value, next, (2, 2))?; + } + } + 2 => (next, minutes) = parse_number(value, next, (2, 2))?, + 3 => { + next = consume_literal(value, next, ":")?; + (next, minutes) = parse_number(value, next, (2, 2))?; + } + 4 => { + (next, minutes) = parse_number(value, next, (2, 2))?; + if matches!(value.as_bytes().get(next), Some(b'0'..=b'9')) { + (next, seconds) = parse_number(value, next, (2, 2))?; + } + } + 5 => { + next = consume_literal(value, next, ":")?; + (next, minutes) = parse_number(value, next, (2, 2))?; + if value[next..].starts_with(':') { + next += 1; + (next, seconds) = parse_number(value, next, (2, 2))?; + } + } + _ => unreachable!("offset pattern width is validated when the pattern is parsed"), + } + + Ok(( + next, + fixed_offset_from_components(sign, hours, minutes, seconds)?, + )) } fn parse_localized_offset(value: &str, position: usize) -> Result<(usize, FixedOffset)> { @@ -397,21 +438,10 @@ fn parse_numeric_offset( include_seconds: bool, require_minutes: bool, ) -> Result<(usize, FixedOffset)> { - let sign = match value.as_bytes().get(position) { - Some(b'+') => 1, - Some(b'-') => -1, - _ => { - return Err(exec_datafusion_err!( - "expected offset sign at byte offset {position}" - )); - } - }; + let sign = parse_offset_sign(value, position)?; let (mut next, hours) = parse_number(value, position + 1, (2, 2))?; if !require_minutes && !matches!(value.as_bytes().get(next), Some(b'0'..=b'9') | Some(b':')) { - let total = sign * hours * 3600; - let offset = FixedOffset::east_opt(total) - .ok_or_else(|| exec_datafusion_err!("invalid datetime offset seconds: {total}"))?; - return Ok((next, offset)); + return Ok((next, fixed_offset_from_components(sign, hours, 0, 0)?)); } if colon { next = consume_literal(value, next, ":")?; @@ -427,10 +457,47 @@ fn parse_numeric_offset( next = parsed.0; seconds = parsed.1; } + Ok(( + next, + fixed_offset_from_components(sign, hours, minutes, seconds)?, + )) +} + +fn parse_offset_sign(value: &str, position: usize) -> Result { + match value.as_bytes().get(position) { + Some(b'+') => Ok(1), + Some(b'-') => Ok(-1), + _ => Err(exec_datafusion_err!( + "expected offset sign at byte offset {position}" + )), + } +} + +fn fixed_offset_from_components( + sign: i32, + hours: i32, + minutes: i32, + seconds: i32, +) -> Result { + if !(0..60).contains(&minutes) { + return Err(exec_datafusion_err!( + "invalid datetime offset minute component: {minutes}" + )); + } + if !(0..60).contains(&seconds) { + return Err(exec_datafusion_err!( + "invalid datetime offset second component: {seconds}" + )); + } + if hours > 18 || (hours == 18 && (minutes != 0 || seconds != 0)) { + return Err(exec_datafusion_err!( + "datetime offset exceeds the valid range of -18:00 to +18:00" + )); + } let total = sign * (hours * 3600 + minutes * 60 + seconds); let offset = FixedOffset::east_opt(total) .ok_or_else(|| exec_datafusion_err!("invalid datetime offset seconds: {total}"))?; - Ok((next, offset)) + Ok(offset) } fn parse_zone_name(value: &str, position: usize) -> Result<(usize, String)> { @@ -451,16 +518,6 @@ fn parse_zone_name(value: &str, position: usize) -> Result<(usize, String)> { } } -fn parse_optional_zone_region(value: &str, position: usize) -> usize { - if !value[position..].starts_with('[') { - return position; - } - value[position + 1..] - .find(']') - .map(|offset| position + offset + 2) - .unwrap_or(position) -} - fn parse_week_of_month_field( value: &str, position: usize, @@ -515,28 +572,16 @@ impl ParseState { fn resolve(mut self) -> Result { self.resolve_sail_fields()?; self.apply_defaults()?; - let add_day = self.hour_24.unwrap_or(false); - let mut date = self + let date = self .parsed .to_naive_date() .map_err(|e| exec_datafusion_err!("invalid parsed date: {e}"))?; - if add_day { - date = date.succ_opt().unwrap_or(date); - } validate_week_of_month(&self, date)?; - let mut datetime = date.and_time( + let datetime = date.and_time( self.parsed .to_naive_time() .map_err(|e| exec_datafusion_err!("invalid parsed time: {e}"))?, ); - if self.leap_second { - if datetime.hour() != 23 || datetime.minute() != 59 { - return Err(exec_datafusion_err!( - "Invalid value for SecondOfMinute (valid leap second must be 23:59:60)" - )); - } - datetime += Duration::seconds(1); - } let date = datetime.date(); let time = datetime.time(); let offset = self @@ -588,9 +633,6 @@ impl ParseState { "Invalid value for ClockHourOfDay (valid values 1 - 24): {hour}" )); } - if hour == 24 { - self.hour_24 = Some(true); - } self.set_hour(if hour == 24 { 0 } else { hour as i32 })?; } if let Some(hour) = self.hour_of_ampm { @@ -770,7 +812,7 @@ fn parse_field_spec( state: &mut ParseState, ) -> Result { match spec.kind { - DateTimeField::Era => parse_era(value, position, locale, state), + DateTimeField::Era => parse_era(value, position, spec.style, locale, state), DateTimeField::YearOfEra => { parse_signed_number(value, position, number_bounds(spec.width, 10)).map( |(next, year)| { @@ -826,13 +868,12 @@ fn parse_field_spec( DateTimeField::AmPmOfDay => parse_am_pm(value, position, locale, state), DateTimeField::HourOfDay => parse_number(value, position, number_bounds(spec.width, 2)) .and_then(|(next, hour)| { - if hour == 24 { - // Hour 24 means midnight of the next day - state.hour_24 = Some(true); - state.set_hour(0)?; - } else { - state.set_hour(hour)?; + if !(0..24).contains(&hour) { + return Err(exec_datafusion_err!( + "Invalid value for HourOfDay (valid values 0 - 23): {hour}" + )); } + state.set_hour(hour)?; Ok(next) }), DateTimeField::ClockHourOfDay => { @@ -865,17 +906,12 @@ fn parse_field_spec( DateTimeField::SecondOfMinute => { parse_number(value, position, number_bounds(spec.width, 2)).and_then( |(next, second)| { - if !(0..=60).contains(&second) { + if !(0..60).contains(&second) { return Err(exec_datafusion_err!( - "Invalid value for SecondOfMinute (valid values 0 - 60): {second}" + "Invalid value for SecondOfMinute (valid values 0 - 59): {second}" )); } - if second == 60 { - state.leap_second = true; - state.set_second(59)?; - } else { - state.set_second(second)?; - } + state.set_second(second)?; Ok(next) }, ) @@ -929,12 +965,17 @@ fn parse_zone_spec( state: &mut ParseState, ) -> Result { match spec.kind { - ZoneField::Offset => { + ZoneField::IsoOffset => { parse_offset(value, position, spec.width, spec.zero_as_z).and_then(|(next, offset)| { state.set_offset(offset)?; Ok(next) }) } + ZoneField::Rfc822Offset => parse_numeric_offset(value, position, false, false, true) + .and_then(|(next, offset)| { + state.set_offset(offset)?; + Ok(next) + }), ZoneField::LocalizedOffset => { parse_localized_offset(value, position).and_then(|(next, offset)| { state.set_offset(offset)?; diff --git a/crates/sail-function/src/scalar/datetime/format/pattern.rs b/crates/sail-function/src/scalar/datetime/format/pattern.rs index 863528727..eb75733a3 100644 --- a/crates/sail-function/src/scalar/datetime/format/pattern.rs +++ b/crates/sail-function/src/scalar/datetime/format/pattern.rs @@ -5,7 +5,6 @@ use super::parser::parse_datetime_pattern; #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct DateTimeFormat { pub(crate) items: Vec, - pub(crate) predefined: Option, pub(crate) locale: LocaleSpec, pub(crate) resolver_style: ResolverStyle, } @@ -25,110 +24,19 @@ pub enum ResolverStyle { } impl DateTimeFormat { - pub fn parse(pattern: &str) -> Result { - if let Some(predefined) = PredefinedFormatter::from_name(pattern) { - let mut format = parse_datetime_pattern(predefined.base_pattern())?; - if predefined.uses_variable_fraction() { - make_fraction_width_variable(&mut format.items); - } - format.predefined = Some(predefined); - format.locale = LocaleSpec::Default; - return Ok(format); - } - let mut format = parse_datetime_pattern(pattern)?; - format.locale = LocaleSpec::Default; - Ok(format) + pub fn for_parsing(pattern: &str) -> Result { + parse_datetime_pattern(pattern, PatternUse::Parsing) } -} -fn make_fraction_width_variable(items: &mut [DateTimeItem]) { - for item in items { - match item { - DateTimeItem::Fraction(spec) => { - spec.min_width = 1; - } - DateTimeItem::Optional(items) => make_fraction_width_variable(items), - _ => {} - } + pub fn for_formatting(pattern: &str) -> Result { + parse_datetime_pattern(pattern, PatternUse::Formatting) } } -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub(crate) enum PredefinedFormatter { - BasicIsoDate, - IsoLocalDate, - IsoOffsetDate, - IsoDate, - IsoLocalTime, - IsoOffsetTime, - IsoTime, - IsoLocalDateTime, - IsoOffsetDateTime, - IsoZonedDateTime, - IsoDateTime, - IsoOrdinalDate, - IsoWeekDate, - IsoInstant, - Rfc1123DateTime, -} - -impl PredefinedFormatter { - fn from_name(name: &str) -> Option { - match name { - "BASIC_ISO_DATE" => Some(Self::BasicIsoDate), - "ISO_LOCAL_DATE" => Some(Self::IsoLocalDate), - "ISO_OFFSET_DATE" => Some(Self::IsoOffsetDate), - "ISO_DATE" => Some(Self::IsoDate), - "ISO_LOCAL_TIME" => Some(Self::IsoLocalTime), - "ISO_OFFSET_TIME" => Some(Self::IsoOffsetTime), - "ISO_TIME" => Some(Self::IsoTime), - "ISO_LOCAL_DATE_TIME" => Some(Self::IsoLocalDateTime), - "ISO_OFFSET_DATE_TIME" => Some(Self::IsoOffsetDateTime), - "ISO_ZONED_DATE_TIME" => Some(Self::IsoZonedDateTime), - "ISO_DATE_TIME" => Some(Self::IsoDateTime), - "ISO_8601" => Some(Self::IsoDateTime), - "ISO_ORDINAL_DATE" => Some(Self::IsoOrdinalDate), - "ISO_WEEK_DATE" => Some(Self::IsoWeekDate), - "ISO_INSTANT" => Some(Self::IsoInstant), - "RFC_1123_DATE_TIME" => Some(Self::Rfc1123DateTime), - _ => None, - } - } - - fn base_pattern(self) -> &'static str { - match self { - Self::BasicIsoDate => "yyyyMMdd", - Self::IsoLocalDate => "yyyy-MM-dd", - Self::IsoOffsetDate => "yyyy-MM-ddXXX", - Self::IsoDate => "yyyy-MM-dd[XXX]", - Self::IsoLocalTime => "HH:mm:ss[.SSSSSSSSS]", - Self::IsoOffsetTime => "HH:mm:ss[.SSSSSSSSS]XXX", - Self::IsoTime => "HH:mm:ss[.SSSSSSSSS][XXX]", - Self::IsoLocalDateTime => "yyyy-MM-dd'T'HH:mm:ss[.SSSSSSSSS]", - Self::IsoOffsetDateTime | Self::IsoZonedDateTime => { - "yyyy-MM-dd'T'HH:mm:ss[.SSSSSSSSS]XXX" - } - Self::IsoDateTime => "yyyy-MM-dd'T'HH:mm:ss[.SSSSSSSSS][XXX]", - Self::IsoOrdinalDate => "yyyy-DDD", - Self::IsoWeekDate => "YYYY-'W'ww-e", - Self::IsoInstant => "yyyy-MM-dd'T'HH:mm:ss[.SSSSSSSSS]XXX", - Self::Rfc1123DateTime => "EEE, d MMM yyyy HH:mm:ss z", - } - } - - fn uses_variable_fraction(self) -> bool { - matches!( - self, - Self::IsoLocalTime - | Self::IsoOffsetTime - | Self::IsoTime - | Self::IsoLocalDateTime - | Self::IsoOffsetDateTime - | Self::IsoZonedDateTime - | Self::IsoDateTime - | Self::IsoInstant - ) - } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum PatternUse { + Parsing, + Formatting, } #[derive(Debug, Clone, PartialEq, Eq, Hash)] @@ -138,7 +46,6 @@ pub(crate) enum DateTimeItem { Fraction(FractionSpec), Zone(ZoneSpec), Optional(Vec), - PadNext { width: usize, pad_char: char }, } #[derive(Debug, Clone, PartialEq, Eq, Hash)] @@ -223,7 +130,8 @@ pub(crate) struct ZoneSpec { #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub(crate) enum ZoneField { - Offset, + IsoOffset, + Rfc822Offset, LocalizedOffset, ZoneId, ZoneName, diff --git a/crates/sail-function/src/scalar/datetime/mod.rs b/crates/sail-function/src/scalar/datetime/mod.rs index 62092303b..735941a2a 100644 --- a/crates/sail-function/src/scalar/datetime/mod.rs +++ b/crates/sail-function/src/scalar/datetime/mod.rs @@ -15,7 +15,6 @@ pub mod spark_time; pub mod spark_time_diff; pub mod spark_time_trunc; pub mod spark_timestamp; -pub mod spark_to_chrono_fmt; pub mod spark_try_to_timestamp; pub mod spark_unix_timestamp; pub mod spark_window_buckets; diff --git a/crates/sail-function/src/scalar/datetime/spark_date.rs b/crates/sail-function/src/scalar/datetime/spark_date.rs index 64c54c297..50281fd0a 100644 --- a/crates/sail-function/src/scalar/datetime/spark_date.rs +++ b/crates/sail-function/src/scalar/datetime/spark_date.rs @@ -2,7 +2,7 @@ use std::collections::HashMap; use std::collections::hash_map::Entry; use std::sync::Arc; -use datafusion::arrow::array::{Array, Date32Array}; +use datafusion::arrow::array::{Array, Date32Array, new_null_array}; use datafusion::arrow::datatypes::{DataType, Date32Type}; use datafusion_common::cast::{as_large_string_array, as_string_array, as_string_view_array}; use datafusion_common::{Result, ScalarValue, exec_datafusion_err, exec_err}; @@ -97,11 +97,17 @@ impl ScalarUDFImpl for SparkDate { } (ColumnarValue::Array(array), format) => { let format = match format { - Some(ColumnarValue::Scalar(scalar)) => scalar - .try_as_str() - .flatten() - .map(DateTimeFormat::parse) - .transpose()?, + Some(ColumnarValue::Scalar(scalar)) => { + match parse_scalar_date_format(&scalar)? { + Some(format) => Some(format), + None => { + return Ok(ColumnarValue::Array(new_null_array( + &DataType::Date32, + array.len(), + ))); + } + } + } Some(ColumnarValue::Array(_)) => unreachable!(), None => None, }; @@ -145,11 +151,14 @@ impl ScalarUDFImpl for SparkDate { } (ColumnarValue::Scalar(scalar), format) => { let format = match format { - Some(ColumnarValue::Scalar(scalar)) => scalar - .try_as_str() - .flatten() - .map(DateTimeFormat::parse) - .transpose()?, + Some(ColumnarValue::Scalar(scalar)) => { + match parse_scalar_date_format(&scalar)? { + Some(format) => Some(format), + None => { + return Ok(ColumnarValue::Scalar(ScalarValue::Date32(None))); + } + } + } Some(ColumnarValue::Array(_)) => unreachable!(), None => None, }; @@ -171,6 +180,14 @@ impl ScalarUDFImpl for SparkDate { } } +fn parse_scalar_date_format(format: &ScalarValue) -> Result> { + match format.try_as_str() { + Some(Some(pattern)) => Ok(Some(DateTimeFormat::for_parsing(pattern)?)), + Some(None) => Ok(None), + None => exec_err!("spark_date format argument must be a string scalar"), + } +} + fn parse_array_with_format_array( array: &Arc, format_array: &Arc, @@ -242,6 +259,6 @@ fn get_or_parse_format<'a>( ) -> Result<&'a DateTimeFormat> { match cache.entry(pattern.to_string()) { Entry::Occupied(entry) => Ok(entry.into_mut()), - Entry::Vacant(entry) => Ok(entry.insert(DateTimeFormat::parse(pattern)?)), + Entry::Vacant(entry) => Ok(entry.insert(DateTimeFormat::for_parsing(pattern)?)), } } diff --git a/crates/sail-function/src/scalar/datetime/spark_date_format.rs b/crates/sail-function/src/scalar/datetime/spark_date_format.rs index 902436bfa..2e22cfcda 100644 --- a/crates/sail-function/src/scalar/datetime/spark_date_format.rs +++ b/crates/sail-function/src/scalar/datetime/spark_date_format.rs @@ -69,7 +69,7 @@ impl ScalarUDFImpl for SparkDateFormat { None => return Ok(null_string_array(array.len())), }; - let format = DateTimeFormat::parse(format_str)?; + let format = DateTimeFormat::for_formatting(format_str)?; let result: StringArray = match array.data_type() { DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View => { @@ -124,7 +124,7 @@ impl ScalarUDFImpl for SparkDateFormat { None => return Ok(ColumnarValue::Scalar(ScalarValue::Utf8(None))), }; - let format = DateTimeFormat::parse(format_str)?; + let format = DateTimeFormat::for_formatting(format_str)?; let result = match scalar { ScalarValue::Utf8(Some(value)) @@ -522,7 +522,7 @@ fn get_or_parse_format<'a>( ) -> Result<&'a DateTimeFormat> { match cache.entry(pattern.to_string()) { Entry::Occupied(entry) => Ok(entry.into_mut()), - Entry::Vacant(entry) => Ok(entry.insert(DateTimeFormat::parse(pattern)?)), + Entry::Vacant(entry) => Ok(entry.insert(DateTimeFormat::for_formatting(pattern)?)), } } diff --git a/crates/sail-function/src/scalar/datetime/spark_time.rs b/crates/sail-function/src/scalar/datetime/spark_time.rs index d74bf650e..43ee7f9be 100644 --- a/crates/sail-function/src/scalar/datetime/spark_time.rs +++ b/crates/sail-function/src/scalar/datetime/spark_time.rs @@ -1,3 +1,5 @@ +use std::collections::HashMap; +use std::collections::hash_map::Entry; use std::sync::Arc; use chrono::{NaiveTime, Timelike}; @@ -10,6 +12,7 @@ use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signatur use datafusion_functions::utils::make_scalar_function; use crate::error::{invalid_arg_count_exec_err, unsupported_data_type_exec_err}; +use crate::scalar::datetime::format::DateTimeFormat; const DEFAULT_TIME_FORMATS: &[&str] = &[ "%H:%M:%S%.f", @@ -25,8 +28,8 @@ const DEFAULT_TIME_FORMATS: &[&str] = &[ /// /// Accepts 1 or 2 arguments: /// - `(expr)` — parses strings with default formats, or casts other types to Time64. -/// - `(expr, format)` — parses strings with the given chrono format. The format -/// may be a scalar string (broadcast) or a string column (per-row). +/// - `(expr, format)` — parses strings with the given Spark Java datetime pattern. +/// The pattern may be a scalar string (broadcast) or a string column (per-row). /// /// `to_time` always errors on invalid input (Spark's `ToTime` does not honor ANSI); /// `try_to_time` (`is_try = true`) returns NULL on parse/cast failure — mirroring @@ -72,16 +75,35 @@ impl SparkTime { fn string_to_time_us_with_format( value: &str, - format: &str, + format: &DateTimeFormat, is_try: bool, ) -> Result> { - match NaiveTime::parse_from_str(value, format) { - Ok(t) => Ok(Some(Self::naive_time_to_us(t))), + match format.parse_datetime_value(value) { + Ok(parsed) => Ok(Some(Self::naive_time_to_us(parsed.datetime.time()))), Err(_) if is_try => Ok(None), - Err(e) => Err(exec_datafusion_err!("{e}")), + Err(e) => Err(e), } } + fn get_or_parse_format<'a>( + cache: &'a mut HashMap>, + pattern: &str, + is_try: bool, + ) -> Result> { + let format = match cache.entry(pattern.to_string()) { + Entry::Occupied(entry) => entry.into_mut(), + Entry::Vacant(entry) => { + let format = match DateTimeFormat::for_parsing(pattern) { + Ok(format) => Some(format), + Err(_) if is_try => None, + Err(e) => return Err(e), + }; + entry.insert(format) + } + }; + Ok(format.as_ref()) + } + fn string_array_iter(array: &ArrayRef) -> Result> + '_>> { match array.data_type() { DataType::Utf8 => Ok(Box::new(as_string_array(array)?.iter())), @@ -116,10 +138,16 @@ impl SparkTime { } let values = Self::string_array_iter(value_arr)?; let formats = Self::string_array_iter(format_arr)?; + let mut cache = HashMap::new(); let out: Time64MicrosecondArray = values .zip(formats) .map(|(v, f)| match (v, f) { - (Some(s), Some(fmt)) => Self::string_to_time_us_with_format(s, fmt, is_try), + (Some(value), Some(pattern)) => { + match Self::get_or_parse_format(&mut cache, pattern, is_try)? { + Some(format) => Self::string_to_time_us_with_format(value, format, is_try), + None => Ok(None), + } + } _ => Ok(None), }) .collect::>()?; diff --git a/crates/sail-function/src/scalar/datetime/spark_timestamp.rs b/crates/sail-function/src/scalar/datetime/spark_timestamp.rs index 09849d2f3..c12cfb28b 100644 --- a/crates/sail-function/src/scalar/datetime/spark_timestamp.rs +++ b/crates/sail-function/src/scalar/datetime/spark_timestamp.rs @@ -86,7 +86,6 @@ impl TimestampParser { let parsed = match format.parse_datetime_value(value) { Ok(v) => v, Err(_e) if is_try => return Ok(None), - Err(e) if is_invalid_leap_second(&e) => return Ok(None), Err(e) => return Err(e), }; match self { @@ -115,21 +114,7 @@ impl TimestampParser { let micros = truncate_datetime_to_microseconds(&datetime); Ok(Some(micros)) } - TimestampParser::Ntz => { - let datetime = if let Some(offset) = parsed.offset { - parsed - .datetime - .and_local_timezone(offset) - .single() - .map(|x| x.to_utc()) - .ok_or_else(|| exec_datafusion_err!("cannot apply parsed offset"))? - } else { - parsed.datetime.and_utc() - }; - // Truncate nanoseconds to microseconds to preserve fractional seconds - let micros = truncate_datetime_to_microseconds(&datetime); - Ok(Some(micros)) - } + TimestampParser::Ntz => Ok(Some(parsed.datetime.and_utc().timestamp_micros())), } } @@ -143,12 +128,6 @@ impl TimestampParser { } } -fn is_invalid_leap_second(error: &datafusion_common::DataFusionError) -> bool { - error - .to_string() - .contains("valid leap second must be 23:59:60") -} - /// Spark-compatible `to_timestamp` / `try_to_timestamp` (and their `_ntz` /// counterparts / `CAST(str AS TIMESTAMP[_NTZ])`). /// @@ -378,7 +357,7 @@ impl ScalarUDFImpl for SparkTimestamp { fn parse_scalar_format(format: Option) -> Result { match format { Some(ColumnarValue::Scalar(scalar)) => match scalar.try_as_str() { - Some(Some(format)) => Ok(ScalarFormat::Format(DateTimeFormat::parse(format)?)), + Some(Some(format)) => Ok(ScalarFormat::Format(DateTimeFormat::for_parsing(format)?)), Some(None) => Ok(ScalarFormat::Null), None => exec_err!("spark_timestamp format argument must be a string scalar"), }, @@ -472,6 +451,6 @@ fn get_or_parse_format<'a>( ) -> Result<&'a DateTimeFormat> { match cache.entry(pattern.to_string()) { Entry::Occupied(entry) => Ok(entry.into_mut()), - Entry::Vacant(entry) => Ok(entry.insert(DateTimeFormat::parse(pattern)?)), + Entry::Vacant(entry) => Ok(entry.insert(DateTimeFormat::for_parsing(pattern)?)), } } diff --git a/crates/sail-function/src/scalar/datetime/spark_to_chrono_fmt.rs b/crates/sail-function/src/scalar/datetime/spark_to_chrono_fmt.rs deleted file mode 100644 index 061145e40..000000000 --- a/crates/sail-function/src/scalar/datetime/spark_to_chrono_fmt.rs +++ /dev/null @@ -1,86 +0,0 @@ -use std::sync::Arc; - -use datafusion::arrow::array::StringArray; -use datafusion::arrow::datatypes::DataType; -use datafusion_common::cast::{as_large_string_array, as_string_array, as_string_view_array}; -use datafusion_common::types::logical_string; -use datafusion_common::{Result, ScalarValue, exec_err}; -use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility}; -use datafusion_expr_common::signature::{Coercion, TypeSignatureClass}; -use sail_common_datafusion::utils::items::ItemTaker; - -use crate::scalar::datetime::utils::spark_datetime_format_to_chrono_strftime; - -#[derive(Debug, PartialEq, Eq, Hash)] -pub struct SparkToChronoFmt { - signature: Signature, -} - -impl Default for SparkToChronoFmt { - fn default() -> Self { - Self::new() - } -} - -impl SparkToChronoFmt { - pub fn new() -> Self { - Self { - signature: Signature::coercible( - vec![Coercion::new_exact(TypeSignatureClass::Native( - logical_string(), - ))], - Volatility::Immutable, - ), - } - } -} - -impl ScalarUDFImpl for SparkToChronoFmt { - fn name(&self) -> &str { - "spark_to_chrono_fmt" - } - - fn signature(&self) -> &Signature { - &self.signature - } - - fn return_type(&self, _arg_types: &[DataType]) -> Result { - Ok(DataType::Utf8) - } - - fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { - let ScalarFunctionArgs { args, .. } = args; - let arg = args.one()?; - match arg { - ColumnarValue::Array(array) => { - let array = match array.data_type() { - DataType::Utf8 => as_string_array(&array)? - .iter() - .map(|x| x.map(spark_datetime_format_to_chrono_strftime).transpose()) - .collect::>()?, - DataType::LargeUtf8 => as_large_string_array(&array)? - .iter() - .map(|x| x.map(spark_datetime_format_to_chrono_strftime).transpose()) - .collect::>()?, - DataType::Utf8View => as_string_view_array(&array)? - .iter() - .map(|x| x.map(spark_datetime_format_to_chrono_strftime).transpose()) - .collect::>()?, - _ => return exec_err!("expected string array for `spark_to_chrono_fmt`"), - }; - Ok(ColumnarValue::Array(Arc::new(array))) - } - ColumnarValue::Scalar(scalar) => { - let value = match scalar.try_as_str() { - Some(x) => x - .map(spark_datetime_format_to_chrono_strftime) - .transpose()?, - _ => { - return exec_err!("expected string scalar for `spark_to_chrono_fmt`"); - } - }; - Ok(ColumnarValue::Scalar(ScalarValue::Utf8(value))) - } - } - } -} diff --git a/crates/sail-function/src/scalar/datetime/spark_unix_timestamp.rs b/crates/sail-function/src/scalar/datetime/spark_unix_timestamp.rs index 692beb320..143715e39 100644 --- a/crates/sail-function/src/scalar/datetime/spark_unix_timestamp.rs +++ b/crates/sail-function/src/scalar/datetime/spark_unix_timestamp.rs @@ -1,36 +1,53 @@ use std::collections::HashMap; use std::collections::hash_map::Entry; +use std::fmt::Display; use std::sync::Arc; use datafusion::arrow::array::timezone::Tz; -use datafusion::arrow::array::{Array, PrimitiveArray}; -use datafusion::arrow::datatypes::{DataType, TimeUnit}; -use datafusion::functions::datetime::to_timestamp::ToTimestampSecondsFunc; -use datafusion_common::cast::{as_large_string_array, as_string_array, as_string_view_array}; -use datafusion_common::{Result, ScalarValue, exec_datafusion_err, exec_err}; -use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility}; +use datafusion::arrow::array::{Array, PrimitiveArray, new_null_array}; +use datafusion::arrow::datatypes::{DataType, Date32Type, Field, FieldRef, Int64Type, TimeUnit}; +use datafusion_common::cast::{ + as_date32_array, as_large_string_array, as_string_array, as_string_view_array, +}; +use datafusion_common::{DataFusionError, Result, ScalarValue, exec_datafusion_err, exec_err}; +use datafusion_expr::{ + ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, +}; use sail_common_datafusion::utils::datetime::localize_with_fallback; +use crate::error::{invalid_arg_count_exec_err, unsupported_data_type_exec_err}; use crate::scalar::datetime::format::DateTimeFormat; -use crate::scalar::datetime::utils::validate_data_types; + +const DEFAULT_PATTERN: &str = "yyyy-MM-dd HH:mm:ss"; + +enum ScalarUnixFormat { + Null, + Format(DateTimeFormat), +} #[derive(Debug, PartialEq, Eq, Hash)] pub struct SparkUnixTimestamp { signature: Signature, session_timezone: Arc, + ansi_mode: bool, } impl SparkUnixTimestamp { - pub fn new(session_timezone: Arc) -> Self { + pub fn new(session_timezone: Arc, ansi_mode: bool) -> Self { Self { - signature: Signature::variadic_any(Volatility::Immutable), + signature: Signature::user_defined(Volatility::Immutable), session_timezone, + ansi_mode, } } pub fn session_timezone(&self) -> &str { &self.session_timezone } + + pub fn ansi_mode(&self) -> bool { + self.ansi_mode + } } impl ScalarUDFImpl for SparkUnixTimestamp { @@ -46,6 +63,49 @@ impl ScalarUDFImpl for SparkUnixTimestamp { Ok(DataType::Int64) } + fn coerce_types(&self, arg_types: &[DataType]) -> Result> { + if !matches!(arg_types.len(), 1 | 2) { + return Err(invalid_arg_count_exec_err( + self.name(), + (1, 2), + arg_types.len(), + )); + } + match &arg_types[0] { + DataType::Utf8 + | DataType::LargeUtf8 + | DataType::Utf8View + | DataType::Date32 + | DataType::Date64 + | DataType::Timestamp(_, _) + | DataType::Null => {} + other => { + return Err(unsupported_data_type_exec_err( + self.name(), + "STRING, DATE, TIMESTAMP or NULL", + other, + )); + } + } + + let mut coerced = arg_types.to_vec(); + if let Some(format) = arg_types.get(1) { + match format { + DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View => {} + DataType::Null => coerced[1] = DataType::Utf8, + other => { + return Err(unsupported_data_type_exec_err(self.name(), "STRING", other)); + } + } + } + Ok(coerced) + } + + fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { + let nullable = !self.ansi_mode || args.arg_fields.iter().any(|field| field.is_nullable()); + Ok(Arc::new(Field::new(self.name(), DataType::Int64, nullable))) + } + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { let [first, ..] = args.args.as_slice() else { return exec_err!("spark_unix_timestamp function requires 1 or more arguments"); @@ -55,36 +115,21 @@ impl ScalarUDFImpl for SparkUnixTimestamp { 2 => Some(&args.args[1]), _ => return exec_err!("spark_unix_timestamp function requires 1 or 2 arguments"), }; - if format.is_none() { - validate_data_types(&args.args, "spark_unix_timestamp", 1)?; - } - - if let Some(format) = format { - return self.invoke_with_format(first, format); - } - + let safe = !self.ansi_mode; match first.data_type() { - DataType::Int32 | DataType::Int64 => args.args[0] - .cast_to( - &DataType::Timestamp(TimeUnit::Second, Some(self.session_timezone.clone())), - None, - )? + DataType::Utf8View | DataType::LargeUtf8 | DataType::Utf8 => match format { + Some(format) => self.invoke_with_format(first, format, safe), + None => self.invoke_with_scalar_format( + first, + ScalarUnixFormat::Format(DateTimeFormat::for_parsing(DEFAULT_PATTERN)?), + safe, + ), + }, + DataType::Date64 | DataType::Date32 => self.invoke_with_date(first), + DataType::Timestamp(_, timezone) => first + .cast_to(&DataType::Timestamp(TimeUnit::Second, timezone), None)? .cast_to(&DataType::Int64, None), - DataType::Date64 | DataType::Date32 | DataType::Timestamp(_, _) => args.args[0] - .cast_to( - &DataType::Timestamp(TimeUnit::Second, Some(self.session_timezone.clone())), - None, - )? - .cast_to(&DataType::Int64, None), - DataType::Utf8View | DataType::LargeUtf8 | DataType::Utf8 => { - ToTimestampSecondsFunc::new_with_config(args.config_options.as_ref()) - .invoke_with_args(args)? - .cast_to( - &DataType::Timestamp(TimeUnit::Second, Some(self.session_timezone.clone())), - None, - )? - .cast_to(&DataType::Int64, None) - } + DataType::Null => null_int64(first), other => { exec_err!("spark_unix_timestamp function unsupported data type: {other}") } @@ -97,35 +142,67 @@ impl SparkUnixTimestamp { &self, first: &ColumnarValue, format: &ColumnarValue, + safe: bool, ) -> Result { - match (first, format) { - (ColumnarValue::Array(array), ColumnarValue::Scalar(format_scalar)) => { - let format = parse_format_scalar(format_scalar)?; - let array: PrimitiveArray = match array - .data_type() - { + match format { + ColumnarValue::Scalar(format_scalar) => { + self.invoke_with_scalar_format(first, parse_format_scalar(format_scalar)?, safe) + } + ColumnarValue::Array(format_array) => match first { + ColumnarValue::Array(array) => { + self.invoke_array_with_format_array(array, format_array, safe) + } + ColumnarValue::Scalar(scalar) => { + let arrays = ColumnarValue::values_to_arrays(&[ + ColumnarValue::Scalar(scalar.clone()), + ColumnarValue::Array(format_array.clone()), + ])?; + self.invoke_array_with_format_array(&arrays[0], &arrays[1], safe) + } + }, + } + } + + fn invoke_with_scalar_format( + &self, + first: &ColumnarValue, + format: ScalarUnixFormat, + safe: bool, + ) -> Result { + match first { + ColumnarValue::Array(array) => { + let ScalarUnixFormat::Format(format) = format else { + return Ok(ColumnarValue::Array(new_null_array( + &DataType::Int64, + array.len(), + ))); + }; + let array: PrimitiveArray = match array.data_type() { DataType::Utf8 => as_string_array(array)? .iter() - .map(|x| { - x.map(|value| self.formatted_string_to_seconds(value, &format)) + .map(|value| { + value + .map(|value| self.formatted_string_to_seconds(value, &format, safe)) .transpose() - .map(|opt| opt.flatten()) + .map(Option::flatten) }) .collect::>()?, DataType::LargeUtf8 => as_large_string_array(array)? .iter() - .map(|x| { - x.map(|value| self.formatted_string_to_seconds(value, &format)) + .map(|value| { + value + .map(|value| self.formatted_string_to_seconds(value, &format, safe)) .transpose() - .map(|opt| opt.flatten()) + .map(Option::flatten) }) .collect::>()?, DataType::Utf8View => as_string_view_array(array)? .iter() - .map(|x| { - x.map(|value| self.formatted_string_to_seconds(value, &format)) + .map(|value| { + value + .map(|value| self.formatted_string_to_seconds(value, &format, safe)) .transpose() - .map(|opt| opt.flatten()) + .map(Option::flatten) }) .collect::>()?, other => { @@ -136,17 +213,16 @@ impl SparkUnixTimestamp { }; Ok(ColumnarValue::Array(Arc::new(array))) } - (ColumnarValue::Array(array), ColumnarValue::Array(format_array)) => { - self.invoke_array_with_format_array(array, format_array) - } - (ColumnarValue::Scalar(scalar), ColumnarValue::Scalar(format_scalar)) => { - let format = parse_format_scalar(format_scalar)?; + ColumnarValue::Scalar(scalar) => { + let ScalarUnixFormat::Format(format) = format else { + return Ok(ColumnarValue::Scalar(ScalarValue::Int64(None))); + }; let value = match scalar.try_as_str() { Some(value) => value - .map(|value| self.formatted_string_to_seconds(value, &format)) + .map(|value| self.formatted_string_to_seconds(value, &format, safe)) .transpose()? .flatten(), - _ => { + None => { return exec_err!( "spark_unix_timestamp function expected string scalar for formatted input" ); @@ -154,15 +230,33 @@ impl SparkUnixTimestamp { }; Ok(ColumnarValue::Scalar(ScalarValue::Int64(value))) } - (ColumnarValue::Scalar(scalar), ColumnarValue::Array(format_array)) => { - let arrays = ColumnarValue::values_to_arrays(&[ - ColumnarValue::Scalar(scalar.clone()), - ColumnarValue::Array(format_array.clone()), - ])?; - let array = arrays[0].clone(); - let format_array = arrays[1].clone(); - self.invoke_array_with_format_array(&array, &format_array) + } + } + + fn invoke_with_date(&self, first: &ColumnarValue) -> Result { + let timezone: Tz = self.session_timezone.parse()?; + match first.cast_to(&DataType::Date32, None)? { + ColumnarValue::Array(array) => { + let dates = as_date32_array(&array)?; + let seconds: PrimitiveArray = dates + .iter() + .map(|days| { + days.map(|days| date32_to_seconds(days, &timezone)) + .transpose() + }) + .collect::>()?; + Ok(ColumnarValue::Array(Arc::new(seconds))) } + ColumnarValue::Scalar(ScalarValue::Date32(days)) => { + let seconds = days + .map(|days| date32_to_seconds(days, &timezone)) + .transpose()?; + Ok(ColumnarValue::Scalar(ScalarValue::Int64(seconds))) + } + other => exec_err!( + "spark_unix_timestamp expected date input after coercion, got {}", + other.data_type() + ), } } @@ -170,6 +264,7 @@ impl SparkUnixTimestamp { &self, array: &dyn datafusion::arrow::array::Array, format_array: &dyn datafusion::arrow::array::Array, + safe: bool, ) -> Result { if array.len() != format_array.len() { return exec_err!( @@ -180,15 +275,15 @@ impl SparkUnixTimestamp { let array = match format_array.data_type() { DataType::Utf8 => { let formats = as_string_array(format_array)?; - self.parse_array_with_formats(array, formats.iter(), &mut cache)? + self.parse_array_with_formats(array, formats.iter(), &mut cache, safe)? } DataType::LargeUtf8 => { let formats = as_large_string_array(format_array)?; - self.parse_array_with_formats(array, formats.iter(), &mut cache)? + self.parse_array_with_formats(array, formats.iter(), &mut cache, safe)? } DataType::Utf8View => { let formats = as_string_view_array(format_array)?; - self.parse_array_with_formats(array, formats.iter(), &mut cache)? + self.parse_array_with_formats(array, formats.iter(), &mut cache, safe)? } _ => return exec_err!("spark_unix_timestamp format argument must be a string array"), }; @@ -200,17 +295,24 @@ impl SparkUnixTimestamp { array: &dyn datafusion::arrow::array::Array, formats: impl Iterator>, cache: &mut HashMap, - ) -> Result> { + safe: bool, + ) -> Result> { match array.data_type() { DataType::Utf8 => { - self.parse_values_with_formats(as_string_array(array)?.iter(), formats, cache) - } - DataType::LargeUtf8 => { - self.parse_values_with_formats(as_large_string_array(array)?.iter(), formats, cache) - } - DataType::Utf8View => { - self.parse_values_with_formats(as_string_view_array(array)?.iter(), formats, cache) + self.parse_values_with_formats(as_string_array(array)?.iter(), formats, cache, safe) } + DataType::LargeUtf8 => self.parse_values_with_formats( + as_large_string_array(array)?.iter(), + formats, + cache, + safe, + ), + DataType::Utf8View => self.parse_values_with_formats( + as_string_view_array(array)?.iter(), + formats, + cache, + safe, + ), other => exec_err!( "spark_unix_timestamp function unsupported formatted input data type: {other}" ), @@ -222,13 +324,14 @@ impl SparkUnixTimestamp { values: impl Iterator>, formats: impl Iterator>, cache: &mut HashMap, - ) -> Result> { + safe: bool, + ) -> Result> { values .zip(formats) .map(|(value, format)| match (value, format) { (Some(value), Some(format)) => { let format = get_or_parse_format(cache, format)?; - self.formatted_string_to_seconds(value, format) + self.formatted_string_to_seconds(value, format, safe) } _ => Ok(None), }) @@ -239,34 +342,84 @@ impl SparkUnixTimestamp { &self, value: &str, format: &DateTimeFormat, + safe: bool, ) -> Result> { - let parsed = format.parse_datetime_value(value)?; + let Some(parsed) = timestamp_parse_result(format.parse_datetime_value(value), safe)? else { + return Ok(None); + }; let timestamp = if let Some(offset) = parsed.offset { - parsed + let localized = parsed .datetime .and_local_timezone(offset) .single() - .map(|x| x.to_utc().timestamp()) - .ok_or_else(|| exec_datafusion_err!("cannot apply parsed offset"))? + .ok_or_else(|| exec_datafusion_err!("cannot apply parsed offset")); + let Some(localized) = timestamp_parse_result(localized, safe)? else { + return Ok(None); + }; + localized.to_utc().timestamp() } else { - let timezone: Tz = parsed + let timezone = parsed .timezone .as_deref() .unwrap_or(&self.session_timezone) - .parse()?; - localize_with_fallback(&timezone, &parsed.datetime)?.timestamp() + .parse::(); + let Some(timezone) = timestamp_parse_result(timezone, safe)? else { + return Ok(None); + }; + let Some(localized) = + timestamp_parse_result(localize_with_fallback(&timezone, &parsed.datetime), safe)? + else { + return Ok(None); + }; + localized.timestamp() }; Ok(Some(timestamp)) } } -fn parse_format_scalar(scalar: &ScalarValue) -> Result { - scalar - .try_as_str() - .flatten() - .map(DateTimeFormat::parse) - .transpose()? - .ok_or_else(|| exec_datafusion_err!("spark_unix_timestamp format cannot be null")) +fn parse_format_scalar(scalar: &ScalarValue) -> Result { + match scalar.try_as_str() { + Some(Some(pattern)) => Ok(ScalarUnixFormat::Format(DateTimeFormat::for_parsing( + pattern, + )?)), + Some(None) => Ok(ScalarUnixFormat::Null), + None => exec_err!("spark_unix_timestamp format argument must be a string scalar"), + } +} + +fn null_int64(value: &ColumnarValue) -> Result { + match value { + ColumnarValue::Array(array) => Ok(ColumnarValue::Array(new_null_array( + &DataType::Int64, + array.len(), + ))), + ColumnarValue::Scalar(_) => Ok(ColumnarValue::Scalar(ScalarValue::Int64(None))), + } +} + +fn date32_to_seconds(days: i32, timezone: &Tz) -> Result { + let date = Date32Type::to_naive_date_opt(days).ok_or_else(|| { + exec_datafusion_err!("spark_unix_timestamp cannot convert date value {days}") + })?; + let datetime = date.and_hms_opt(0, 0, 0).ok_or_else(|| { + exec_datafusion_err!("spark_unix_timestamp cannot construct midnight for {date}") + })?; + Ok(localize_with_fallback(timezone, &datetime)?.timestamp()) +} + +fn timestamp_parse_result(result: std::result::Result, safe: bool) -> Result> +where + E: Display, +{ + match result { + Ok(value) => Ok(Some(value)), + Err(_) if safe => Ok(None), + Err(error) => Err(timestamp_parse_error(error)), + } +} + +fn timestamp_parse_error(error: impl Display) -> DataFusionError { + exec_datafusion_err!("Error parsing timestamp: [CANNOT_PARSE_TIMESTAMP] {error}") } fn get_or_parse_format<'a>( @@ -275,6 +428,6 @@ fn get_or_parse_format<'a>( ) -> Result<&'a DateTimeFormat> { match cache.entry(pattern.to_string()) { Entry::Occupied(entry) => Ok(entry.into_mut()), - Entry::Vacant(entry) => Ok(entry.insert(DateTimeFormat::parse(pattern)?)), + Entry::Vacant(entry) => Ok(entry.insert(DateTimeFormat::for_parsing(pattern)?)), } } diff --git a/crates/sail-function/src/scalar/datetime/utils.rs b/crates/sail-function/src/scalar/datetime/utils.rs index 9aa543829..47fe23a16 100644 --- a/crates/sail-function/src/scalar/datetime/utils.rs +++ b/crates/sail-function/src/scalar/datetime/utils.rs @@ -1,159 +1,7 @@ use datafusion::arrow::array::types::{Decimal128Type, Int32Type, Time64MicrosecondType}; use datafusion::arrow::array::{AsArray, Int32Array, PrimitiveArray}; -use datafusion::arrow::datatypes::DataType; use datafusion_common::{Result, ScalarValue, exec_err}; use datafusion_expr::ColumnarValue; -use lazy_static::lazy_static; -use regex::Regex; - -/// [Credit]: -pub(crate) fn validate_data_types(args: &[ColumnarValue], name: &str, skip: usize) -> Result<()> { - for (idx, a) in args.iter().skip(skip).enumerate() { - match a.data_type() { - DataType::Utf8View | DataType::LargeUtf8 | DataType::Utf8 => { - // all good - } - _ => { - return exec_err!( - "{name} function unsupported data type at index {}: {}", - idx + 1, - a.data_type() - ); - } - } - } - - Ok(()) -} - -lazy_static! { - /// Precomputed (regex, replacement) pairs for converting Spark datetime - /// pattern strings to chrono `strftime` format. - /// - /// The `(?P
      ^|[^%])` prefix in each regex prevents re-matching a
-    /// letter that is immediately preceded by `%`. This protects the
-    /// canonical chrono directives (`%Y`, `%m`, `%H`, ...) from being
-    /// re-substituted as Spark tokens. It does NOT protect letters produced
-    /// via the short `%-X` forms (`%-m`, `%-d`, `%-H`, ...), where the
-    /// meaningful letter sits after a `-` and remains eligible for later
-    /// matching.
-    ///
-    /// IMPORTANT: pattern order is load-bearing. Longer patterns within each
-    /// family (e.g., `SSSSSSSSS` before `S`, `MMMM` before `M`) MUST come
-    /// first because `replace_all` is applied sequentially per regex.
-    /// Do not alphabetize.
-    static ref CHRONO_REPLACEMENTS: Vec<(Regex, String)> = {
-        let patterns: &[(&str, &str)] = &[
-            // Fractional seconds patterns (from nanoseconds to deciseconds)
-            ("SSSSSSSSS", "%.9f"), // Nanoseconds
-            ("SSSSSSSS", "%.8f"),
-            ("SSSSSSS", "%.7f"),
-            ("SSSSSS", "%.6f"), // Microseconds
-            ("SSSSS", "%.5f"),
-            ("SSSS", "%.4f"),
-            ("SSS", "%.3f"), // Milliseconds
-            ("SS", "%.2f"),  // Centiseconds
-            ("S", "%.1f"),   // Deciseconds
-            // Year patterns
-            ("yyyy", "%Y"),
-            ("yyy", "%Y"),
-            ("yy", "%y"),
-            ("y", "%Y"),
-            // Day-of-year pattern
-            ("D", "%j"),
-            // Month patterns
-            ("MMMM", "%B"),
-            ("MMM", "%b"),
-            ("MM", "%m"),
-            ("M", "%-m"),
-            ("LLLL", "%B"),
-            ("LLL", "%b"),
-            ("LL", "%m"),
-            ("L", "%-m"),
-            // Day-of-month patterns
-            ("dd", "%d"),
-            ("d", "%-d"),
-            // Weekday patterns
-            ("EEEE", "%A"),
-            ("EEE", "%a"),
-            ("E", "%a"),
-            // Hour patterns
-            ("hh", "%I"), // 12-hour clock (01–12)
-            ("h", "%-I"), // 12-hour clock (1–12)
-            ("HH", "%H"), // 24-hour clock (00–23)
-            ("H", "%-H"), // 24-hour clock (0–23)
-            ("KK", "%I"), // 12-hour clock (01–12), but Spark's 'K' is 0–11
-            ("K", "%l"),  // 12-hour clock (1–12), space-padded
-            // Minute patterns
-            ("mm", "%M"),
-            ("m", "%-M"),
-            // Second patterns
-            ("ss", "%S"),
-            ("s", "%-S"),
-            // AM/PM
-            ("a", "%p"),
-            // Timezone patterns
-            ("XXXXX", "%::z"), // ±HH:MM:SS
-            ("XXXX", "%z"),    // ±HHMM
-            ("XXX", "%:z"),    // ±HH:MM
-            ("XX", "%z"),      // ±HHMM
-            ("X", "%z"),       // ±HHMM
-            ("xxxxx", "%::z"), // ±HH:MM:SS
-            ("xxxx", "%z"),    // ±HHMM
-            ("xxx", "%:z"),    // ±HH:MM
-            ("xx", "%z"),      // ±HHMM
-            ("x", "%z"),       // ±HHMM
-            ("ZZZZZ", "%::z"), // ±HH:MM:SS
-            ("ZZZZ", "%:z"),   // ±HH:MM
-            ("ZZZ", "%z"),     // ±HHMM
-            ("ZZ", "%z"),      // ±HHMM
-            ("Z", "%z"),       // ±HHMM
-            ("zzzz", "%Z"),
-            ("zzz", "%Z"),
-            ("zz", "%Z"),
-            ("z", "%Z"),
-            ("OOOO", "%Z"),
-            ("OO", "%Z"),
-            ("VV", "%Z"),
-        ];
-        patterns
-            .iter()
-            .map(|(pattern, replacement)| {
-                let pattern = format!(r"(?P
      ^|[^%]){}", regex::escape(pattern));
-                #[expect(clippy::expect_used)]
-                let re = Regex::new(&pattern).expect("valid datetime pattern");
-                // Precompute the FULL replacement string including the
-                // `${pre}` capture group reference to avoid per-call allocation.
-                let replacement = format!("${{pre}}{replacement}");
-                (re, replacement)
-            })
-            .collect()
-    };
-}
-
-pub fn spark_datetime_format_to_chrono_strftime(format: &str) -> Result {
-    // TODO: This doesn't cover everything.
-    //  https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
-    //  https://docs.rs/chrono/latest/chrono/format/strftime/index.html#specifiers
-
-    let mut result = format.to_string();
-    for (re, replacement) in CHRONO_REPLACEMENTS.iter() {
-        // `replace_all` returns `Cow::Borrowed` when nothing matched; in that
-        // case the input is unchanged and there is no need to allocate a new
-        // `String`. Only reassign `result` when an actual substitution
-        // happened (`Cow::Owned`).
-        if let std::borrow::Cow::Owned(x) = re.replace_all(&result, replacement.as_str()) {
-            result = x;
-        }
-    }
-
-    // Fix double-dot issue: chrono's %.Nf already includes a leading dot,
-    // so when the Spark format has a literal '.' before S-patterns (e.g., "ss.SSS"),
-    // the result would have ".%.Nf" which produces "..NNN". Remove the extra dot.
-    result = result.replace(".%.", "%.");
-
-    Ok(result)
-}
 
 // Shared array conversion helpers for make_timestamp functions
 
diff --git a/crates/sail-function/src/scalar/json/from_json.rs b/crates/sail-function/src/scalar/json/from_json.rs
index 66f8334b9..af07c632f 100644
--- a/crates/sail-function/src/scalar/json/from_json.rs
+++ b/crates/sail-function/src/scalar/json/from_json.rs
@@ -61,21 +61,21 @@ impl SparkFromJsonOptions {
     fn from_map(map: &MapArray) -> Result {
         let timestamp_format = find_key_value(map, Self::TIMESTAMP_FORMAT_OPTION)
             .as_deref()
-            .map(DateTimeFormat::parse)
+            .map(DateTimeFormat::for_parsing)
             .transpose()?
             .unwrap_or_else(|| {
                 #[expect(clippy::expect_used)]
-                DateTimeFormat::parse(Self::TIMESTAMP_FORMAT_DEFAULT)
+                DateTimeFormat::for_parsing(Self::TIMESTAMP_FORMAT_DEFAULT)
                     .expect("default timestamp format should be valid")
             });
 
         let date_format = find_key_value(map, Self::DATE_FORMAT_OPTION)
             .as_deref()
-            .map(DateTimeFormat::parse)
+            .map(DateTimeFormat::for_parsing)
             .transpose()?
             .unwrap_or_else(|| {
                 #[expect(clippy::expect_used)]
-                DateTimeFormat::parse(Self::DATE_FORMAT_DEFAULT)
+                DateTimeFormat::for_parsing(Self::DATE_FORMAT_DEFAULT)
                     .expect("default date format should be valid")
             });
 
@@ -90,9 +90,9 @@ impl Default for SparkFromJsonOptions {
     #[expect(clippy::expect_used)]
     fn default() -> Self {
         Self {
-            timestamp_format: DateTimeFormat::parse(Self::TIMESTAMP_FORMAT_DEFAULT)
+            timestamp_format: DateTimeFormat::for_parsing(Self::TIMESTAMP_FORMAT_DEFAULT)
                 .expect("default timestamp format should be valid"),
-            date_format: DateTimeFormat::parse(Self::DATE_FORMAT_DEFAULT)
+            date_format: DateTimeFormat::for_parsing(Self::DATE_FORMAT_DEFAULT)
                 .expect("default date format should be valid"),
         }
     }
diff --git a/crates/sail-function/src/scalar/json/to_json.rs b/crates/sail-function/src/scalar/json/to_json.rs
index 20138412c..e63c58ad5 100644
--- a/crates/sail-function/src/scalar/json/to_json.rs
+++ b/crates/sail-function/src/scalar/json/to_json.rs
@@ -57,21 +57,21 @@ impl ToJsonOptions {
     fn from_map(map: &MapArray) -> Result {
         let timestamp_format = find_key_value(map, Self::TIMESTAMP_FORMAT_OPTION)
             .as_deref()
-            .map(DateTimeFormat::parse)
+            .map(DateTimeFormat::for_formatting)
             .transpose()?
             .unwrap_or_else(|| {
                 #[expect(clippy::expect_used)]
-                DateTimeFormat::parse(Self::TIMESTAMP_FORMAT_DEFAULT)
+                DateTimeFormat::for_formatting(Self::TIMESTAMP_FORMAT_DEFAULT)
                     .expect("default timestamp format should be valid")
             });
 
         let date_format = find_key_value(map, Self::DATE_FORMAT_OPTION)
             .as_deref()
-            .map(DateTimeFormat::parse)
+            .map(DateTimeFormat::for_formatting)
             .transpose()?
             .unwrap_or_else(|| {
                 #[expect(clippy::expect_used)]
-                DateTimeFormat::parse(Self::DATE_FORMAT_DEFAULT)
+                DateTimeFormat::for_formatting(Self::DATE_FORMAT_DEFAULT)
                     .expect("default date format should be valid")
             });
 
@@ -86,9 +86,9 @@ impl Default for ToJsonOptions {
     #[expect(clippy::expect_used)]
     fn default() -> Self {
         Self {
-            timestamp_format: DateTimeFormat::parse(Self::TIMESTAMP_FORMAT_DEFAULT)
+            timestamp_format: DateTimeFormat::for_formatting(Self::TIMESTAMP_FORMAT_DEFAULT)
                 .expect("default timestamp format should be valid"),
-            date_format: DateTimeFormat::parse(Self::DATE_FORMAT_DEFAULT)
+            date_format: DateTimeFormat::for_formatting(Self::DATE_FORMAT_DEFAULT)
                 .expect("default date format should be valid"),
         }
     }
diff --git a/crates/sail-function/src/scalar/xml/from_xml.rs b/crates/sail-function/src/scalar/xml/from_xml.rs
index e55c07c56..4df75855a 100644
--- a/crates/sail-function/src/scalar/xml/from_xml.rs
+++ b/crates/sail-function/src/scalar/xml/from_xml.rs
@@ -1,6 +1,5 @@
 use std::sync::Arc;
 
-use chrono::{NaiveDate, NaiveDateTime};
 use datafusion::arrow::array::timezone::Tz;
 use datafusion::arrow::array::*;
 use datafusion::arrow::buffer::{NullBuffer, OffsetBuffer, ScalarBuffer};
@@ -20,7 +19,7 @@ use sail_sql_analyzer::parser as sail_parser;
 use xee_xpath::Documents;
 
 use crate::functions_utils::make_scalar_function;
-use crate::scalar::datetime::utils::spark_datetime_format_to_chrono_strftime;
+use crate::scalar::datetime::format::{DateTimeFormat, ParsedDateTime};
 
 #[cfg(test)]
 const DEFAULT_SESSION_TIMEZONE: &str = "UTC";
@@ -60,9 +59,9 @@ struct SparkFromXmlOptions {
     null_value: Option,
     attribute_prefix: String,
     value_tag: String,
-    timestamp_ltz_format: String,
-    timestamp_ntz_format: String,
-    date_format: String,
+    timestamp_ltz_format: DateTimeFormat,
+    timestamp_ntz_format: DateTimeFormat,
+    date_format: DateTimeFormat,
     mode: ParseMode,
 }
 
@@ -76,9 +75,9 @@ impl SparkFromXmlOptions {
     const TIMESTAMP_NTZ_FORMAT_OPTION: &'static str = "timestampNTZFormat";
     const DATE_FORMAT_OPTION: &'static str = "dateFormat";
     const MODE_OPTION: &'static str = "mode";
-    const TIMESTAMP_LTZ_FORMAT_DEFAULT: &'static str = "%Y-%m-%dT%H:%M:%S%.3f";
-    const TIMESTAMP_NTZ_FORMAT_DEFAULT: &'static str = "%Y-%m-%dT%H:%M:%S%.3f";
-    const DATE_FORMAT_DEFAULT: &'static str = "%Y-%m-%d";
+    const TIMESTAMP_LTZ_FORMAT_DEFAULT: &'static str = "yyyy-MM-dd'T'HH:mm:ss.SSS";
+    const TIMESTAMP_NTZ_FORMAT_DEFAULT: &'static str = "yyyy-MM-dd'T'HH:mm:ss.SSS";
+    const DATE_FORMAT_DEFAULT: &'static str = "yyyy-MM-dd";
 
     fn from_map(map: &MapArray) -> Result {
         let null_value = find_key_value(map, Self::NULL_VALUE_OPTION);
@@ -94,19 +93,31 @@ impl SparkFromXmlOptions {
         }
         let timestamp_ltz_format = find_key_value(map, Self::TIMESTAMP_FORMAT_OPTION)
             .as_deref()
-            .map(spark_datetime_format_to_chrono_strftime)
+            .map(DateTimeFormat::for_parsing)
             .transpose()?
-            .unwrap_or_else(|| Self::TIMESTAMP_LTZ_FORMAT_DEFAULT.to_string());
+            .unwrap_or_else(|| {
+                #[expect(clippy::expect_used)]
+                DateTimeFormat::for_parsing(Self::TIMESTAMP_LTZ_FORMAT_DEFAULT)
+                    .expect("default timestamp LTZ format should be valid")
+            });
         let timestamp_ntz_format = find_key_value(map, Self::TIMESTAMP_NTZ_FORMAT_OPTION)
             .as_deref()
-            .map(spark_datetime_format_to_chrono_strftime)
+            .map(DateTimeFormat::for_parsing)
             .transpose()?
-            .unwrap_or_else(|| Self::TIMESTAMP_NTZ_FORMAT_DEFAULT.to_string());
+            .unwrap_or_else(|| {
+                #[expect(clippy::expect_used)]
+                DateTimeFormat::for_parsing(Self::TIMESTAMP_NTZ_FORMAT_DEFAULT)
+                    .expect("default timestamp NTZ format should be valid")
+            });
         let date_format = find_key_value(map, Self::DATE_FORMAT_OPTION)
             .as_deref()
-            .map(spark_datetime_format_to_chrono_strftime)
+            .map(DateTimeFormat::for_parsing)
             .transpose()?
-            .unwrap_or_else(|| Self::DATE_FORMAT_DEFAULT.to_string());
+            .unwrap_or_else(|| {
+                #[expect(clippy::expect_used)]
+                DateTimeFormat::for_parsing(Self::DATE_FORMAT_DEFAULT)
+                    .expect("default date format should be valid")
+            });
         let mode = match find_key_value(map, Self::MODE_OPTION)
             .as_deref()
             .map(|s| s.to_ascii_uppercase())
@@ -136,9 +147,21 @@ impl Default for SparkFromXmlOptions {
             null_value: None,
             attribute_prefix: Self::ATTRIBUTE_PREFIX_DEFAULT.to_string(),
             value_tag: Self::VALUE_TAG_DEFAULT.to_string(),
-            timestamp_ltz_format: Self::TIMESTAMP_LTZ_FORMAT_DEFAULT.to_string(),
-            timestamp_ntz_format: Self::TIMESTAMP_NTZ_FORMAT_DEFAULT.to_string(),
-            date_format: Self::DATE_FORMAT_DEFAULT.to_string(),
+            timestamp_ltz_format: {
+                #[expect(clippy::expect_used)]
+                DateTimeFormat::for_parsing(Self::TIMESTAMP_LTZ_FORMAT_DEFAULT)
+                    .expect("default timestamp LTZ format should be valid")
+            },
+            timestamp_ntz_format: {
+                #[expect(clippy::expect_used)]
+                DateTimeFormat::for_parsing(Self::TIMESTAMP_NTZ_FORMAT_DEFAULT)
+                    .expect("default timestamp NTZ format should be valid")
+            },
+            date_format: {
+                #[expect(clippy::expect_used)]
+                DateTimeFormat::for_parsing(Self::DATE_FORMAT_DEFAULT)
+                    .expect("default date format should be valid")
+            },
             mode: ParseMode::Permissive,
         }
     }
@@ -813,7 +836,7 @@ fn append_text(
         },
 
         XmlFieldBuilder::String(b) => b.append_value(raw),
-        XmlFieldBuilder::Date32(b) => match NaiveDate::parse_from_str(raw, &options.date_format) {
+        XmlFieldBuilder::Date32(b) => match options.date_format.parse_date_value(raw) {
             Ok(d) => b.append_value(Date32Type::from_naive_date(d)),
             Err(_) => b.append_null(),
         },
@@ -823,12 +846,12 @@ fn append_text(
             } else {
                 &options.timestamp_ntz_format
             };
-            match parse_naive_datetime(raw, fmt) {
-                Some(naive) => {
-                    let micros = to_utc_micros(naive, *has_tz, session_timezone)?;
+            match fmt.parse_datetime_value(raw) {
+                Ok(parsed) => {
+                    let micros = parsed_timestamp_to_micros(parsed, *has_tz, session_timezone)?;
                     b.append_value(micros);
                 }
-                None => b.append_null(),
+                Err(_) => b.append_null(),
             }
         }
         XmlFieldBuilder::TimestampNanosecond { builder: b, has_tz } => {
@@ -837,12 +860,12 @@ fn append_text(
             } else {
                 &options.timestamp_ntz_format
             };
-            match parse_naive_datetime(raw, fmt) {
-                Some(naive) => {
-                    let micros = to_utc_micros(naive, *has_tz, session_timezone)?;
+            match fmt.parse_datetime_value(raw) {
+                Ok(parsed) => {
+                    let micros = parsed_timestamp_to_micros(parsed, *has_tz, session_timezone)?;
                     b.append_value(micros * 1_000);
                 }
-                None => b.append_null(),
+                Err(_) => b.append_null(),
             }
         }
         XmlFieldBuilder::List { .. } | XmlFieldBuilder::Struct { .. } => {
@@ -853,24 +876,29 @@ fn append_text(
     Ok(())
 }
 
-fn parse_naive_datetime(raw: &str, fmt: &str) -> Option {
-    NaiveDateTime::parse_from_str(raw, fmt).ok().or_else(|| {
-        NaiveDate::parse_from_str(raw, fmt)
-            .ok()
-            .and_then(|d| d.and_hms_opt(0, 0, 0))
-    })
-}
+fn parsed_timestamp_to_micros(
+    parsed: ParsedDateTime,
+    has_timezone: bool,
+    session_timezone: &str,
+) -> Result {
+    if !has_timezone {
+        return Ok(parsed.datetime.and_utc().timestamp_micros());
+    }
 
-fn to_utc_micros(naive: NaiveDateTime, has_tz: bool, session_timezone: &str) -> Result {
-    if has_tz {
-        let tz: Tz = session_timezone
-            .parse()
-            .map_err(|e| DataFusionError::Execution(format!("Invalid timezone: {e}")))?;
-        let localized = localize_with_fallback(&tz, &naive)?;
-        Ok(localized.timestamp_micros())
-    } else {
-        Ok(naive.and_utc().timestamp_micros())
+    if let Some(offset) = parsed.offset {
+        return parsed
+            .datetime
+            .and_local_timezone(offset)
+            .single()
+            .map(|datetime| datetime.to_utc().timestamp_micros())
+            .ok_or_else(|| DataFusionError::Execution("cannot apply parsed offset".to_string()));
     }
+
+    let timezone_name = parsed.timezone.as_deref().unwrap_or(session_timezone);
+    let timezone: Tz = timezone_name.parse().map_err(|e| {
+        DataFusionError::Execution(format!("Invalid timezone '{timezone_name}': {e}"))
+    })?;
+    Ok(localize_with_fallback(&timezone, &parsed.datetime)?.timestamp_micros())
 }
 
 fn parse_decimal128(raw: &str, precision: u8, scale: i8) -> Option {
diff --git a/crates/sail-function/src/scalar/xml/to_xml.rs b/crates/sail-function/src/scalar/xml/to_xml.rs
index 9e89d6d7e..69d949f4f 100644
--- a/crates/sail-function/src/scalar/xml/to_xml.rs
+++ b/crates/sail-function/src/scalar/xml/to_xml.rs
@@ -81,26 +81,26 @@ impl SparkToXmlOptions {
 
         let timestamp_ltz_format = find_key_value(map, Self::TIMESTAMP_FORMAT_OPTION)
             .as_deref()
-            .map(DateTimeFormat::parse)
+            .map(DateTimeFormat::for_formatting)
             .transpose()?;
 
         let timestamp_ntz_format = find_key_value(map, Self::TIMESTAMP_NTZ_FORMAT_OPTION)
             .as_deref()
-            .map(DateTimeFormat::parse)
+            .map(DateTimeFormat::for_formatting)
             .transpose()?
             .unwrap_or_else(|| {
                 #[expect(clippy::expect_used)]
-                DateTimeFormat::parse(Self::TIMESTAMP_NTZ_FORMAT_DEFAULT)
+                DateTimeFormat::for_formatting(Self::TIMESTAMP_NTZ_FORMAT_DEFAULT)
                     .expect("default timestamp NTZ format should be valid")
             });
 
         let date_format = find_key_value(map, Self::DATE_FORMAT_OPTION)
             .as_deref()
-            .map(DateTimeFormat::parse)
+            .map(DateTimeFormat::for_formatting)
             .transpose()?
             .unwrap_or_else(|| {
                 #[expect(clippy::expect_used)]
-                DateTimeFormat::parse(Self::DATE_FORMAT_DEFAULT)
+                DateTimeFormat::for_formatting(Self::DATE_FORMAT_DEFAULT)
                     .expect("default date format should be valid")
             });
 
@@ -140,9 +140,11 @@ impl Default for SparkToXmlOptions {
             value_tag: Self::VALUE_TAG_DEFAULT.to_string(),
             null_value: None,
             timestamp_ltz_format: None,
-            timestamp_ntz_format: DateTimeFormat::parse(Self::TIMESTAMP_NTZ_FORMAT_DEFAULT)
-                .expect("default timestamp NTZ format should be valid"),
-            date_format: DateTimeFormat::parse(Self::DATE_FORMAT_DEFAULT)
+            timestamp_ntz_format: DateTimeFormat::for_formatting(
+                Self::TIMESTAMP_NTZ_FORMAT_DEFAULT,
+            )
+            .expect("default timestamp NTZ format should be valid"),
+            date_format: DateTimeFormat::for_formatting(Self::DATE_FORMAT_DEFAULT)
                 .expect("default date format should be valid"),
             session_timezone: "UTC".to_string(),
         }
@@ -945,7 +947,7 @@ fn format_timestamp_field(
             // Use default format
             #[expect(clippy::expect_used)]
             let default_fmt =
-                DateTimeFormat::parse(SparkToXmlOptions::TIMESTAMP_LTZ_FORMAT_DEFAULT)
+                DateTimeFormat::for_formatting(SparkToXmlOptions::TIMESTAMP_LTZ_FORMAT_DEFAULT)
                     .expect("default timestamp LTZ format should be valid");
             Ok(default_fmt.format(input).map_err(|e| {
                 DataFusionError::Execution(format!("Failed to format timestamp: {e}"))
diff --git a/crates/sail-plan/src/function/scalar/datetime.rs b/crates/sail-plan/src/function/scalar/datetime.rs
index cce587161..2c9448a1b 100644
--- a/crates/sail-plan/src/function/scalar/datetime.rs
+++ b/crates/sail-plan/src/function/scalar/datetime.rs
@@ -26,7 +26,6 @@ use sail_function::scalar::datetime::spark_time::SparkTime;
 use sail_function::scalar::datetime::spark_time_diff::SparkTimeDiff;
 use sail_function::scalar::datetime::spark_time_trunc::SparkTimeTrunc;
 use sail_function::scalar::datetime::spark_timestamp::SparkTimestamp;
-use sail_function::scalar::datetime::spark_to_chrono_fmt::SparkToChronoFmt;
 use sail_function::scalar::datetime::spark_unix_timestamp::SparkUnixTimestamp;
 use sail_function::scalar::datetime::spark_window_buckets::SparkWindowBuckets;
 use sail_function::scalar::datetime::spark_year::SparkYear;
@@ -39,9 +38,6 @@ use crate::config::DefaultTimestampType;
 use crate::error::{PlanError, PlanResult};
 use crate::function::common::{ScalarFunction, ScalarFunctionInput};
 
-fn to_chrono_fmt(format: Expr) -> Expr {
-    ScalarUDF::from(SparkToChronoFmt::new()).call(vec![format])
-}
 fn integer_part(expr: Expr, part: &str) -> Expr {
     cast(
         expr_fn::date_part(lit(part.to_uppercase()), expr),
@@ -400,39 +396,105 @@ fn current_timezone(input: ScalarFunctionInput) -> PlanResult {
     Ok(session_tz)
 }
 
+fn coerce_datetime_format(
+    function_name: &str,
+    format: Expr,
+    schema: &DFSchemaRef,
+) -> PlanResult {
+    let data_type = format.get_type(schema)?;
+    if data_type.is_nested() {
+        Err(PlanError::invalid(format!(
+            "{function_name} format argument must be a string, got {data_type}"
+        )))
+    } else if data_type.is_string() {
+        Ok(format)
+    } else {
+        Ok(cast(format, DataType::Utf8))
+    }
+}
+
+fn declare_nullable_result(expr: Expr, nullable: bool, schema: &DFSchemaRef) -> PlanResult {
+    if nullable && !expr.nullable(schema)? {
+        let null = lit(ScalarValue::try_from(&expr.get_type(schema)?)?);
+        Ok(when(lit(true), expr).otherwise(null)?)
+    } else {
+        Ok(expr)
+    }
+}
+
 fn to_date(input: ScalarFunctionInput) -> PlanResult {
     if input.arguments.len() == 1 {
         // If format is not supplied, the function is a synonym for cast(expr AS DATE).
         crate::function::scalar::conversion::cast_to_date(input)
     } else if input.arguments.len() == 2 {
         let expr = input.arguments[0].clone();
+        let format = coerce_datetime_format(
+            "to_date",
+            input.arguments[1].clone(),
+            input.function_context.schema,
+        )?;
         let expr_type = expr.get_type(input.function_context.schema);
-        if let Ok(DataType::Timestamp(_, _)) = expr_type {
-            let expr = expr_fn::to_local_time(vec![expr]);
-            return Ok(cast(expr, DataType::Date32)); // In case of data type timestamp, ignore format
+        let date = match &expr_type {
+            Ok(DataType::Timestamp(_, _)) => Some(expr_fn::to_local_time(vec![expr.clone()])),
+            Ok(DataType::Date32 | DataType::Date64) => Some(expr.clone()),
+            _ => None,
+        };
+        if let Some(date) = date {
+            let nullable = !input.function_context.plan_config.ansi_mode
+                || expr.nullable(input.function_context.schema)?
+                || format.nullable(input.function_context.schema)?;
+            return declare_nullable_result(
+                cast(date, DataType::Date32),
+                nullable,
+                input.function_context.schema,
+            );
         }
         let expr = match expr_type {
             Ok(_other) => expr,
             Err(_) => cast(expr, DataType::Utf8), // In case of error, cast to string
         };
-        let mut arguments = input.arguments;
-        arguments[0] = expr;
-        Ok(ScalarUDF::from(SparkDate::new(false)).call(arguments))
+        Ok(ScalarUDF::from(SparkDate::new(false)).call(vec![expr, format]))
     } else {
         Err(PlanError::invalid("to_date requires 1 or 2 arguments"))
     }
 }
 
+fn validate_unix_timestamp_format(format: &Expr, schema: &DFSchemaRef) -> PlanResult<()> {
+    match format.get_type(schema)? {
+        DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View | DataType::Null => Ok(()),
+        data_type => Err(PlanError::invalid(format!(
+            "unix_timestamp format argument must be a string, got {data_type}"
+        ))),
+    }
+}
+
 fn unix_timestamp(input: ScalarFunctionInput) -> PlanResult {
     let timezone = input.function_context.plan_config.session_timezone.clone();
+    let ansi_mode = input.function_context.plan_config.ansi_mode;
     if input.arguments.is_empty() {
         let expr = ScalarUDF::from(TimestampNow::new(timezone, TimeUnit::Second)).call(vec![]);
         Ok(cast(expr, DataType::Int64))
     } else if input.arguments.len() == 1 {
-        Ok(ScalarUDF::from(SparkUnixTimestamp::new(timezone)).call(input.arguments))
+        Ok(ScalarUDF::from(SparkUnixTimestamp::new(timezone, ansi_mode)).call(input.arguments))
     } else if input.arguments.len() == 2 {
         let (expr, format) = input.arguments.two()?;
-        Ok(ScalarUDF::from(SparkUnixTimestamp::new(timezone)).call(vec![expr, format]))
+        if matches!(
+            expr.get_type(input.function_context.schema)?,
+            DataType::Date32 | DataType::Date64 | DataType::Timestamp(_, _) | DataType::Null
+        ) {
+            validate_unix_timestamp_format(&format, input.function_context.schema)?;
+            let nullable = !ansi_mode
+                || expr.nullable(input.function_context.schema)?
+                || format.nullable(input.function_context.schema)?;
+            let result =
+                ScalarUDF::from(SparkUnixTimestamp::new(timezone, ansi_mode)).call(vec![expr]);
+            declare_nullable_result(result, nullable, input.function_context.schema)
+        } else {
+            Ok(
+                ScalarUDF::from(SparkUnixTimestamp::new(timezone, ansi_mode))
+                    .call(vec![expr, format]),
+            )
+        }
     } else {
         Err(PlanError::invalid(
             "unix_timestamp requires 0, 1, or 2 arguments",
@@ -504,7 +566,8 @@ fn try_to_time(input: ScalarFunctionInput) -> PlanResult {
 }
 
 /// Shared `to_time` / `try_to_time` planner. Routes through `SparkTime`, which
-/// parses strings (with an optional chrono format) or casts time/timestamp args.
+/// parses strings (with an optional Spark Java datetime pattern) or casts
+/// time/timestamp args.
 /// `to_time` errors on failure (Spark's `ToTime` is ANSI-invariant); `try_to_time`
 /// (`is_try`) returns NULL.
 fn time_with_try(input: ScalarFunctionInput, is_try: bool) -> PlanResult {
@@ -518,7 +581,6 @@ fn time_with_try(input: ScalarFunctionInput, is_try: bool) -> PlanResult {
         // cast to Utf8 here would route non-string inputs through string parsing,
         // bypassing the coercion checks and diverging from the 1-arg behavior.
         let (expr, format) = input.arguments.two()?;
-        let format = to_chrono_fmt(format);
         Ok(udf.call(vec![expr, format]))
     } else {
         let name = if is_try { "try_to_time" } else { "to_time" };
diff --git a/python/pysail/tests/spark/function/features/datetime/datetime_edge_cases.feature b/python/pysail/tests/spark/function/features/datetime/datetime_edge_cases.feature
index a8cb59279..512d2b736 100644
--- a/python/pysail/tests/spark/function/features/datetime/datetime_edge_cases.feature
+++ b/python/pysail/tests/spark/function/features/datetime/datetime_edge_cases.feature
@@ -90,21 +90,59 @@ Feature: datetime edge cases
     Background:
       Given config spark.sql.session.timeZone = UTC
 
-    Scenario Outline: Clock hour: 
+    Scenario Outline: Strict clock field H/HH rejects hour 24: 
+      Given config spark.sql.ansi.enabled = true
       When query
         """
-        SELECT to_timestamp('', '') AS result
+        SELECT to_timestamp('2026-06-15 24:30:45', '')
+        """
+      Then query error .*
+
+      Examples:
+        | case                                             | fmt                 |
+        | `to_timestamp` rejects H=24 in ANSI mode         | yyyy-MM-dd H:mm:ss  |
+        | `to_timestamp` rejects HH=24 in ANSI mode        | yyyy-MM-dd HH:mm:ss |
+
+    Scenario Outline: Strict clock field k/kk maps hour 24 to same-day midnight: 
+      Given config spark.sql.ansi.enabled = true
+      When query
+        """
+        SELECT to_timestamp('2026-06-15 24:30:45', '') AS result
+        """
+      Then query result
+        | result   |
+        |  |
+
+      Examples:
+        | case                                                   | fmt                 | result              |
+        | `to_timestamp` maps k=24 to same-day midnight          | yyyy-MM-dd k:mm:ss  | 2026-06-15 00:30:45 |
+        | `to_timestamp` maps kk=24 to same-day midnight         | yyyy-MM-dd kk:mm:ss | 2026-06-15 00:30:45 |
+
+    Scenario Outline: 12-hour clock: 
+      When query
+        """
+        SELECT to_timestamp('', 'yyyy-MM-dd hh:mm:ss a') AS result
         """
       Then query result
         | result   |
         |  |
 
       Examples:
-        | case                                                | in                     | fmt                   | result              |
-        | `to_timestamp` parses 24:00:00 as midnight next day | 2026-06-15 24:00:00    | yyyy-MM-dd HH:mm:ss   | 2026-06-16 00:00:00 |
-        | `to_timestamp` parses k=24 clock hour format        | 2026-06-15 24:00:00    | yyyy-MM-dd kk:mm:ss   | 2026-06-16 00:00:00 |
-        | `to_timestamp` parses 12-hour midnight with AM      | 2026-06-15 12:00:00 AM | yyyy-MM-dd hh:mm:ss a | 2026-06-15 00:00:00 |
-        | `to_timestamp` parses 12-hour noon with PM          | 2026-06-15 12:00:00 PM | yyyy-MM-dd hh:mm:ss a | 2026-06-15 12:00:00 |
+        | case                                           | in                     | result              |
+        | `to_timestamp` parses 12-hour midnight with AM | 2026-06-15 12:00:00 AM | 2026-06-15 00:00:00 |
+        | `to_timestamp` parses 12-hour noon with PM     | 2026-06-15 12:00:00 PM | 2026-06-15 12:00:00 |
+
+    Scenario: Strict clock field try_to_timestamp returns NULL for invalid hour and second
+      Given config spark.sql.ansi.enabled = true
+      When query
+        """
+        SELECT
+          try_to_timestamp('2026-06-15 24:30:45', 'yyyy-MM-dd H:mm:ss') AS hour_24,
+          try_to_timestamp('2026-06-15 23:59:60', 'yyyy-MM-dd HH:mm:ss') AS second_60
+        """
+      Then query result
+        | hour_24 | second_60 |
+        | NULL    | NULL      |
 
   Rule: Fractional seconds precision
 
@@ -140,19 +178,13 @@ Feature: datetime edge cases
     Background:
       Given config spark.sql.session.timeZone = UTC
 
-    Scenario Outline: Leap second: 
+    Scenario: Strict clock field to_timestamp rejects second 60
+      Given config spark.sql.ansi.enabled = true
       When query
         """
-        SELECT to_timestamp('', 'yyyy-MM-dd HH:mm:ss') AS result
+        SELECT to_timestamp('2026-06-15 23:59:60', 'yyyy-MM-dd HH:mm:ss')
         """
-      Then query result
-        | result   |
-        |  |
-
-      Examples:
-        | case                                                     | in                  | result              |
-        | `to_timestamp` handles leap second 23:59:60              | 2026-06-15 23:59:60 | 2026-06-16 00:00:00 |
-        | `to_timestamp` rejects invalid leap second at wrong time | 2026-06-15 12:30:60 | NULL                |
+      Then query error .*
 
   Rule: Era handling with BC dates
 
@@ -182,6 +214,13 @@ Feature: datetime edge cases
     Background:
       Given config spark.sql.session.timeZone = UTC
 
+    Scenario: `date_format` rejects week-of-month pattern W
+      When query
+        """
+        SELECT date_format(DATE '2026-06-15', 'W')
+        """
+      Then query error .*
+
     Scenario Outline: Week-based field: 
       When query
         """
@@ -193,7 +232,6 @@ Feature: datetime edge cases
 
       Examples:
         | case                                        | fmt | result |
-        | `date_format` formats week-of-month         | W   | 3      |
         | `date_format` formats aligned week-of-month | F   | 1      |
         | `date_format` formats quarter               | Q   | 2      |
         | `date_format` formats quarter with text     | QQQ | Q2     |
@@ -203,7 +241,7 @@ Feature: datetime edge cases
     Background:
       Given config spark.sql.session.timeZone = UTC
 
-    Scenario Outline: Optional section date_format: 
+    Scenario Outline: Optional section formatting with fractional seconds: 
       When query
         """
         SELECT date_format(TIMESTAMP '', 'yyyy-MM-dd HH:mm:ss[.SSSSSS]') AS result
@@ -213,9 +251,18 @@ Feature: datetime edge cases
         |  |
 
       Examples:
-        | case                                                  | ts                         | result                     |
-        | `date_format` omits optional section when zero        | 2026-06-15 14:30:45        | 2026-06-15 14:30:45        |
-        | `date_format` includes optional section when non-zero | 2026-06-15 14:30:45.123456 | 2026-06-15 14:30:45.123456 |
+        | case                                                               | ts                         | result                     |
+        | `date_format` includes optional section when fraction is zero     | 2026-06-15 14:30:45        | 2026-06-15 14:30:45.000000 |
+        | `date_format` includes optional section when fraction is non-zero | 2026-06-15 14:30:45.123456 | 2026-06-15 14:30:45.123456 |
+
+    Scenario: Optional section formatting includes all-zero time fields
+      When query
+        """
+        SELECT date_format(TIMESTAMP '2026-06-15 00:00:00', 'yyyy-MM-dd[ HH:mm:ss]') AS result
+        """
+      Then query result
+        | result              |
+        | 2026-06-15 00:00:00 |
 
     Scenario Outline: Optional section to_timestamp: 
       When query
diff --git a/python/pysail/tests/spark/function/features/datetime/datetime_format.feature b/python/pysail/tests/spark/function/features/datetime/datetime_format.feature
index 431af91a5..1a02ee789 100644
--- a/python/pysail/tests/spark/function/features/datetime/datetime_format.feature
+++ b/python/pysail/tests/spark/function/features/datetime/datetime_format.feature
@@ -18,218 +18,95 @@ Feature: datetime format strings
 
       Examples:
         | case                                                            | ts                         | fmt                                       | result               |
-        | `date_format` formats ISO 8601 timestamp with literal separator | 2026-06-01 10:30:45        | 'yyyy-MM-dd''T''HH:mm:ss'                 | 2026-06-01T10:30:45  |
+        | `date_format` formats ISO 8601 timestamp with literal separator | 2026-06-01 10:30:45        | "yyyy-MM-dd'T'HH:mm:ss"                   | 2026-06-01T10:30:45  |
         | `date_format` formats standalone fractional seconds             | 2026-06-15 14:30:45.123456 | 'SSSSSS'                                  | 123456               |
         | `date_format` formats text fields                               | 2026-06-15 14:30:45        | 'EEEE, dd MMMM yyyy'                      | Monday, 15 June 2026 |
-        | `date_format` formats a dynamic format expression               | 2026-06-15 14:30:45        | concat('yyyy-MM-dd', '''T''', 'HH:mm:ss') | 2026-06-15T14:30:45  |
+        | `date_format` formats a dynamic format expression               | 2026-06-15 14:30:45        | concat('yyyy-MM-dd', "'T'", 'HH:mm:ss')   | 2026-06-15T14:30:45  |
 
-  Rule: Java predefined DateTimeFormatter constants
+  Rule: Spark datetime pattern validation for formatting
 
     Background:
       Given config spark.sql.session.timeZone = UTC
 
-    Scenario: `date_format` formats predefined local ISO constants
+    Scenario Outline: Predefined formatter name: 
       When query
         """
-        SELECT
-          date_format(TIMESTAMP '2026-06-15 14:30:45.123456', 'BASIC_ISO_DATE') AS basic_date,
-          date_format(TIMESTAMP '2026-06-15 14:30:45.123456', 'ISO_LOCAL_DATE') AS local_date,
-          date_format(TIMESTAMP '2026-06-15 14:30:45.123456', 'ISO_LOCAL_TIME') AS local_time,
-          date_format(TIMESTAMP '2026-06-15 14:30:45.123456', 'ISO_LOCAL_DATE_TIME') AS local_datetime
-        """
-      Then query result
-        | basic_date | local_date | local_time      | local_datetime             |
-        | 20260615   | 2026-06-15 | 14:30:45.123456 | 2026-06-15T14:30:45.123456 |
-
-    Scenario: `date_format` formats predefined ISO offset constants
-      When query
-        """
-        SELECT
-          date_format(TIMESTAMP '2026-06-15 14:30:45 UTC', 'ISO_OFFSET_DATE') AS offset_date,
-          date_format(TIMESTAMP '2026-06-15 14:30:45 UTC', 'ISO_OFFSET_TIME') AS offset_time,
-          date_format(TIMESTAMP '2026-06-15 14:30:45 UTC', 'ISO_OFFSET_DATE_TIME') AS offset_datetime,
-          date_format(TIMESTAMP '2026-06-15 14:30:45 UTC', 'ISO_INSTANT') AS instant
-        """
-      Then query result
-        | offset_date | offset_time | offset_datetime      | instant              |
-        | 2026-06-15Z | 14:30:45Z   | 2026-06-15T14:30:45Z | 2026-06-15T14:30:45Z |
-
-    Scenario: `date_format` formats predefined ISO date variants with optional offset
-      When query
-        """
-        SELECT
-          date_format(TIMESTAMP '2026-06-15 14:30:45', 'ISO_DATE') AS local_date,
-          date_format(TIMESTAMP '2026-06-15 14:30:45 UTC', 'ISO_DATE') AS offset_date,
-          date_format(DATE '2026-06-15', 'ISO_DATE') AS date_only,
-          date_format(TIMESTAMP '2026-06-15 14:30:45 UTC', 'ISO_DATE_TIME') AS datetime_with_zone
-        """
-      Then query result
-        | local_date  | offset_date | date_only  | datetime_with_zone        |
-        | 2026-06-15Z | 2026-06-15Z | 2026-06-15 | 2026-06-15T14:30:45Z[UTC] |
-
-    Scenario: `date_format` formats predefined ordinal and week dates
-      When query
-        """
-        SELECT
-          date_format(TIMESTAMP '2026-06-15 14:30:45', 'ISO_ORDINAL_DATE') AS ordinal_date,
-          date_format(TIMESTAMP '2026-06-15 14:30:45', 'ISO_WEEK_DATE') AS week_date,
-          date_format(DATE '2026-06-15', 'ISO_ORDINAL_DATE') AS ordinal_date_only,
-          date_format(DATE '2026-06-15', 'ISO_WEEK_DATE') AS week_date_only
-        """
-      Then query result
-        | ordinal_date | week_date   | ordinal_date_only | week_date_only |
-        | 2026-166Z    | 2026-W25-1Z | 2026-166          | 2026-W25-1     |
-
-    Scenario: `date_format` formats ISO_LOCAL_DATE with different input types
-      When query
-        """
-        SELECT
-          date_format(DATE '2026-06-15', 'ISO_LOCAL_DATE') AS date_only,
-          date_format(TIMESTAMP '2026-06-15 14:30:45', 'ISO_LOCAL_DATE') AS timestamp_local,
-          date_format(TIMESTAMP '2026-06-15 14:30:45 UTC', 'ISO_LOCAL_DATE') AS timestamp_utc
+        SELECT date_format(TIMESTAMP '2018-11-17 13:33:33.333', '')
         """
-      Then query result
-        | date_only  | timestamp_local | timestamp_utc |
-        | 2026-06-15 | 2026-06-15      | 2026-06-15    |
-
-    Scenario: `date_format` formats ISO_LOCAL_TIME with different time components
-      When query
-        """
-        SELECT
-          date_format(TIMESTAMP '2026-06-15 00:00:00', 'ISO_LOCAL_TIME') AS midnight,
-          date_format(TIMESTAMP '2026-06-15 12:00:00', 'ISO_LOCAL_TIME') AS noon,
-          date_format(TIMESTAMP '2026-06-15 23:59:59', 'ISO_LOCAL_TIME') AS last_second,
-          date_format(TIMESTAMP '2026-06-15 14:30:45.123456789', 'ISO_LOCAL_TIME') AS with_nanos
-        """
-      Then query result
-        | midnight | noon     | last_second | with_nanos      |
-        | 00:00:00 | 12:00:00 | 23:59:59    | 14:30:45.123456 |
-
-    Scenario: `date_format` formats ISO_LOCAL_DATE_TIME with different components
-      When query
-        """
-        SELECT
-          date_format(TIMESTAMP '2026-06-15 00:00:00', 'ISO_LOCAL_DATE_TIME') AS midnight,
-          date_format(TIMESTAMP '2026-06-15 14:30:45.123456', 'ISO_LOCAL_DATE_TIME') AS with_fractional,
-          date_format(TIMESTAMP '2026-06-15 23:59:59', 'ISO_LOCAL_DATE_TIME') AS last_second
-        """
-      Then query result
-        | midnight            | with_fractional            | last_second         |
-        | 2026-06-15T00:00:00 | 2026-06-15T14:30:45.123456 | 2026-06-15T23:59:59 |
-
-    Scenario: `date_format` formats ISO_OFFSET_DATE with non-UTC timezones
-      When query
-        """
-        SELECT
-          date_format(TIMESTAMP '2026-06-15 14:30:45+02:00', 'ISO_OFFSET_DATE') AS positive_offset,
-          date_format(TIMESTAMP '2026-06-15 14:30:45-05:00', 'ISO_OFFSET_DATE') AS negative_offset,
-          date_format(TIMESTAMP '2026-06-15 14:30:45+05:30', 'ISO_OFFSET_DATE') AS half_hour_offset
-        """
-      Then query result
-        | positive_offset | negative_offset | half_hour_offset |
-        | 2026-06-15Z     | 2026-06-15Z     | 2026-06-15Z      |
-
-    Scenario: `date_format` formats ISO_OFFSET_TIME with non-UTC timezones
-      When query
-        """
-        SELECT
-          date_format(TIMESTAMP '2026-06-15 14:30:45+02:00', 'ISO_OFFSET_TIME') AS positive_offset,
-          date_format(TIMESTAMP '2026-06-15 14:30:45-05:00', 'ISO_OFFSET_TIME') AS negative_offset,
-          date_format(TIMESTAMP '2026-06-15 14:30:45+05:30', 'ISO_OFFSET_TIME') AS half_hour_offset,
-          date_format(TIMESTAMP '2026-06-15 14:30:45.123456+02:00', 'ISO_OFFSET_TIME') AS with_fractional
-        """
-      Then query result
-        | positive_offset | negative_offset | half_hour_offset | with_fractional  |
-        | 12:30:45Z       | 19:30:45Z       | 09:00:45Z        | 12:30:45.123456Z |
+      Then query error .*
 
-    Scenario: `date_format` formats ISO_OFFSET_DATE_TIME with non-UTC timezones
-      When query
-        """
-        SELECT
-          date_format(TIMESTAMP '2026-06-15 14:30:45+02:00', 'ISO_OFFSET_DATE_TIME') AS positive_offset,
-          date_format(TIMESTAMP '2026-06-15 14:30:45-05:00', 'ISO_OFFSET_DATE_TIME') AS negative_offset,
-          date_format(TIMESTAMP '2026-06-15 14:30:45+05:30', 'ISO_OFFSET_DATE_TIME') AS half_hour_offset,
-          date_format(TIMESTAMP '2026-06-15 14:30:45.123456+02:00', 'ISO_OFFSET_DATE_TIME') AS with_fractional
-        """
-      Then query result
-        | positive_offset      | negative_offset      | half_hour_offset     | with_fractional             |
-        | 2026-06-15T12:30:45Z | 2026-06-15T19:30:45Z | 2026-06-15T09:00:45Z | 2026-06-15T12:30:45.123456Z |
-
-    Scenario: `date_format` formats ISO_INSTANT with different timezone inputs
-      When query
-        """
-        SELECT
-          date_format(TIMESTAMP '2026-06-15 14:30:45 UTC', 'ISO_INSTANT') AS utc_instant,
-          date_format(TIMESTAMP '2026-06-15 14:30:45+02:00', 'ISO_INSTANT') AS positive_offset,
-          date_format(TIMESTAMP '2026-06-15 14:30:45-05:00', 'ISO_INSTANT') AS negative_offset
-        """
-      Then query result
-        | utc_instant          | positive_offset      | negative_offset      |
-        | 2026-06-15T14:30:45Z | 2026-06-15T12:30:45Z | 2026-06-15T19:30:45Z |
-
-    Scenario: `date_format` formats BASIC_ISO_DATE with different input types
-      When query
-        """
-        SELECT
-          date_format(DATE '2026-06-15', 'BASIC_ISO_DATE') AS date_only,
-          date_format(TIMESTAMP '2026-06-15 14:30:45', 'BASIC_ISO_DATE') AS timestamp_local,
-          date_format(TIMESTAMP '2026-06-15 14:30:45 UTC', 'BASIC_ISO_DATE') AS timestamp_utc
-        """
-      Then query result
-        | date_only | timestamp_local | timestamp_utc |
-        | 20260615  | 20260615        | 20260615      |
+      Examples:
+        | case                                                       | fmt                  |
+        | `date_format` rejects BASIC_ISO_DATE as a predefined name  | BASIC_ISO_DATE       |
+        | `date_format` rejects ISO_LOCAL_DATE as a predefined name  | ISO_LOCAL_DATE       |
+        | `date_format` rejects ISO_WEEK_DATE as a predefined name   | ISO_WEEK_DATE        |
+        | `date_format` rejects RFC_1123_DATE_TIME as a predefined name | RFC_1123_DATE_TIME |
 
-    Scenario: `date_format` handles NULL with ISO formats
+    Scenario Outline: Disabled week-based pattern: 
       When query
         """
-        SELECT
-          date_format(CAST(NULL AS TIMESTAMP), 'ISO_DATE') AS null_timestamp,
-          date_format(CAST(NULL AS DATE), 'ISO_LOCAL_DATE') AS null_date,
-          date_format(CAST(NULL AS TIMESTAMP), 'ISO_OFFSET_TIME') AS null_time
+        SELECT date_format(TIMESTAMP '2018-11-17 13:33:33.333', '')
         """
-      Then query result
-        | null_timestamp | null_date | null_time |
-        | NULL           | NULL      | NULL      |
+      Then query error .*
 
-    Scenario: `date_format` formats extreme dates with ISO formats
-      When query
-        """
-        SELECT
-          date_format(DATE '0001-01-01', 'ISO_DATE') AS min_date,
-          date_format(DATE '9999-12-31', 'ISO_DATE') AS max_date,
-          date_format(TIMESTAMP '0001-01-01 00:00:00', 'ISO_DATE_TIME') AS min_timestamp,
-          date_format(TIMESTAMP '9999-12-31 23:59:59', 'ISO_DATE_TIME') AS max_timestamp
-        """
-      Then query result
-        | min_date   | max_date   | min_timestamp             | max_timestamp             |
-        | 0001-01-01 | 9999-12-31 | 0001-01-01T00:00:00Z[UTC] | 9999-12-31T23:59:59Z[UTC] |
-
-    Scenario: `date_format` formats timezone offset edge cases
-      When query
-        """
-        SELECT
-          date_format(TIMESTAMP '2026-06-15 14:30:45+00:00', 'ISO_OFFSET_DATE') AS zero_offset,
-          date_format(TIMESTAMP '2026-06-15 14:30:45-00:00', 'ISO_OFFSET_DATE') AS negative_zero,
-          date_format(TIMESTAMP '2026-06-15 14:30:45+14:00', 'ISO_OFFSET_DATE') AS max_positive,
-          date_format(TIMESTAMP '2026-06-15 14:30:45-12:00', 'ISO_OFFSET_DATE') AS max_negative
-        """
-      Then query result
-        | zero_offset | negative_zero | max_positive | max_negative |
-        | 2026-06-15Z | 2026-06-15Z   | 2026-06-15Z  | 2026-06-16Z  |
+      Examples:
+        | case                                             | fmt |
+        | `date_format` rejects week-based year            | Y   |
+        | `date_format` rejects week of month              | W   |
+        | `date_format` rejects week of year               | w   |
+        | `date_format` rejects ISO day number             | u   |
+        | `date_format` rejects localized day number       | e   |
+        | `date_format` rejects stand-alone day number     | c   |
 
-    Scenario Outline: Predefined constant: 
+    Scenario Outline: Invalid Java datetime pattern: 
       When query
         """
-        SELECT date_format(TIMESTAMP '2026-06-15 14:30:45 UTC', '') AS result
+        SELECT date_format(TIMESTAMP '2018-11-17 13:33:33.333', '')
         """
-      Then query result
-        | result   |
-        |  |
+      Then query error .*
 
       Examples:
-        | case                                                | fmt                | result                        |
-        | `date_format` formats predefined ISO time variants  | ISO_OFFSET_TIME    | 14:30:45Z                     |
-        | `date_format` formats predefined RFC 1123 date time | RFC_1123_DATE_TIME | Mon, 15 Jun 2026 14:30:45 GMT |
+        | case                                                        | fmt        |
+        | `date_format` rejects narrow era                            | GGGGG      |
+        | `date_format` rejects narrow month                          | MMMMM      |
+        | `date_format` rejects narrow stand-alone month              | LLLLL      |
+        | `date_format` rejects narrow day name                       | EEEEE      |
+        | `date_format` rejects narrow quarter                        | QQQQQ      |
+        | `date_format` rejects narrow stand-alone quarter            | qqqqq      |
+        | `date_format` rejects year wider than six digits            | yyyyyyy    |
+        | `date_format` rejects repeated aligned day of week in month | FF         |
+        | `date_format` rejects three day-of-month letters            | ddd        |
+        | `date_format` rejects four day-of-year letters              | DDDD       |
+        | `date_format` rejects three 24-hour letters                 | HHH        |
+        | `date_format` rejects three 12-hour letters                 | hhh        |
+        | `date_format` rejects three clock-hour letters              | kkk        |
+        | `date_format` rejects three am-pm hour letters              | KKK        |
+        | `date_format` rejects three minute letters                  | mmm        |
+        | `date_format` rejects three second letters                  | sss        |
+        | `date_format` rejects ten fractional-second letters         | SSSSSSSSSS |
+        | `date_format` rejects repeated am-pm marker                 | aa         |
+        | `date_format` rejects single zone ID letter                 | V          |
+        | `date_format` rejects five zone-name letters                | zzzzz      |
+        | `date_format` rejects six ISO offset letters                | XXXXXX     |
+        | `date_format` rejects six localized offset letters          | ZZZZZZ     |
+        | `date_format` rejects two localized-zone offset letters     | OO         |
+        | `date_format` rejects six lower-case offset letters         | xxxxxx     |
+        | `date_format` rejects millisecond-of-day                    | A          |
+        | `date_format` rejects day-period                            | B          |
+        | `date_format` rejects nano-of-second                        | n          |
+        | `date_format` rejects nano-of-day                           | N          |
+        | `date_format` rejects pad-next                              | p          |
+        | `date_format` rejects unknown pattern letter C              | C          |
+        | `date_format` rejects unknown pattern letter I              | I          |
+
+    Scenario: `date_format` treats quoted restricted letters as literals
+      When query
+        """
+        SELECT date_format(DATE '2026-06-15', "yyyy-MM-dd 'Y' 'B' 'E' 'Q'") AS result
+        """
+      Then query result
+        | result             |
+        | 2026-06-15 Y B E Q |
 
   Rule: Edge cases and special scenarios
 
@@ -254,9 +131,50 @@ Feature: datetime format strings
         | `date_format` formats date only                      | DATE '2026-06-15'                         | 'yyyy-MM-dd'                    | 2026-06-15                    |
         | `date_format` formats with quarter                   | DATE '2026-06-15'                         | 'yyyy-Q-dd'                     | 2026-2-15                     |
         | `date_format` formats with era                       | DATE '2026-06-15'                         | 'GGGG yyyy-MM-dd'               | Anno Domini 2026-06-15        |
-        | `date_format` formats with week-based year           | DATE '2026-06-15'                         | 'YYYY-ww-e'                     | 2026-25-1                     |
         | `date_format` formats with day of year               | DATE '2026-06-15'                         | 'yyyy-DDD'                      | 2026-166                      |
 
+  Rule: Java offset pattern formatting semantics
+
+    Scenario: Offset pattern semantics formats UTC with X and x widths 1 through 5
+      Given config spark.sql.session.timeZone = UTC
+      When query
+        """
+        SELECT
+          date_format(TIMESTAMP '2026-06-15 14:30:45', 'X') AS upper_1,
+          date_format(TIMESTAMP '2026-06-15 14:30:45', 'XX') AS upper_2,
+          date_format(TIMESTAMP '2026-06-15 14:30:45', 'XXX') AS upper_3,
+          date_format(TIMESTAMP '2026-06-15 14:30:45', 'XXXX') AS upper_4,
+          date_format(TIMESTAMP '2026-06-15 14:30:45', 'XXXXX') AS upper_5,
+          date_format(TIMESTAMP '2026-06-15 14:30:45', 'x') AS lower_1,
+          date_format(TIMESTAMP '2026-06-15 14:30:45', 'xx') AS lower_2,
+          date_format(TIMESTAMP '2026-06-15 14:30:45', 'xxx') AS lower_3,
+          date_format(TIMESTAMP '2026-06-15 14:30:45', 'xxxx') AS lower_4,
+          date_format(TIMESTAMP '2026-06-15 14:30:45', 'xxxxx') AS lower_5
+        """
+      Then query result
+        | upper_1 | upper_2 | upper_3 | upper_4 | upper_5 | lower_1 | lower_2 | lower_3 | lower_4 | lower_5 |
+        | Z       | Z       | Z       | Z       | Z       | +00     | +0000   | +00:00  | +0000   | +00:00  |
+
+    Scenario: Offset pattern semantics formats a historical second-precision offset
+      Given config spark.sql.session.timeZone = Europe/Paris
+      When query
+        """
+        SELECT
+          date_format(TIMESTAMP '1900-01-01 12:00:00', 'X') AS upper_1,
+          date_format(TIMESTAMP '1900-01-01 12:00:00', 'XX') AS upper_2,
+          date_format(TIMESTAMP '1900-01-01 12:00:00', 'XXX') AS upper_3,
+          date_format(TIMESTAMP '1900-01-01 12:00:00', 'XXXX') AS upper_4,
+          date_format(TIMESTAMP '1900-01-01 12:00:00', 'XXXXX') AS upper_5,
+          date_format(TIMESTAMP '1900-01-01 12:00:00', 'x') AS lower_1,
+          date_format(TIMESTAMP '1900-01-01 12:00:00', 'xx') AS lower_2,
+          date_format(TIMESTAMP '1900-01-01 12:00:00', 'xxx') AS lower_3,
+          date_format(TIMESTAMP '1900-01-01 12:00:00', 'xxxx') AS lower_4,
+          date_format(TIMESTAMP '1900-01-01 12:00:00', 'xxxxx') AS lower_5
+        """
+      Then query result
+        | upper_1 | upper_2 | upper_3 | upper_4 | upper_5  | lower_1 | lower_2 | lower_3 | lower_4 | lower_5  |
+        | +0009   | +0009   | +00:09  | +000921 | +00:09:21 | +0009   | +0009   | +00:09  | +000921 | +00:09:21 |
+
   Rule: Width variation tests for month patterns
 
     Background:
@@ -277,7 +195,6 @@ Feature: datetime format strings
         | `date_format` formats month with width 2 (MM)    | MM    | 06     |
         | `date_format` formats month with width 3 (MMM)   | MMM   | Jun    |
         | `date_format` formats month with width 4 (MMMM)  | MMMM  | June   |
-        | `date_format` formats month with width 5 (MMMMM) | MMMMM | J      |
 
   Rule: Width variation tests for day-of-week patterns
 
@@ -299,7 +216,6 @@ Feature: datetime format strings
         | `date_format` formats day-of-week with width 2 (EE)    | EE    | Mon    |
         | `date_format` formats day-of-week with width 3 (EEE)   | EEE   | Mon    |
         | `date_format` formats day-of-week with width 4 (EEEE)  | EEEE  | Monday |
-        | `date_format` formats day-of-week with width 5 (EEEEE) | EEEEE | M      |
 
   Rule: Width variation tests for era patterns
 
@@ -320,7 +236,7 @@ Feature: datetime format strings
         | `date_format` formats era with width 1 (G)     | G     | AD     |
         | `date_format` formats era with width 2 (GG)    | GG    | AD     |
         | `date_format` formats era with width 3 (GGG)   | GGG   | AD     |
-        | `date_format` formats era with width 5 (GGGGG) | GGGGG | A      |
+        | `date_format` formats era with width 4 (GGGG)  | GGGG  | Anno Domini |
 
   Rule: Width variation tests for quarter patterns
 
@@ -342,7 +258,6 @@ Feature: datetime format strings
         | `date_format` formats quarter with width 2 (QQ)    | QQ    | 02          |
         | `date_format` formats quarter with width 3 (QQQ)   | QQQ   | Q2          |
         | `date_format` formats quarter with width 4 (QQQQ)  | QQQQ  | 2nd quarter |
-        | `date_format` formats quarter with width 5 (QQQQQ) | QQQQQ | 2           |
 
   Rule: Padding tests for numeric fields
 
diff --git a/python/pysail/tests/spark/function/features/datetime/datetime_parse.feature b/python/pysail/tests/spark/function/features/datetime/datetime_parse.feature
index b0ba69177..7adbafb30 100644
--- a/python/pysail/tests/spark/function/features/datetime/datetime_parse.feature
+++ b/python/pysail/tests/spark/function/features/datetime/datetime_parse.feature
@@ -2,83 +2,75 @@
 @datetime_parse
 Feature: datetime parsing with format strings
 
-  Rule: Java predefined DateTimeFormatter constants for parsing
+  Rule: Spark datetime pattern validation for parsing
 
     Background:
       Given config spark.sql.session.timeZone = UTC
 
-    Scenario Outline: Predefined constant: 
+    Scenario Outline: Predefined formatter name: 
       When query
         """
-        SELECT to_timestamp('', '') AS result
+        SELECT ('', '')
         """
-      Then query result
-        | result   |
-        |  |
+      Then query error .*
 
       Examples:
-        | case                                                      | in                            | fmt                  | result                     |
-        | `to_timestamp` parses ISO_LOCAL_DATE_TIME format          | 2026-06-15T14:30:45           | ISO_LOCAL_DATE_TIME  | 2026-06-15 14:30:45        |
-        | `to_timestamp` parses ISO_OFFSET_DATE_TIME format         | 2026-06-15T14:30:45Z          | ISO_OFFSET_DATE_TIME | 2026-06-15 14:30:45        |
-        | `to_timestamp` parses ISO_INSTANT format                  | 2026-06-15T14:30:45Z          | ISO_INSTANT          | 2026-06-15 14:30:45        |
-        | `to_timestamp` parses ISO_LOCAL_DATE format               | 2026-06-15                    | ISO_LOCAL_DATE       | 2026-06-15 00:00:00        |
-        | `to_timestamp` parses ISO_LOCAL_TIME format               | 14:30:45                      | ISO_LOCAL_TIME       | 1970-01-01 14:30:45        |
-        | `to_timestamp` parses BASIC_ISO_DATE format               | 20260615                      | BASIC_ISO_DATE       | 2026-06-15 00:00:00        |
-        | `to_timestamp` parses ISO_ORDINAL_DATE format             | 2026-166                      | ISO_ORDINAL_DATE     | 2026-06-15 00:00:00        |
-        | `to_timestamp` parses ISO_WEEK_DATE format                | 2026-W25-1                    | ISO_WEEK_DATE        | 2026-06-15 00:00:00        |
-        | `to_timestamp` parses ISO_DATE format with offset         | 2026-06-15Z                   | ISO_DATE             | 2026-06-15 00:00:00        |
-        | `to_timestamp` parses ISO_DATE format without offset      | 2026-06-15                    | ISO_DATE             | 2026-06-15 00:00:00        |
-        | `to_timestamp` parses ISO_TIME format with offset         | 14:30:45Z                     | ISO_TIME             | 1970-01-01 14:30:45        |
-        | `to_timestamp` parses ISO_TIME format without offset      | 14:30:45.123                  | ISO_TIME             | 1970-01-01 14:30:45.123    |
-        | `to_timestamp` parses ISO_OFFSET_DATE format              | 2026-06-15Z                   | ISO_OFFSET_DATE      | 2026-06-15 00:00:00        |
-        | `to_timestamp` parses ISO_OFFSET_TIME format              | 14:30:45Z                     | ISO_OFFSET_TIME      | 1970-01-01 14:30:45        |
-        | `to_timestamp` parses ISO_DATE_TIME format with zone      | 2026-06-15T14:30:45Z[UTC]     | ISO_DATE_TIME        | 2026-06-15 14:30:45        |
-        | `to_timestamp` parses ISO_DATE_TIME format without offset | 2026-06-15T14:30:45.123456    | ISO_DATE_TIME        | 2026-06-15 14:30:45.123456 |
-        | `to_timestamp` parses ISO_8601 format alias               | 2026-06-01T10:30:45           | ISO_8601             | 2026-06-01 10:30:45        |
-        | `to_timestamp` parses ISO_ZONED_DATE_TIME format          | 2026-06-15T14:30:45Z[UTC]     | ISO_ZONED_DATE_TIME  | 2026-06-15 14:30:45        |
-        | `to_timestamp` parses RFC_1123_DATE_TIME format           | Mon, 15 Jun 2026 14:30:45 GMT | RFC_1123_DATE_TIME   | 2026-06-15 14:30:45        |
-
-  Rule: to_date with predefined formatters
+        | case                                                          | fn           | in         | fmt                |
+        | `to_timestamp` rejects ISO_LOCAL_DATE as a predefined name    | to_timestamp | 2026-06-15 | ISO_LOCAL_DATE     |
+        | `to_date` rejects BASIC_ISO_DATE as a predefined name          | to_date      | 2026-06-15 | BASIC_ISO_DATE     |
+        | `to_timestamp` rejects ISO_WEEK_DATE as a predefined name      | to_timestamp | 2026-06-15 | ISO_WEEK_DATE      |
+        | `to_timestamp` rejects RFC_1123_DATE_TIME as a predefined name | to_timestamp | 2026-06-15 | RFC_1123_DATE_TIME |
 
-    Background:
-      Given config spark.sql.session.timeZone = UTC
-
-    Scenario Outline: to_date constant: 
+    Scenario Outline: Disabled week-based parsing pattern: 
       When query
         """
-        SELECT to_date('', '') AS result
+        SELECT to_timestamp('2018-11-17', '')
         """
-      Then query result
-        | result   |
-        |  |
+      Then query error .*
 
       Examples:
-        | case                                            | in          | fmt              | result     |
-        | `to_date` parses ISO_LOCAL_DATE format          | 2026-06-15  | ISO_LOCAL_DATE   | 2026-06-15 |
-        | `to_date` parses BASIC_ISO_DATE format          | 20260615    | BASIC_ISO_DATE   | 2026-06-15 |
-        | `to_date` parses ISO_ORDINAL_DATE format        | 2026-166    | ISO_ORDINAL_DATE | 2026-06-15 |
-        | `to_date` parses ISO_WEEK_DATE format           | 2026-W25-1  | ISO_WEEK_DATE    | 2026-06-15 |
-        | `to_date` parses ISO_DATE format with offset    | 2026-06-15Z | ISO_DATE         | 2026-06-15 |
-        | `to_date` parses ISO_DATE format without offset | 2026-06-15  | ISO_DATE         | 2026-06-15 |
-        | `to_date` parses ISO_OFFSET_DATE format         | 2026-06-15Z | ISO_OFFSET_DATE  | 2026-06-15 |
+        | case                                              | fmt |
+        | `to_timestamp` rejects week-based year            | Y   |
+        | `to_timestamp` rejects week of month              | W   |
+        | `to_timestamp` rejects week of year               | w   |
+        | `to_timestamp` rejects ISO day number             | u   |
+        | `to_timestamp` rejects localized day number       | e   |
+        | `to_timestamp` rejects stand-alone day number     | c   |
 
-  Rule: Error handling for invalid formats
+    Scenario Outline: Parsing-only restricted pattern: 
+      When query
+        """
+        SELECT to_timestamp('2018-11-17', '')
+        """
+      Then query error .*
 
-    Background:
-      Given config spark.sql.session.timeZone = UTC
+      Examples:
+        | case                                                  | fmt |
+        | `to_timestamp` rejects day-of-week text while parsing | E   |
+        | `to_timestamp` rejects aligned day while parsing      | F   |
+        | `to_timestamp` rejects stand-alone quarter parsing    | q   |
+        | `to_timestamp` rejects quarter parsing                | Q   |
 
-    Scenario Outline: Invalid format: 
+    Scenario Outline: Invalid Java datetime parsing pattern: 
       When query
         """
-        SELECT ('', '')
+        SELECT to_timestamp('2018-11-17', '')
         """
       Then query error .*
 
       Examples:
-        | case                                                 | fn           | in         | fmt                 |
-        | `to_timestamp` errors on invalid ISO_LOCAL_DATE_TIME | to_timestamp | invalid    | ISO_LOCAL_DATE_TIME |
-        | `to_date` errors on invalid BASIC_ISO_DATE           | to_date      | invalid    | BASIC_ISO_DATE      |
-        | `to_timestamp` errors on mismatched format           | to_timestamp | 2026-06-15 | ISO_LOCAL_TIME      |
+        | case                                                   | fmt        |
+        | `to_timestamp` rejects narrow month                    | MMMMM      |
+        | `to_timestamp` rejects year wider than six digits      | yyyyyyy    |
+        | `to_timestamp` rejects three 24-hour letters           | HHH        |
+        | `to_timestamp` rejects ten fractional-second letters   | SSSSSSSSSS |
+        | `to_timestamp` rejects millisecond-of-day              | A          |
+        | `to_timestamp` rejects day-period                      | B          |
+        | `to_timestamp` rejects nano-of-second                  | n          |
+        | `to_timestamp` rejects nano-of-day                     | N          |
+        | `to_timestamp` rejects pad-next                        | p          |
+        | `to_timestamp` rejects unknown pattern letter C        | C          |
+        | `to_timestamp` rejects unknown pattern letter I        | I          |
 
   Rule: Parsing with custom format patterns
 
@@ -100,69 +92,25 @@ Feature: datetime parsing with format strings
         | `to_timestamp` parses custom format with fractional seconds | to_timestamp     | 2026-06-15 14:30:45.123456 | 'yyyy-MM-dd HH:mm:ss.SSSSSS' | 2026-06-15 14:30:45.123456 |
         | `to_date` parses custom format                              | to_date          | 2026/06/15                 | 'yyyy/MM/dd'                 | 2026-06-15                 |
         | `to_timestamp` parses with month name                       | to_timestamp     | 15 June 2026               | 'dd MMMM yyyy'               | 2026-06-15 00:00:00        |
-        | `to_timestamp` parses with day name                         | to_timestamp     | Monday, 15 June 2026       | 'EEEE, dd MMMM yyyy'         | 2026-06-15 00:00:00        |
         | `to_timestamp_ltz` parses custom format with offset         | to_timestamp_ltz | 2026-06-15T16:30:45+02:00  | "yyyy-MM-dd'T'HH:mm:ssXXX"   | 2026-06-15 14:30:45        |
-        | `to_timestamp_ntz` parses custom format with offset         | to_timestamp_ntz | 2026-06-15T16:30:45+02:00  | "yyyy-MM-dd'T'HH:mm:ssXXX"   | 2026-06-15 14:30:45        |
+        | `to_timestamp_ntz` ignores a positive offset                | to_timestamp_ntz | 2026-06-15T16:30:45+02:00  | "yyyy-MM-dd'T'HH:mm:ssXXX"   | 2026-06-15 16:30:45        |
 
-  Rule: Day-of-week parsing
+  Rule: Formatted TIMESTAMP_NTZ offset handling
 
     Background:
       Given config spark.sql.session.timeZone = UTC
 
-    Scenario: `to_timestamp` parses day-of-week with width 1-3 (E, EE, EEE)
+    Scenario: `to_timestamp_ntz` ignores a negative offset
       When query
         """
-        SELECT
-          to_timestamp('Mon, 15 June 2026', 'EEE, dd MMMM yyyy') AS day_mon,
-          to_timestamp('Tue, 16 June 2026', 'EEE, dd MMMM yyyy') AS day_tue,
-          to_timestamp('Sun, 14 June 2026', 'EEE, dd MMMM yyyy') AS day_sun
-        """
-      Then query result
-        | day_mon             | day_tue             | day_sun             |
-        | 2026-06-15 00:00:00 | 2026-06-16 00:00:00 | 2026-06-14 00:00:00 |
-
-    Scenario: `to_timestamp` parses day-of-week with width 4 (EEEE)
-      When query
-        """
-        SELECT
-          to_timestamp('Monday, 15 June 2026', 'EEEE, dd MMMM yyyy') AS day_monday,
-          to_timestamp('Sunday, 14 June 2026', 'EEEE, dd MMMM yyyy') AS day_sunday
-        """
-      Then query result
-        | day_monday          | day_sunday          |
-        | 2026-06-15 00:00:00 | 2026-06-14 00:00:00 |
-
-  Rule: Quarter parsing
-
-    Background:
-      Given config spark.sql.session.timeZone = UTC
-
-    Scenario Outline: Quarter width: 
-      When query
-        """
-        SELECT
-          to_timestamp('', '') AS q2,
-          to_timestamp('', '') AS q4
-        """
-      Then query result
-        | q2                  | q4                  |
-        | 2026-04-01 00:00:00 | 2026-10-01 00:00:00 |
-
-      Examples:
-        | case                                              | q2in             | q4in             | fmt       |
-        | `to_timestamp` parses quarter with width 1 (Q)    | 2026 Q2          | 2026 Q4          | yyyy Q    |
-        | `to_timestamp` parses quarter with width 2 (QQ)   | 2026 Q02         | 2026 Q04         | yyyy QQ   |
-        | `to_timestamp` parses quarter with width 3 (QQQ)  | 2026 Q2          | 2026 Q4          | yyyy QQQ  |
-        | `to_timestamp` parses quarter with width 4 (QQQQ) | 2026 2nd quarter | 2026 4th quarter | yyyy QQQQ |
-
-    Scenario: `to_timestamp` parses 12-hour timestamp with AM/PM marker
-      When query
-        """
-        SELECT to_timestamp('5/9/2026 4:53:33 PM', 'M/d/yyyy h:mm:ss a') AS result
+        SELECT to_timestamp_ntz(
+          '2026-06-15T16:30:45-05:00',
+          "yyyy-MM-dd'T'HH:mm:ssXXX"
+        ) AS result
         """
       Then query result
         | result              |
-        | 2026-05-09 16:53:33 |
+        | 2026-06-15 16:30:45 |
 
   Rule: AM/PM marker parsing variations
 
@@ -207,27 +155,37 @@ Feature: datetime parsing with format strings
     Background:
       Given config spark.sql.session.timeZone = UTC
 
-    Scenario: `to_timestamp` parses AD era
+    Scenario Outline: Strict era parsing accepts width-matched era text: 
+      Given config spark.sql.ansi.enabled = true
       When query
         """
-        SELECT
-          to_timestamp('AD 2026-06-15', 'G yyyy-MM-dd') AS era_ad,
-          to_timestamp('2026-06-15 AD', 'yyyy-MM-dd G') AS era_ad_suffix
+        SELECT to_timestamp('', '') AS result
         """
       Then query result
-        | era_ad              | era_ad_suffix       |
-        | 2026-06-15 00:00:00 | 2026-06-15 00:00:00 |
+        | result              |
+        | 2026-06-15 00:00:00 |
 
-    Scenario: `to_timestamp` parses CE era
+      Examples:
+        | case                              | in                     | fmt             |
+        | short AD prefix with G            | AD 2026-06-15          | G yyyy-MM-dd    |
+        | short AD suffix with G            | 2026-06-15 AD          | yyyy-MM-dd G    |
+        | full Anno Domini prefix with GGGG | Anno Domini 2026-06-15 | GGGG yyyy-MM-dd |
+
+    Scenario Outline: Strict era parsing rejects unsupported or width-mismatched era text: 
+      Given config spark.sql.ansi.enabled = true
       When query
         """
-        SELECT
-          to_timestamp('CE 2026-06-15', 'G yyyy-MM-dd') AS era_ce,
-          to_timestamp('2026-06-15 CE', 'yyyy-MM-dd G') AS era_ce_suffix
+        SELECT to_timestamp('', '')
         """
-      Then query result
-        | era_ce              | era_ce_suffix       |
-        | 2026-06-15 00:00:00 | 2026-06-15 00:00:00 |
+      Then query error .*
+
+      Examples:
+        | case                         | in                     | fmt             |
+        | CE prefix with G             | CE 2026-06-15          | G yyyy-MM-dd    |
+        | CE suffix with G             | 2026-06-15 CE          | yyyy-MM-dd G    |
+        | full Anno Domini text with G | Anno Domini 2026-06-15 | G yyyy-MM-dd    |
+        | short AD text with GGGG      | AD 2026-06-15          | GGGG yyyy-MM-dd |
+        | narrow A text with G         | A 2026-06-15           | G yyyy-MM-dd    |
 
     Scenario Outline: NULL handling: 
       When query
@@ -302,16 +260,27 @@ Feature: datetime parsing with format strings
     Background:
       Given config spark.sql.session.timeZone = UTC
 
-    Scenario: `to_timestamp` parses with extra whitespace
+    Scenario Outline: `to_timestamp` rejects extra whitespace under strict input consumption: 
+      Given config spark.sql.ansi.enabled = true
       When query
         """
-        SELECT
-          to_timestamp('2026-06-15  14:30:45', 'yyyy-MM-dd HH:mm:ss') AS extra_space,
-          to_timestamp('  2026-06-15 14:30:45  ', 'yyyy-MM-dd HH:mm:ss') AS leading_trailing
+        SELECT to_timestamp(, 'yyyy-MM-dd HH:mm:ss')
         """
-      Then query result
-        | extra_space         | leading_trailing    |
-        | 2026-06-15 14:30:45 | 2026-06-15 14:30:45 |
+      Then query error .*
+
+      Examples:
+        | case                                           | in                                        |
+        | leading space before the timestamp             | concat(' ', '2026-06-15 14:30:45')        |
+        | trailing space after the timestamp             | concat('2026-06-15 14:30:45', ' ')        |
+        | double space where the pattern has one space   | '2026-06-15  14:30:45'                    |
+
+    Scenario: `to_timestamp` rejects an unpatterned bracket suffix under strict input consumption
+      Given config spark.sql.ansi.enabled = true
+      When query
+        """
+        SELECT to_timestamp('2026-06-15 14:30:45[garbage]', 'yyyy-MM-dd HH:mm:ss')
+        """
+      Then query error .*
 
     Scenario: `to_timestamp` parses with case-insensitive month names
       When query
@@ -325,17 +294,6 @@ Feature: datetime parsing with format strings
         | lower_june          | upper_june          | mixed_june          |
         | 2026-06-15 00:00:00 | 2026-06-15 00:00:00 | 2026-06-15 00:00:00 |
 
-    Scenario: `to_timestamp` parses with case-insensitive day names
-      When query
-        """
-        SELECT
-          to_timestamp('monday, 15 June 2026', 'EEEE, dd MMMM yyyy') AS lower_mon,
-          to_timestamp('MONDAY, 15 June 2026', 'EEEE, dd MMMM yyyy') AS upper_mon
-        """
-      Then query result
-        | lower_mon           | upper_mon           |
-        | 2026-06-15 00:00:00 | 2026-06-15 00:00:00 |
-
   Rule: Parsing with different timezones
 
     Background:
@@ -395,18 +353,27 @@ Feature: datetime parsing with format strings
         | date_only           | with_time           | with_frac               |
         | 2026-06-15 00:00:00 | 2026-06-15 14:30:45 | 2026-06-15 14:30:45.789 |
 
-    Scenario: `to_timestamp` parses with optional timezone
+    Scenario: `to_timestamp` parses with optional timezone under strict input consumption
+      Given config spark.sql.ansi.enabled = true
       When query
         """
         SELECT
           to_timestamp('2026-06-15 14:30:45', 'yyyy-MM-dd HH:mm:ss[ XXX]') AS without_tz,
-          to_timestamp('2026-06-15 14:30:45+02:00', 'yyyy-MM-dd HH:mm:ss[ XXX]') AS with_tz
+          to_timestamp('2026-06-15 14:30:45 +02:00', 'yyyy-MM-dd HH:mm:ss[ XXX]') AS with_tz
         """
       Then query result
         | without_tz          | with_tz             |
         | 2026-06-15 14:30:45 | 2026-06-15 12:30:45 |
 
-  Rule: Two-digit year parsing (yy and uu)
+    Scenario: `to_timestamp` rejects optional timezone without its literal space under strict input consumption
+      Given config spark.sql.ansi.enabled = true
+      When query
+        """
+        SELECT to_timestamp('2026-06-15 14:30:45+02:00', 'yyyy-MM-dd HH:mm:ss[ XXX]')
+        """
+      Then query error .*
+
+  Rule: Two-digit year parsing
 
     Background:
       Given config spark.sql.session.timeZone = UTC
@@ -426,7 +393,6 @@ Feature: datetime parsing with format strings
       Examples:
         | case                                                          | pat |
         | `to_timestamp` parses two-digit year with yy (base year 2000) | yy  |
-        | `to_timestamp` parses two-digit year with uu (base year 2000) | uu  |
 
     Scenario: `to_date` parses two-digit year with yy
       When query
@@ -476,7 +442,7 @@ Feature: datetime parsing with format strings
         """
       Then query result
         | hour_1              | hour_24             |
-        | 2026-06-15 01:30:45 | 2026-06-16 00:00:00 |
+        | 2026-06-15 01:30:45 | 2026-06-15 00:00:00 |
 
     Scenario: `to_timestamp` parses 23-hour clock with H (0-23)
       When query
@@ -489,16 +455,22 @@ Feature: datetime parsing with format strings
         | hour_0              | hour_23             |
         | 2026-06-15 00:30:45 | 2026-06-15 23:59:59 |
 
-    Scenario: `to_timestamp` distinguishes k and H at 24:00:00
+    Scenario: `to_timestamp` distinguishes k and H at 24:00:00: k succeeds
       When query
         """
-        SELECT
-          to_timestamp('2026-06-15 24:00:00', 'yyyy-MM-dd k:mm:ss') AS clock_hour_24,
-          to_timestamp('2026-06-15 24:00:00', 'yyyy-MM-dd HH:mm:ss') AS hour_24
+        SELECT to_timestamp('2026-06-15 24:00:00', 'yyyy-MM-dd k:mm:ss') AS clock_hour_24
         """
       Then query result
-        | clock_hour_24       | hour_24             |
-        | 2026-06-16 00:00:00 | 2026-06-16 00:00:00 |
+        | clock_hour_24       |
+        | 2026-06-15 00:00:00 |
+
+    Scenario: `to_timestamp` distinguishes k and H at 24:00:00: H errors in ANSI mode
+      Given config spark.sql.ansi.enabled = true
+      When query
+        """
+        SELECT to_timestamp('2026-06-15 24:00:00', 'yyyy-MM-dd HH:mm:ss')
+        """
+      Then query error .*
 
   Rule: Width variation parsing for month patterns
 
@@ -554,27 +526,56 @@ Feature: datetime parsing with format strings
     Background:
       Given config spark.sql.session.timeZone = UTC
 
-    Scenario: `to_timestamp` parses zone offset with X (Z for zero)
+    Scenario Outline: Offset pattern semantics parses X and x width  to the same instant
+      Given config spark.sql.ansi.enabled = true
       When query
         """
         SELECT
-          to_timestamp('2026-06-15 14:30:45Z', 'yyyy-MM-dd HH:mm:ssX') AS offset_z,
-          to_timestamp('2026-06-15 14:30:45+02', 'yyyy-MM-dd HH:mm:ssX') AS offset_02
+          to_timestamp('', 'yyyy-MM-dd HH:mm:ss') AS upper_result,
+          to_timestamp('', 'yyyy-MM-dd HH:mm:ss') AS lower_result
         """
       Then query result
-        | offset_z            | offset_02           |
-        | 2026-06-15 14:30:45 | 2026-06-15 12:30:45 |
+        | upper_result        | lower_result        |
+        | 2026-06-15 12:00:45 | 2026-06-15 12:00:45 |
+
+      Examples:
+        | case                            | local               | upper_offset | upper | lower_offset | lower |
+        | 1 zero                          | 2026-06-15 12:00:45 | Z            | X     | +00          | x     |
+        | 1 hour only                     | 2026-06-15 14:00:45 | +02          | X     | +02          | x     |
+        | 1 with minute                   | 2026-06-15 14:30:45 | +0230        | X     | +0230        | x     |
+        | 2                               | 2026-06-15 14:30:45 | +0230        | XX    | +0230        | xx    |
+        | 3                               | 2026-06-15 14:30:45 | +02:30       | XXX   | +02:30       | xxx   |
+        | 4 with second                   | 2026-06-15 14:31:00 | +023015      | XXXX  | +023015      | xxxx  |
+        | 5 with second                   | 2026-06-15 14:31:00 | +02:30:15    | XXXXX | +02:30:15    | xxxxx |
 
-    Scenario: `to_timestamp` parses zone offset with x (+00 for zero)
+    Scenario Outline: Offset pattern semantics rejects invalid offset components and ranges: 
+      Given config spark.sql.ansi.enabled = true
+      When query
+        """
+        SELECT to_timestamp('2026-06-15 12:00:45', 'yyyy-MM-dd HH:mm:ss')
+        """
+      Then query error .*
+
+      Examples:
+        | case                              | offset    | fmt   |
+        | minute component 60               | +01:60    | XXX   |
+        | second component 60               | +01:00:60 | XXXXX |
+        | positive offset beyond +18:00      | +19:00    | XXX   |
+        | negative offset beyond -18:00      | -18:01    | XXX   |
+
+    Scenario: Offset pattern semantics try_to_timestamp returns NULL for invalid offsets
+      Given config spark.sql.ansi.enabled = true
       When query
         """
         SELECT
-          to_timestamp('2026-06-15 14:30:45+00', 'yyyy-MM-dd HH:mm:ssx') AS offset_00,
-          to_timestamp('2026-06-15 14:30:45+02', 'yyyy-MM-dd HH:mm:ssx') AS offset_02
+          try_to_timestamp('2026-06-15 12:00:45+01:60', 'yyyy-MM-dd HH:mm:ssXXX') AS minute_60,
+          try_to_timestamp('2026-06-15 12:00:45+01:00:60', 'yyyy-MM-dd HH:mm:ssXXXXX') AS second_60,
+          try_to_timestamp('2026-06-15 12:00:45+19:00', 'yyyy-MM-dd HH:mm:ssXXX') AS hour_19,
+          try_to_timestamp('2026-06-15 12:00:45-18:01', 'yyyy-MM-dd HH:mm:ssXXX') AS past_negative_limit
         """
       Then query result
-        | offset_00           | offset_02           |
-        | 2026-06-15 14:30:45 | 2026-06-15 12:30:45 |
+        | minute_60 | second_60 | hour_19 | past_negative_limit |
+        | NULL      | NULL      | NULL    | NULL                |
 
     Scenario: `to_timestamp` parses zone offset with Z
       When query
@@ -638,56 +639,6 @@ Feature: datetime parsing with format strings
         | day_166             | day_001             | day_365             |
         | 2026-06-15 00:00:00 | 2026-01-01 00:00:00 | 2026-12-31 00:00:00 |
 
-    Scenario: `to_timestamp` parses week-based date
-      When query
-        """
-        SELECT
-          to_timestamp('2026-W25-1', 'YYYY-\'W\'ww-e') AS week_date,
-          to_timestamp('2026-W01-1', 'YYYY-\'W\'ww-e') AS first_week,
-          to_timestamp('2026-W52-7', 'YYYY-\'W\'ww-e') AS last_week
-        """
-      Then query result
-        | week_date           | first_week          | last_week           |
-        | 2026-06-15 00:00:00 | 2025-12-29 00:00:00 | 2026-12-27 00:00:00 |
-
-  Rule: Week-based pattern variations
-
-    Background:
-      Given config spark.sql.session.timeZone = UTC
-
-    Scenario: `to_timestamp` parses week year with YYYY
-      When query
-        """
-        SELECT
-          to_timestamp('2025-W01-1', 'YYYY-\'W\'ww-e') AS week_year_2025,
-          to_timestamp('2026-W01-1', 'YYYY-\'W\'ww-e') AS week_year_2026
-        """
-      Then query result
-        | week_year_2025      | week_year_2026      |
-        | 2024-12-30 00:00:00 | 2025-12-29 00:00:00 |
-
-    Scenario: `to_timestamp` parses week of month with W
-      When query
-        """
-        SELECT
-          to_timestamp('2026-06-3', 'yyyy-MM-W') AS week_3,
-          to_timestamp('2026-06-1', 'yyyy-MM-W') AS week_1
-        """
-      Then query result
-        | week_3              | week_1              |
-        | 2026-06-15 00:00:00 | 2026-06-01 00:00:00 |
-
-    Scenario: `to_timestamp` parses day of week with e
-      When query
-        """
-        SELECT
-          to_timestamp('2026-W25-1', 'YYYY-\'W\'ww-e') AS day_1,
-          to_timestamp('2026-W25-7', 'YYYY-\'W\'ww-e') AS day_7
-        """
-      Then query result
-        | day_1               | day_7               |
-        | 2026-06-15 00:00:00 | 2026-06-21 00:00:00 |
-
   Rule: Invalid input handling
 
     Background:
@@ -710,13 +661,12 @@ Feature: datetime parsing with format strings
         | `to_timestamp` errors on non-leap year Feb 29 | 2023-02-29           | yyyy-MM-dd          |
 
     Scenario: `to_timestamp` errors on invalid second
+      Given config spark.sql.ansi.enabled = true
       When query
         """
-        SELECT to_timestamp('2026-06-15 14:30:60', 'yyyy-MM-dd HH:mm:ss') AS result
+        SELECT to_timestamp('2026-06-15 14:30:60', 'yyyy-MM-dd HH:mm:ss')
         """
-      Then query result
-        | result |
-        | NULL   |
+      Then query error .*
 
   Rule: Timezone parsing variations
 
@@ -894,16 +844,46 @@ Feature: datetime parsing with format strings
         | adjacent_date       | adjacent_datetime   |
         | 2026-06-15 00:00:00 | 2026-06-15 14:30:45 |
 
-    Scenario: `to_timestamp` parses adjacent with fractional seconds
+    Scenario: `to_timestamp` parses a fraction adjacent to fixed-width seconds
       When query
         """
         SELECT
-          to_timestamp('20260615143045123', 'yyyyMMddHHmmssSSS') AS adjacent_ms,
-          to_timestamp('20260615143045123456', 'yyyyMMddHHmmssSSSSSS') AS adjacent_us
+          to_timestamp(
+            '20181202.210400123',
+            'yyyyMMdd.HHmmssSSS'
+          ) AS literal_boundary,
+          to_timestamp(
+            '260615143045123',
+            'yyMMddHHmmssSSS'
+          ) AS fixed_run,
+          to_timestamp(
+            '202606151430451',
+            'yyyyMMddHHmmssS'
+          ) AS fixed_fraction,
+          to_timestamp(
+            '20260615.1430451',
+            'yyyyMMdd[.HHmmss]S'
+          ) AS optional_literal_boundary
         """
       Then query result
-        | adjacent_ms             | adjacent_us                |
-        | 2026-06-15 14:30:45.123 | 2026-06-15 14:30:45.123456 |
+        | literal_boundary        | fixed_run               | fixed_fraction         | optional_literal_boundary |
+        | 2018-12-02 21:04:00.123 | 2026-06-15 14:30:45.123 | 2026-06-15 14:30:45.1 | 2026-06-15 14:30:45.1 |
+
+    Scenario Outline: `to_timestamp` rejects adjacent fractional seconds under strict input consumption: 
+      Given config spark.sql.ansi.enabled = true
+      When query
+        """
+        SELECT to_timestamp('', '')
+        """
+      Then query error .*
+
+      Examples:
+        | case                                      | in                   | fmt                    |
+        | adjacent millisecond fraction with SSS    | 20260615143045123     | yyyyMMddHHmmssSSS      |
+        | adjacent microsecond fraction with SSSSSS | 20260615143045123456  | yyyyMMddHHmmssSSSSSS   |
+        | present numeric optional before fraction  | 20260615143045123     | yyyyMMdd[HHmmss]SSS    |
+        | absent numeric optional before fraction   | 20260615123           | yyyyMMdd[HHmmss]SSS    |
+        | numeric optional before fixed fraction     | 202606151430451       | yyyyMMdd[HHmmss]S      |
 
   Rule: Spark-specific deviation tests
 
@@ -972,23 +952,21 @@ Feature: datetime parsing with format strings
         | us_date             | eu_date             | iso_date            |
         | 2026-06-15 00:00:00 | 2026-06-15 00:00:00 | 2026-06-15 00:00:00 |
 
-    Scenario: `to_timestamp` handles 24:00:00 as next day midnight
+    Scenario: `to_timestamp` rejects H=24 at 24:00:00 in ANSI mode
+      Given config spark.sql.ansi.enabled = true
       When query
         """
-        SELECT to_timestamp('2026-06-15 24:00:00', 'yyyy-MM-dd HH:mm:ss') AS midnight_next
+        SELECT to_timestamp('2026-06-15 24:00:00', 'yyyy-MM-dd HH:mm:ss')
         """
-      Then query result
-        | midnight_next       |
-        | 2026-06-16 00:00:00 |
+      Then query error .*
 
-    Scenario: `to_timestamp` handles leap second 23:59:60
+    Scenario: `to_timestamp` rejects leap second 23:59:60 in ANSI mode
+      Given config spark.sql.ansi.enabled = true
       When query
         """
-        SELECT to_timestamp('2026-06-15 23:59:60', 'yyyy-MM-dd HH:mm:ss') AS leap_second
+        SELECT to_timestamp('2026-06-15 23:59:60', 'yyyy-MM-dd HH:mm:ss')
         """
-      Then query result
-        | leap_second         |
-        | 2026-06-16 00:00:00 |
+      Then query error .*
 
     Scenario: `to_timestamp` parses with Thai Buddhist calendar
       When query
@@ -1015,10 +993,10 @@ Feature: datetime parsing with format strings
       When query
         """
         SELECT
-          to_timestamp('2026-06-15', 'yyyy-MM-dd[ T][HH][:mm][:ss][.SSS]') AS minimal,
-          to_timestamp('2026-06-15T14', 'yyyy-MM-dd[ T][HH][:mm][:ss][.SSS]') AS with_hour,
-          to_timestamp('2026-06-15T14:30', 'yyyy-MM-dd[ T][HH][:mm][:ss][.SSS]') AS with_min,
-          to_timestamp('2026-06-15T14:30:45', 'yyyy-MM-dd[ T][HH][:mm][:ss][.SSS]') AS with_sec
+          to_timestamp('2026-06-15', "yyyy-MM-dd['T'][HH][:mm][:ss][.SSS]") AS minimal,
+          to_timestamp('2026-06-15T14', "yyyy-MM-dd['T'][HH][:mm][:ss][.SSS]") AS with_hour,
+          to_timestamp('2026-06-15T14:30', "yyyy-MM-dd['T'][HH][:mm][:ss][.SSS]") AS with_min,
+          to_timestamp('2026-06-15T14:30:45', "yyyy-MM-dd['T'][HH][:mm][:ss][.SSS]") AS with_sec
         """
       Then query result
         | minimal             | with_hour           | with_min            | with_sec            |
@@ -1061,6 +1039,32 @@ Feature: datetime parsing with format strings
     Background:
       Given config spark.sql.session.timeZone = UTC
 
+    Scenario Outline: Six-letter fractional pattern: 
+      When query
+        """
+        SELECT
+          to_timestamp(
+            '2026-06-15 14:30:45.',
+            'yyyy-MM-dd HH:mm:ss.SSSSSS'
+          ) AS parsed,
+          date_format(
+            to_timestamp(
+              '2026-06-15 14:30:45.',
+              'yyyy-MM-dd HH:mm:ss.SSSSSS'
+            ),
+            'SSSSSS'
+          ) AS formatted_fraction
+        """
+      Then query result
+        | parsed   | formatted_fraction   |
+        |  |  |
+
+      Examples:
+        | case                                                    | fraction | parsed                     | formatted_fraction |
+        | `to_timestamp` parses one fractional digit with SSSSSS  | 1        | 2026-06-15 14:30:45.1      | 100000             |
+        | `to_timestamp` parses three fractional digits with SSSSSS | 123    | 2026-06-15 14:30:45.123    | 123000             |
+        | `to_timestamp` parses six fractional digits with SSSSSS | 123456   | 2026-06-15 14:30:45.123456 | 123456             |
+
     Scenario: `to_timestamp` parses fractional seconds with varying widths
       When query
         """
@@ -1344,55 +1348,6 @@ Feature: datetime parsing with format strings
         | last_second         | last_nano                  |
         | 2026-06-15 23:59:59 | 2026-06-15 23:59:59.999999 |
 
-  Rule: Aligned week-of-month parsing
-
-    Background:
-      Given config spark.sql.session.timeZone = UTC
-
-    Scenario: `to_timestamp` parses aligned week-of-month with F
-      When query
-        """
-        SELECT
-          to_timestamp('2026-06-15 1', 'yyyy-MM-dd F') AS week_1,
-          to_timestamp('2026-06-08 2', 'yyyy-MM-dd F') AS week_2
-        """
-      Then query error (?i).*
-
-    Scenario: `to_timestamp` errors on inconsistent aligned week-of-month
-      When query
-        """
-        SELECT to_timestamp('2026-06-15 2', 'yyyy-MM-dd F')
-        """
-      Then query error .*
-
-  Rule: Week-of-month validation
-
-    Background:
-      Given config spark.sql.session.timeZone = UTC
-
-    Scenario: `to_timestamp` validates week-of-month consistency
-      When query
-        """
-        SELECT
-          to_timestamp('2026-06-15 3 1', 'yyyy-MM-dd W F') AS valid_week,
-          to_timestamp('2026-06-01 1 1', 'yyyy-MM-dd W F') AS first_week
-        """
-      Then query result
-        | valid_week          | first_week          |
-        | 2026-06-15 00:00:00 | 2026-06-01 00:00:00 |
-
-    Scenario Outline: Week-of-month error: 
-      When query
-        """
-        SELECT to_timestamp('', 'yyyy-MM-dd W F')
-        """
-      Then query error .*
-
-      Examples:
-        | case                                                | in             |
-        | `to_timestamp` errors on invalid week-of-month      | 2026-06-15 6 1 |
-        | `to_timestamp` errors on inconsistent week-of-month | 2026-06-15 2 1 |
-
   Rule: Pattern parsing with optional sections and literals
 
     Background:
@@ -1419,9 +1374,14 @@ Feature: datetime parsing with format strings
         | 2026-06-15 00:00:00 |
 
       Examples:
-        | case                                                          | in               | fmt                  |
-        | `to_timestamp` parses quoted literals with special characters | "2026-06-15 'Q'" | "yyyy-MM-dd '''Q'''" |
-        | `to_timestamp` parses pattern with quarter field              | '2026-06-15 Q2'  | "yyyy-MM-dd 'Q'Q"    |
+        | case                                                      | in             | fmt                  |
+        | `to_timestamp` treats quoted day name as a literal        | '2026-06-15 E' | "yyyy-MM-dd 'E'" |
+        | `to_timestamp` treats quoted aligned day as a literal      | '2026-06-15 F' | "yyyy-MM-dd 'F'" |
+        | `to_timestamp` treats quoted stand-alone quarter literally | '2026-06-15 q' | "yyyy-MM-dd 'q'" |
+        | `to_timestamp` treats quoted quarter as a literal         | '2026-06-15 Q' | "yyyy-MM-dd 'Q'" |
+        | `to_timestamp` treats quoted week-based year literally    | '2026-06-15 Y' | "yyyy-MM-dd 'Y'" |
+        | `to_timestamp` treats quoted day-period as a literal       | '2026-06-15 B' | "yyyy-MM-dd 'B'" |
+        | `to_timestamp` treats quoted unknown letter as a literal   | '2026-06-15 C' | "yyyy-MM-dd 'C'" |
 
   Rule: Pattern validation and error handling
 
@@ -1437,6 +1397,5 @@ Feature: datetime parsing with format strings
 
       Examples:
         | case                                              | in               | fmt         |
-        | `to_timestamp` rejects invalid pattern width      | 2026-06-15       | MMMMMM      |
         | `to_timestamp` rejects unclosed optional section  | 2026-06-15T14:30 | yyyy-MM-dd[ |
         | `to_timestamp` rejects unexpected closing bracket | 2026-06-15T14:30 | yyyy-MM-dd] |
diff --git a/python/pysail/tests/spark/function/features/datetime/to_date.feature b/python/pysail/tests/spark/function/features/datetime/to_date.feature
index 47fcea62d..d49c0e0d2 100644
--- a/python/pysail/tests/spark/function/features/datetime/to_date.feature
+++ b/python/pysail/tests/spark/function/features/datetime/to_date.feature
@@ -27,9 +27,166 @@ Feature: to_date with an argument coming from a column
         | 2016-12-31 |
         | 2016-12-31 |
 
+  Rule: Explicit NULL format semantics
+
+    Scenario: To date NULL format semantics distinguishes omitted and explicit format
+      When query
+        """
+        SELECT
+          to_date('2024-01-15') AS omitted_format,
+          to_date('2024-01-15', CAST(NULL AS STRING)) AS explicit_null_format
+        """
+      Then query result
+        | omitted_format | explicit_null_format |
+        | 2024-01-15     | NULL                 |
+
+    Scenario: To date NULL format semantics propagates a column format for a scalar value
+      When query
+        """
+        SELECT id, format, to_date('2024-01-15', format) AS result
+        FROM VALUES
+          (1, 'yyyy-MM-dd'),
+          (2, CAST(NULL AS STRING))
+          AS t(id, format)
+        ORDER BY id
+        """
+      Then query result ordered
+        | id | format     | result     |
+        | 1  | yyyy-MM-dd | 2024-01-15 |
+        | 2  | NULL       | NULL       |
+
+    Scenario: To date NULL format semantics propagates a scalar NULL format for column values
+      When query
+        """
+        SELECT id, value, to_date(value, CAST(NULL AS STRING)) AS result
+        FROM VALUES
+          (1, '2024-01-15'),
+          (2, '2024-01-16'),
+          (3, CAST(NULL AS STRING))
+          AS t(id, value)
+        ORDER BY id
+        """
+      Then query result ordered
+        | id | value      | result |
+        | 1  | 2024-01-15 | NULL   |
+        | 2  | 2024-01-16 | NULL   |
+        | 3  | NULL       | NULL   |
+
+    Scenario: To date NULL format semantics propagates paired value and format columns
+      When query
+        """
+        SELECT id, to_date(value, format) AS result
+        FROM VALUES
+          (1, '2024-01-15', 'yyyy-MM-dd'),
+          (2, '15/01/2024', 'dd/MM/yyyy'),
+          (3, '2024-01-15', CAST(NULL AS STRING)),
+          (4, CAST(NULL AS STRING), 'yyyy-MM-dd')
+          AS t(id, value, format)
+        ORDER BY id
+        """
+      Then query result ordered
+        | id | result     |
+        | 1  | 2024-01-15 |
+        | 2  | 2024-01-15 |
+        | 3  | NULL       |
+        | 4  | NULL       |
+
+    Scenario Outline: To date NULL format semantics ignores format for a  input
+      When query
+        """
+        SELECT id, format, to_date(, format) AS result
+        FROM VALUES
+          (1, 'yyyy-MM-dd'),
+          (2, CAST(NULL AS STRING)),
+          (3, 'invalid_format')
+          AS t(id, format)
+        ORDER BY id
+        """
+      Then query result ordered
+        | id | format         | result     |
+        | 1  | yyyy-MM-dd     | 2024-01-15 |
+        | 2  | NULL           | 2024-01-15 |
+        | 3  | invalid_format | 2024-01-15 |
+
+      Examples:
+        | type      | value                                |
+        | DATE      | DATE '2024-01-15'                    |
+        | TIMESTAMP | TIMESTAMP '2024-01-15 23:45:00'      |
+
+    Scenario: To date typed DATE rejects a complex format
+      When query
+        """
+        SELECT to_date(
+          DATE '2024-01-15',
+          array('yyyy-MM-dd')
+        )
+        """
+      Then query error (?i)(DATATYPE_MISMATCH\.UNEXPECTED_INPUT_TYPE|format.*string.*(array|list)|expects.*STRING.*(array|list)|requires.*STRING.*(array|list))
+
+    Scenario: To date typed DATE accepts an atomic format through string coercion
+      When query
+        """
+        SELECT to_date(DATE '2024-01-15', 123) AS result
+        """
+      Then query result
+        | result     |
+        | 2024-01-15 |
+
+    Scenario Outline: To date NULL format semantics short-circuits a bad value with ANSI 
+      Given config spark.sql.ansi.enabled = 
+      When query
+        """
+        SELECT to_date('not-a-date', CAST(NULL AS STRING)) AS result
+        """
+      Then query result
+        | result |
+        | NULL   |
+
+      Examples:
+        | ansi  |
+        | true  |
+        | false |
+
   @spark_null
   Rule: Output schema
 
+    Scenario Outline: a typed DATE with a literal format respects ANSI  nullability
+      Given config spark.sql.ansi.enabled = 
+      When query
+        """
+        SELECT to_date(DATE '2024-01-15', 'yyyy-MM-dd') AS result
+        """
+      Then query schema
+        """
+        root
+         |-- result: date (nullable = )
+        """
+
+      Examples:
+        | ansi  | nullable |
+        | false | true     |
+        | true  | false    |
+
+    Scenario: a typed DATE with a nullable format stays nullable in ANSI mode
+      Given config spark.sql.ansi.enabled = true
+      When query
+        """
+        SELECT to_date(DATE '2024-01-15', format) AS result
+        FROM VALUES
+          ('yyyy-MM-dd'),
+          (CAST(NULL AS STRING))
+          AS t(format)
+        """
+      Then query result
+        | result     |
+        | 2024-01-15 |
+        | 2024-01-15 |
+      And query schema
+        """
+        root
+         |-- result: date (nullable = true)
+        """
+
     Scenario: a non-null string literal yields a date
       When query
         """
diff --git a/python/pysail/tests/spark/function/features/datetime/to_time.feature b/python/pysail/tests/spark/function/features/datetime/to_time.feature
index ad5f788e6..d9380847a 100644
--- a/python/pysail/tests/spark/function/features/datetime/to_time.feature
+++ b/python/pysail/tests/spark/function/features/datetime/to_time.feature
@@ -2,12 +2,6 @@
 Feature: to_time (strict variant)
   Strict to_time that throws an error on invalid input.
 
-  NOTE: TIME type is a preview feature in Spark 4.x; the JVM raises
-  `[UNSUPPORTED_TIME_TYPE] The data type TIME is not supported` for
-  any TIME usage. The scenarios below reflect Sail's implementation.
-  Once Spark stabilises TIME support, remove @sail-only and update
-  expected values to match Spark JVM output. — owners to decide.
-
   Rule: Valid input parses
 
     @sail-only
@@ -92,3 +86,43 @@ Feature: to_time (strict variant)
       Then query result
         | result |
         | NULL   |
+
+  Rule: Spark Java datetime pattern contract
+
+    Background:
+      Given config spark.sql.timeType.enabled = true
+
+    Scenario: Java datetime pattern contract parses quoted literals and fractional widths with to_time
+      When query
+        """
+        SELECT
+          to_time(
+            '10B30',
+            concat('HH', chr(39), 'B', chr(39), 'mm')
+          ) AS quoted_literal,
+          to_time('10:30:45.1', 'HH:mm:ss.SSSSSS') AS short_fraction,
+          to_time(CAST(NULL AS STRING), 'HH:mm:ss') AS null_input
+        """
+      Then query schema
+        """
+        root
+         |-- quoted_literal: time(6) (nullable = true)
+         |-- short_fraction: time(6) (nullable = true)
+         |-- null_input: time(6) (nullable = true)
+        """
+      And query result
+        | quoted_literal | short_fraction | null_input |
+        | 10:30:00       | 10:30:45.1     | NULL       |
+
+    Scenario Outline: Java datetime pattern contract makes to_time reject 
+      When query
+        """
+        SELECT to_time(, )
+        """
+      Then query error 
+
+      Examples:
+        | case                                      | value                | format                         | error                                                             |
+        | a fraction wider than the pattern         | '10:30:45.1234'      | 'HH:mm:ss.SSS'                  | (?i)CANNOT_PARSE_TIME\|datetime value does not match format        |
+        | an unquoted restricted pattern letter     | '10B30'              | 'HHBmm'                         | (?i)INVALID_DATETIME_PATTERN\|invalid datetime pattern             |
+        | a pattern with an unmatched literal quote | '10:30'              | concat('HH', chr(39), 'mm')     | (?i)INVALID_DATETIME_PATTERN\|invalid datetime pattern             |
diff --git a/python/pysail/tests/spark/function/features/datetime/to_unix_timestamp.feature b/python/pysail/tests/spark/function/features/datetime/to_unix_timestamp.feature
index e3ae1295a..6913769e3 100644
--- a/python/pysail/tests/spark/function/features/datetime/to_unix_timestamp.feature
+++ b/python/pysail/tests/spark/function/features/datetime/to_unix_timestamp.feature
@@ -27,10 +27,230 @@ Feature: to_unix_timestamp with an argument coming from a column
         | 1460073600 |
         | 1460073600 |
 
+  Rule: Spark parsing, NULL format, and typed input contract
+
+    Background:
+      Given config spark.sql.session.timeZone = UTC
+
+    Scenario: Unix timestamp parsing contract uses the one-argument default format
+      Given config spark.sql.ansi.enabled = false
+      When query
+        """
+        SELECT
+          to_unix_timestamp('2024-01-15') AS date_only,
+          to_unix_timestamp('2024-01-15 01:02:03') AS full_timestamp
+        """
+      Then query result
+        | date_only | full_timestamp |
+        | NULL      | 1705280523     |
+      And query schema
+        """
+        root
+         |-- date_only: long (nullable = true)
+         |-- full_timestamp: long (nullable = true)
+        """
+
+    Scenario Outline: Unix timestamp parsing contract scalar formats have ANSI  nullability
+      Given config spark.sql.ansi.enabled = 
+      When query
+        """
+        SELECT
+          to_unix_timestamp('2024-01-15', 'yyyy-MM-dd') AS parsed,
+          to_unix_timestamp('2024-01-15', CAST(NULL AS STRING)) AS null_format
+        """
+      Then query result
+        | parsed     | null_format |
+        | 1705276800 | NULL        |
+      And query schema
+        """
+        root
+         |-- parsed: long (nullable = )
+         |-- null_format: long (nullable = true)
+        """
+
+      Examples:
+        | ansi  | parsed_nullable |
+        | true  | false           |
+        | false | true            |
+
+    Scenario: Unix timestamp parsing contract accepts a scalar value and format column
+      Given config spark.sql.ansi.enabled = false
+      When query
+        """
+        SELECT to_unix_timestamp('2024-01-15', format) AS result
+        FROM VALUES
+          (1, 'yyyy-MM-dd'),
+          (2, CAST(NULL AS STRING))
+        AS t(id, format)
+        ORDER BY id
+        """
+      Then query result ordered
+        | result     |
+        | 1705276800 |
+        | NULL       |
+      And query schema
+        """
+        root
+         |-- result: long (nullable = true)
+        """
+
+    Scenario: Unix timestamp parsing contract accepts a value column and scalar format
+      Given config spark.sql.ansi.enabled = false
+      When query
+        """
+        SELECT to_unix_timestamp(value, 'yyyy-MM-dd') AS result
+        FROM VALUES
+          (1, '2024-01-15'),
+          (2, CAST(NULL AS STRING))
+        AS t(id, value)
+        ORDER BY id
+        """
+      Then query result ordered
+        | result     |
+        | 1705276800 |
+        | NULL       |
+      And query schema
+        """
+        root
+         |-- result: long (nullable = true)
+        """
+
+    Scenario: Unix timestamp parsing contract handles value and format columns row by row with ANSI false
+      Given config spark.sql.ansi.enabled = false
+      When query
+        """
+        SELECT to_unix_timestamp(value, format) AS result
+        FROM VALUES
+          (1, '2024-01-15', 'yyyy-MM-dd'),
+          (2, '15/01/2024', 'dd/MM/yyyy'),
+          (3, 'bad-value', 'yyyy-MM-dd'),
+          (4, '2024-01-15', CAST(NULL AS STRING)),
+          (5, CAST(NULL AS STRING), 'yyyy-MM-dd')
+        AS t(id, value, format)
+        ORDER BY id
+        """
+      Then query result ordered
+        | result     |
+        | 1705276800 |
+        | 1705276800 |
+        | NULL       |
+        | NULL       |
+        | NULL       |
+      And query schema
+        """
+        root
+         |-- result: long (nullable = true)
+        """
+
+    Scenario: Unix timestamp parsing contract errors on a mixed value column with ANSI true
+      Given config spark.sql.ansi.enabled = true
+      When query
+        """
+        SELECT id, to_unix_timestamp(value, 'yyyy-MM-dd') AS result
+        FROM VALUES
+          (1, '2024-01-15'),
+          (2, 'bad-value')
+        AS t(id, value)
+        ORDER BY id
+        """
+      Then query error CANNOT_PARSE_TIMESTAMP
+
+    Scenario Outline: Unix timestamp parsing contract ignores scalar and column formats for 
+      Given config spark.sql.ansi.enabled = true
+      When query
+        """
+        SELECT
+          to_unix_timestamp(, format) AS column_format,
+          to_unix_timestamp(, CAST(NULL AS STRING)) AS scalar_null_format,
+          to_unix_timestamp(, 'invalid[') AS scalar_invalid_format
+        FROM VALUES
+          (1, 'yyyy-MM-dd'),
+          (2, CAST(NULL AS STRING)),
+          (3, 'invalid[')
+        AS t(id, format)
+        ORDER BY id
+        """
+      Then query result ordered
+        | column_format | scalar_null_format | scalar_invalid_format |
+        |       |             |               |
+        |       |             |               |
+        |       |             |               |
+      And query schema
+        """
+        root
+         |-- column_format: long (nullable = true)
+         |-- scalar_null_format: long (nullable = true)
+         |-- scalar_invalid_format: long (nullable = false)
+        """
+
+      Examples:
+        | type          | input                                | result     |
+        | DATE          | DATE '2024-01-15'                    | 1705276800 |
+        | TIMESTAMP     | TIMESTAMP '2024-01-15 01:02:03'      | 1705280523 |
+        | TIMESTAMP_NTZ | TIMESTAMP_NTZ '2024-01-15 01:02:03'  | 1705280523 |
+
+    Scenario Outline: Unix timestamp parsing contract rejects a  format for a typed DATE input
+      Given config spark.sql.ansi.enabled = true
+      When query
+        """
+        SELECT to_unix_timestamp(
+          DATE '2024-01-15',
+          
+        )
+        """
+      Then query error (?i)(DATATYPE_MISMATCH\.UNEXPECTED_INPUT_TYPE|expects.*STRING|requires.*STRING|must.*STRING)
+
+      Examples:
+        | case       | format             |
+        | atomic     | 123                |
+        | collection | array('yyyy-MM-dd') |
+
+    Scenario Outline: Unix timestamp parsing contract does not evaluate an ignored format for 
+      Given config spark.sql.session.timeZone = UTC
+      Given config spark.sql.ansi.enabled = true
+      When query
+        """
+        SELECT to_unix_timestamp(
+          ,
+          CAST(raise_error(CAST(id AS STRING)) AS STRING)
+        ) AS result
+        FROM range(1)
+        """
+      Then query result
+        | result   |
+        |  |
+
+      Examples:
+        | case       | input             | result |
+        | typed DATE | DATE '1970-01-01' | 0      |
+        | NULL       | NULL              | NULL   |
+
+    Scenario: Unix timestamp parsing contract applies the session zone to typed inputs
+      Given config spark.sql.session.timeZone = America/Los_Angeles
+      Given config spark.sql.ansi.enabled = true
+      When query
+        """
+        SELECT
+          to_unix_timestamp(DATE '2024-01-15', CAST(NULL AS STRING)) AS date_null_format,
+          to_unix_timestamp(DATE '2024-01-15', 'invalid[') AS date_invalid_format,
+          to_unix_timestamp(TIMESTAMP '2024-01-15 01:02:03', 'invalid[') AS timestamp_result,
+          to_unix_timestamp(TIMESTAMP_NTZ '2024-01-15 01:02:03', 'invalid[') AS timestamp_ntz_result
+        """
+      Then query result
+        | date_null_format | date_invalid_format | timestamp_result | timestamp_ntz_result |
+        | 1705305600       | 1705305600          | 1705309323       | 1705280523           |
+      And query schema
+        """
+        root
+         |-- date_null_format: long (nullable = true)
+         |-- date_invalid_format: long (nullable = false)
+         |-- timestamp_result: long (nullable = false)
+         |-- timestamp_ntz_result: long (nullable = false)
+        """
+
   @spark_null
   Rule: Output schema
 
-    @sail-bug
     Scenario: a non-null literal input to to_unix_timestamp yields the schema Spark declares
       When query
         """
@@ -42,7 +262,6 @@ Feature: to_unix_timestamp with an argument coming from a column
          |-- result: long (nullable = false)
         """
 
-    @sail-bug
     Scenario: a non-null column input to to_unix_timestamp yields the schema Spark declares
       When query
         """
diff --git a/python/pysail/tests/spark/function/features/datetime/try_to_time.feature b/python/pysail/tests/spark/function/features/datetime/try_to_time.feature
index 73390b9ed..4781a8caf 100644
--- a/python/pysail/tests/spark/function/features/datetime/try_to_time.feature
+++ b/python/pysail/tests/spark/function/features/datetime/try_to_time.feature
@@ -3,12 +3,6 @@ Feature: try_to_time
   Safe variant of to_time that returns NULL on parse failure
   instead of throwing an exception.
 
-  NOTE: TIME type is a preview feature in Spark 4.x; the JVM raises
-  `[UNSUPPORTED_TIME_TYPE] The data type TIME is not supported` for
-  any TIME usage. The scenarios below reflect Sail's implementation.
-  Once Spark stabilises TIME support, remove @sail-only and update
-  expected values to match Spark JVM output. — owners to decide.
-
   Rule: Single-argument form parses with default formats
 
     @sail-only
@@ -108,3 +102,65 @@ Feature: try_to_time
         | NULL     |
         | 00:00:00 |
         | NULL     |
+
+  Rule: Spark Java datetime pattern contract
+
+    Background:
+      Given config spark.sql.timeType.enabled = true
+
+    Scenario: Java datetime pattern contract applies dynamic formats and NULLs with try_to_time
+      When query
+        """
+        SELECT id, try_to_time(
+          value,
+          CASE id
+            WHEN 1 THEN concat('HH', chr(39), 'B', chr(39), 'mm')
+            WHEN 2 THEN 'HH-mm-ss.SSS'
+            WHEN 3 THEN 'HHBmm'
+            WHEN 4 THEN 'HH:mm:ss.SSS'
+            WHEN 5 THEN 'HH:mm:ss'
+            ELSE CAST(NULL AS STRING)
+          END
+        ) AS result
+        FROM VALUES
+          (1, '10B30'),
+          (2, '11-31-42.7'),
+          (3, '10B30'),
+          (4, '10:30:45.1234'),
+          (5, CAST(NULL AS STRING)),
+          (6, '12:34:56')
+          AS t(id, value)
+        ORDER BY id
+        """
+      Then query result
+        | id | result      |
+        | 1  | 10:30:00    |
+        | 2  | 11:31:42.7  |
+        | 3  | NULL        |
+        | 4  | NULL        |
+        | 5  | NULL        |
+        | 6  | NULL        |
+
+    Scenario: Java datetime pattern contract makes try_to_time suppress literal failures
+      When query
+        """
+        SELECT
+          try_to_time('10:30:45.1234', 'HH:mm:ss.SSS') AS fraction_too_wide,
+          try_to_time('10B30', 'HHBmm') AS unsupported_pattern,
+          try_to_time(
+            '10:30',
+            concat('HH', chr(39), 'mm')
+          ) AS invalid_pattern,
+          try_to_time(CAST(NULL AS STRING), 'HH:mm:ss') AS null_input
+        """
+      Then query schema
+        """
+        root
+         |-- fraction_too_wide: time(6) (nullable = true)
+         |-- unsupported_pattern: time(6) (nullable = true)
+         |-- invalid_pattern: time(6) (nullable = true)
+         |-- null_input: time(6) (nullable = true)
+        """
+      And query result
+        | fraction_too_wide | unsupported_pattern | invalid_pattern | null_input |
+        | NULL              | NULL                | NULL            | NULL       |
diff --git a/python/pysail/tests/spark/function/features/datetime/unix_timestamp.feature b/python/pysail/tests/spark/function/features/datetime/unix_timestamp.feature
index 9f8b55c38..26ea928fd 100644
--- a/python/pysail/tests/spark/function/features/datetime/unix_timestamp.feature
+++ b/python/pysail/tests/spark/function/features/datetime/unix_timestamp.feature
@@ -27,10 +27,230 @@ Feature: unix_timestamp with an argument coming from a column
         | 1460073600 |
         | 1460073600 |
 
+  Rule: Spark parsing, NULL format, and typed input contract
+
+    Background:
+      Given config spark.sql.session.timeZone = UTC
+
+    Scenario: Unix timestamp parsing contract uses the one-argument default format
+      Given config spark.sql.ansi.enabled = false
+      When query
+        """
+        SELECT
+          unix_timestamp('2024-01-15') AS date_only,
+          unix_timestamp('2024-01-15 01:02:03') AS full_timestamp
+        """
+      Then query result
+        | date_only | full_timestamp |
+        | NULL      | 1705280523     |
+      And query schema
+        """
+        root
+         |-- date_only: long (nullable = true)
+         |-- full_timestamp: long (nullable = true)
+        """
+
+    Scenario Outline: Unix timestamp parsing contract scalar formats have ANSI  nullability
+      Given config spark.sql.ansi.enabled = 
+      When query
+        """
+        SELECT
+          unix_timestamp('2024-01-15', 'yyyy-MM-dd') AS parsed,
+          unix_timestamp('2024-01-15', CAST(NULL AS STRING)) AS null_format
+        """
+      Then query result
+        | parsed     | null_format |
+        | 1705276800 | NULL        |
+      And query schema
+        """
+        root
+         |-- parsed: long (nullable = )
+         |-- null_format: long (nullable = true)
+        """
+
+      Examples:
+        | ansi  | parsed_nullable |
+        | true  | false           |
+        | false | true            |
+
+    Scenario: Unix timestamp parsing contract accepts a scalar value and format column
+      Given config spark.sql.ansi.enabled = false
+      When query
+        """
+        SELECT unix_timestamp('2024-01-15', format) AS result
+        FROM VALUES
+          (1, 'yyyy-MM-dd'),
+          (2, CAST(NULL AS STRING))
+        AS t(id, format)
+        ORDER BY id
+        """
+      Then query result ordered
+        | result     |
+        | 1705276800 |
+        | NULL       |
+      And query schema
+        """
+        root
+         |-- result: long (nullable = true)
+        """
+
+    Scenario: Unix timestamp parsing contract accepts a value column and scalar format
+      Given config spark.sql.ansi.enabled = false
+      When query
+        """
+        SELECT unix_timestamp(value, 'yyyy-MM-dd') AS result
+        FROM VALUES
+          (1, '2024-01-15'),
+          (2, CAST(NULL AS STRING))
+        AS t(id, value)
+        ORDER BY id
+        """
+      Then query result ordered
+        | result     |
+        | 1705276800 |
+        | NULL       |
+      And query schema
+        """
+        root
+         |-- result: long (nullable = true)
+        """
+
+    Scenario: Unix timestamp parsing contract handles value and format columns row by row with ANSI false
+      Given config spark.sql.ansi.enabled = false
+      When query
+        """
+        SELECT unix_timestamp(value, format) AS result
+        FROM VALUES
+          (1, '2024-01-15', 'yyyy-MM-dd'),
+          (2, '15/01/2024', 'dd/MM/yyyy'),
+          (3, 'bad-value', 'yyyy-MM-dd'),
+          (4, '2024-01-15', CAST(NULL AS STRING)),
+          (5, CAST(NULL AS STRING), 'yyyy-MM-dd')
+        AS t(id, value, format)
+        ORDER BY id
+        """
+      Then query result ordered
+        | result     |
+        | 1705276800 |
+        | 1705276800 |
+        | NULL       |
+        | NULL       |
+        | NULL       |
+      And query schema
+        """
+        root
+         |-- result: long (nullable = true)
+        """
+
+    Scenario: Unix timestamp parsing contract errors on a mixed value column with ANSI true
+      Given config spark.sql.ansi.enabled = true
+      When query
+        """
+        SELECT id, unix_timestamp(value, 'yyyy-MM-dd') AS result
+        FROM VALUES
+          (1, '2024-01-15'),
+          (2, 'bad-value')
+        AS t(id, value)
+        ORDER BY id
+        """
+      Then query error CANNOT_PARSE_TIMESTAMP
+
+    Scenario Outline: Unix timestamp parsing contract ignores scalar and column formats for 
+      Given config spark.sql.ansi.enabled = true
+      When query
+        """
+        SELECT
+          unix_timestamp(, format) AS column_format,
+          unix_timestamp(, CAST(NULL AS STRING)) AS scalar_null_format,
+          unix_timestamp(, 'invalid[') AS scalar_invalid_format
+        FROM VALUES
+          (1, 'yyyy-MM-dd'),
+          (2, CAST(NULL AS STRING)),
+          (3, 'invalid[')
+        AS t(id, format)
+        ORDER BY id
+        """
+      Then query result ordered
+        | column_format | scalar_null_format | scalar_invalid_format |
+        |       |             |               |
+        |       |             |               |
+        |       |             |               |
+      And query schema
+        """
+        root
+         |-- column_format: long (nullable = true)
+         |-- scalar_null_format: long (nullable = true)
+         |-- scalar_invalid_format: long (nullable = false)
+        """
+
+      Examples:
+        | type          | input                                | result     |
+        | DATE          | DATE '2024-01-15'                    | 1705276800 |
+        | TIMESTAMP     | TIMESTAMP '2024-01-15 01:02:03'      | 1705280523 |
+        | TIMESTAMP_NTZ | TIMESTAMP_NTZ '2024-01-15 01:02:03'  | 1705280523 |
+
+    Scenario Outline: Unix timestamp parsing contract rejects a  format for a typed DATE input
+      Given config spark.sql.ansi.enabled = true
+      When query
+        """
+        SELECT unix_timestamp(
+          DATE '2024-01-15',
+          
+        )
+        """
+      Then query error (?i)(DATATYPE_MISMATCH\.UNEXPECTED_INPUT_TYPE|expects.*STRING|requires.*STRING|must.*STRING)
+
+      Examples:
+        | case       | format             |
+        | atomic     | 123                |
+        | collection | array('yyyy-MM-dd') |
+
+    Scenario Outline: Unix timestamp parsing contract does not evaluate an ignored format for 
+      Given config spark.sql.session.timeZone = UTC
+      Given config spark.sql.ansi.enabled = true
+      When query
+        """
+        SELECT unix_timestamp(
+          ,
+          CAST(raise_error(CAST(id AS STRING)) AS STRING)
+        ) AS result
+        FROM range(1)
+        """
+      Then query result
+        | result   |
+        |  |
+
+      Examples:
+        | case       | input             | result |
+        | typed DATE | DATE '1970-01-01' | 0      |
+        | NULL       | NULL              | NULL   |
+
+    Scenario: Unix timestamp parsing contract applies the session zone to typed inputs
+      Given config spark.sql.session.timeZone = America/Los_Angeles
+      Given config spark.sql.ansi.enabled = true
+      When query
+        """
+        SELECT
+          unix_timestamp(DATE '2024-01-15', CAST(NULL AS STRING)) AS date_null_format,
+          unix_timestamp(DATE '2024-01-15', 'invalid[') AS date_invalid_format,
+          unix_timestamp(TIMESTAMP '2024-01-15 01:02:03', 'invalid[') AS timestamp_result,
+          unix_timestamp(TIMESTAMP_NTZ '2024-01-15 01:02:03', 'invalid[') AS timestamp_ntz_result
+        """
+      Then query result
+        | date_null_format | date_invalid_format | timestamp_result | timestamp_ntz_result |
+        | 1705305600       | 1705305600          | 1705309323       | 1705280523           |
+      And query schema
+        """
+        root
+         |-- date_null_format: long (nullable = true)
+         |-- date_invalid_format: long (nullable = false)
+         |-- timestamp_result: long (nullable = false)
+         |-- timestamp_ntz_result: long (nullable = false)
+        """
+
   @spark_null
   Rule: Output schema
 
-    @sail-bug
     Scenario: a non-null string literal yields a bigint
       When query
         """
diff --git a/python/pysail/tests/spark/function/features/xml/from_xml.feature b/python/pysail/tests/spark/function/features/xml/from_xml.feature
index 1cd6dee4a..984ee8ccf 100644
--- a/python/pysail/tests/spark/function/features/xml/from_xml.feature
+++ b/python/pysail/tests/spark/function/features/xml/from_xml.feature
@@ -217,6 +217,57 @@ Feature: from_xml parses an XML string into a struct value
         | result              |
         | 2026-06-17 18:00:00 |
 
+    Scenario: From XML Java datetime pattern contract for quoted literals and fractions
+      When query
+        """
+        SELECT
+          CAST(from_xml(
+            '
      2026B06B17
      ',
+            'd DATE',
+            map(
+              'dateFormat',
+              concat(
+                'yyyy', chr(39), 'B', chr(39),
+                'MM', chr(39), 'B', chr(39), 'dd'
+              )
+            )
+          ).d AS STRING) AS date_result,
+          CAST(from_xml(
+            '
      2026-06-17 18:00:01.1
      ',
+            'ts TIMESTAMP',
+            map('timestampFormat', 'yyyy-MM-dd HH:mm:ss.SSSSSS')
+          ).ts AS STRING) AS timestamp_result
+        """
+      Then query result
+        | date_result | timestamp_result      |
+        | 2026-06-17  | 2026-06-17 18:00:01.1 |
+
+    Scenario: From XML Java datetime pattern contract for timestamp zones
+      Given config spark.sql.session.timeZone = America/Los_Angeles
+      When query
+        """
+        SELECT
+          CAST(from_xml(
+            '
      2026-06-17T18:00:00+02:00
      ',
+            'ts TIMESTAMP',
+            map(
+              'timestampFormat',
+              concat('yyyy-MM-dd', chr(39), 'T', chr(39), 'HH:mm:ssXXX')
+            )
+          ).ts AS STRING) AS timestamp_ltz,
+          CAST(from_xml(
+            '
      2026-06-17T18:00:00
      ',
+            'ts TIMESTAMP_NTZ',
+            map(
+              'timestampNTZFormat',
+              concat('yyyy-MM-dd', chr(39), 'T', chr(39), 'HH:mm:ss')
+            )
+          ).ts AS STRING) AS timestamp_ntz
+        """
+      Then query result
+        | timestamp_ltz       | timestamp_ntz       |
+        | 2026-06-17 09:00:00 | 2026-06-17 18:00:00 |
+
   Rule: XML entity unescaping
 
     Scenario Outline: Entity: 

From f6bdd9567b8657f1331ddc39a0aa501c0cde7470 Mon Sep 17 00:00:00 2001
From: Xiaolong Liang 
Date: Thu, 30 Jul 2026 18:55:42 +0800
Subject: [PATCH 100/131] fix: allow Iceberg REST create-and-write with access
 session hints (#2292)

---
 crates/sail-catalog-iceberg/src/provider.rs   |  82 +++++++---
 .../spark/catalog/iceberg_rest/conftest.py    | 144 +++++++++++++++++-
 .../catalog/iceberg_rest/test_lakekeeper.py   |  78 ++++++++++
 3 files changed, 279 insertions(+), 25 deletions(-)
 create mode 100644 python/pysail/tests/spark/catalog/iceberg_rest/test_lakekeeper.py

diff --git a/crates/sail-catalog-iceberg/src/provider.rs b/crates/sail-catalog-iceberg/src/provider.rs
index 95397ee06..2a1069f3b 100644
--- a/crates/sail-catalog-iceberg/src/provider.rs
+++ b/crates/sail-catalog-iceberg/src/provider.rs
@@ -379,27 +379,36 @@ impl IcebergRestCatalogProvider {
         catalog_config: &CatalogConfig<'_>,
         result: &crate::r#gen::LoadTableResult,
     ) -> CatalogResult<()> {
-        let rest_session =
-            Self::rest_table_session_ref(catalog, database, table, catalog_config, result)?;
-        let mut requirements = Vec::new();
-        if rest_session.scan_planning_mode.as_deref() == Some("server") {
-            requirements.push("server-side scan planning");
+        let scan_planning_mode =
+            Self::effective_scan_planning_mode(result.config.as_ref(), catalog_config)?;
+        if scan_planning_mode.as_deref() == Some("server") {
+            return Err(CatalogError::UnsupportedCapability(
+                "Iceberg REST access session requirements returned by create_table are not supported for create+write yet: server-side scan planning".to_string(),
+            ));
         }
-        if rest_session.remote_signing_enabled {
-            requirements.push("remote signing");
+
+        let mut configured_storage_fallbacks = Vec::new();
+        if Self::remote_signing_enabled(result.config.as_ref(), catalog_config) {
+            configured_storage_fallbacks.push("remote signing");
         }
-        if rest_session.storage_credential_count > 0 {
-            requirements.push("vended credentials");
+        if result
+            .storage_credentials
+            .as_ref()
+            .is_some_and(|credentials| !credentials.is_empty())
+        {
+            configured_storage_fallbacks.push("vended credentials");
         }
-
-        if requirements.is_empty() {
-            Ok(())
-        } else {
-            Err(CatalogError::UnsupportedCapability(format!(
-                "Iceberg REST access session requirements returned by create_table are not supported for create+write yet: {}",
-                requirements.join(", ")
-            )))
+        if !configured_storage_fallbacks.is_empty() {
+            log::warn!(
+                "Iceberg REST catalog {} create_table for {}.{} returned {}; using configured object-store credentials for create+write",
+                catalog,
+                quote_namespace_if_needed(database),
+                quote_name_if_needed(table),
+                configured_storage_fallbacks.join(", "),
+            );
         }
+
+        Ok(())
     }
 
     /// Converts an Iceberg REST API table load result into a catalog `TableStatus`.
@@ -2039,7 +2048,7 @@ mod tests {
         }
     }
 
-    fn create_table_response_with_access_session_requirements() -> serde_json::Value {
+    fn create_table_response_with_access_session_hints() -> serde_json::Value {
         serde_json::json!({
             "metadata-location": "s3://bucket/table/metadata/v1.metadata.json",
             "metadata": {
@@ -2063,7 +2072,6 @@ mod tests {
                 ]
             },
             "config": {
-                "scan-planning-mode": "server",
                 "s3.remote-signing-enabled": "true"
             },
             "storage-credentials": [
@@ -2078,6 +2086,12 @@ mod tests {
         })
     }
 
+    fn create_table_response_with_server_side_scan_planning() -> serde_json::Value {
+        let mut result = create_table_response_with_access_session_hints();
+        result["config"]["scan-planning-mode"] = serde_json::json!("server");
+        result
+    }
+
     async fn load_merged_test_config(
         defaults: HashMap,
         mut client_props: HashMap,
@@ -2957,13 +2971,33 @@ mod tests {
     }
 
     #[tokio::test]
-    async fn create_table_rejects_rest_access_session_requirements() {
+    async fn create_table_allows_rest_access_session_hints() {
+        let ctx = TestContext::new(Some("test")).await;
+        let namespace = Namespace::try_from(vec!["db1".to_string()]).unwrap();
+
+        ctx.mock_post_json(
+            &ctx.path("/namespaces/db1/tables"),
+            create_table_response_with_access_session_hints(),
+        )
+        .await;
+
+        let status = ctx
+            .catalog
+            .create_table(&namespace, "table1", simple_create_table_options())
+            .await
+            .unwrap();
+
+        assert_eq!(status.name, "table1");
+    }
+
+    #[tokio::test]
+    async fn create_table_rejects_server_side_scan_planning() {
         let ctx = TestContext::new(Some("test")).await;
         let namespace = Namespace::try_from(vec!["db1".to_string()]).unwrap();
 
         ctx.mock_post_json(
             &ctx.path("/namespaces/db1/tables"),
-            create_table_response_with_access_session_requirements(),
+            create_table_response_with_server_side_scan_planning(),
         )
         .await;
 
@@ -2974,17 +3008,17 @@ mod tests {
             .unwrap_err();
 
         assert!(matches!(err, CatalogError::UnsupportedCapability(_)));
-        assert!(err.to_string().contains("Iceberg REST access session"));
+        assert!(err.to_string().contains("server-side scan planning"));
     }
 
     #[tokio::test]
-    async fn metadata_only_create_table_allows_rest_access_session_requirements() {
+    async fn metadata_only_create_table_allows_server_side_scan_planning() {
         let ctx = TestContext::new(Some("test")).await;
         let namespace = Namespace::try_from(vec!["db1".to_string()]).unwrap();
 
         ctx.mock_post_json(
             &ctx.path("/namespaces/db1/tables"),
-            create_table_response_with_access_session_requirements(),
+            create_table_response_with_server_side_scan_planning(),
         )
         .await;
 
diff --git a/python/pysail/tests/spark/catalog/iceberg_rest/conftest.py b/python/pysail/tests/spark/catalog/iceberg_rest/conftest.py
index 5e9da1ec8..3ad930412 100644
--- a/python/pysail/tests/spark/catalog/iceberg_rest/conftest.py
+++ b/python/pysail/tests/spark/catalog/iceberg_rest/conftest.py
@@ -10,8 +10,10 @@
 from typing import TYPE_CHECKING
 
 import pytest
+import requests
 from testcontainers.core.container import DockerContainer
 from testcontainers.core.network import Network
+from testcontainers.core.wait_strategies import LogMessageWaitStrategy
 from testcontainers.core.waiting_utils import wait_for_logs
 
 from pysail.testing.spark.session import spark_connect_server
@@ -20,6 +22,9 @@
     from collections.abc import Generator
     from pathlib import Path
 
+LAKEKEEPER_IMAGE = "quay.io/lakekeeper/catalog:v0.12.1"
+LAKEKEEPER_DATABASE_URL = "postgresql://postgres:postgres@lakekeeper-db:5432/postgres"
+LAKEKEEPER_PROJECT_ID = "00000000-0000-0000-0000-000000000000"
 NESSIE_NAMESPACE_SEPARATOR = "-"
 
 
@@ -54,9 +59,9 @@ def seaweedfs_container(
         .with_exposed_ports(8333)
         .with_network(docker_network)
         .with_network_aliases("seaweedfs")
+        .waiting_for(LogMessageWaitStrategy("Start Seaweed S3 API").with_startup_timeout(120))
     )
     container.start()
-    wait_for_logs(container, "Start Seaweed S3 API", timeout=120)
     yield container
     container.stop()
 
@@ -102,6 +107,143 @@ def _create_s3_bucket(seaweedfs_host_endpoint: str) -> None:
             return
 
 
+def lakekeeper_command_container(
+    docker_network: Network,
+    command: str,
+) -> DockerContainer:
+    """Configure a Lakekeeper container for the test PostgreSQL database."""
+    return (
+        DockerContainer(LAKEKEEPER_IMAGE)
+        .with_command([command])
+        .with_env("LAKEKEEPER__PG_ENCRYPTION_KEY", "This-is-NOT-Secure!")
+        .with_env("LAKEKEEPER__PG_DATABASE_URL_READ", LAKEKEEPER_DATABASE_URL)
+        .with_env("LAKEKEEPER__PG_DATABASE_URL_WRITE", LAKEKEEPER_DATABASE_URL)
+        .with_network(docker_network)
+    )
+
+
+@pytest.fixture(scope="module")
+def lakekeeper_database_container(
+    docker_network: Network,
+) -> Generator[DockerContainer, None, None]:
+    """Start the PostgreSQL database used by Lakekeeper."""
+    container = (
+        DockerContainer("postgres:17")
+        .with_env("POSTGRES_PASSWORD", "postgres")
+        .with_network(docker_network)
+        .with_network_aliases("lakekeeper-db")
+        .waiting_for(LogMessageWaitStrategy("database system is ready to accept connections").with_startup_timeout(120))
+    )
+    container.start()
+    try:
+        yield container
+    finally:
+        container.stop()
+
+
+@pytest.fixture(scope="module")
+def _lakekeeper_migration(
+    docker_network: Network,
+    lakekeeper_database_container: DockerContainer,  # noqa: ARG001
+) -> None:
+    """Apply Lakekeeper database migrations before starting the server."""
+    container = lakekeeper_command_container(docker_network, "migrate")
+    container.start()
+    try:
+        result = container.get_wrapped_container().wait(timeout=120)
+        if result["StatusCode"] != 0:
+            stdout, stderr = container.get_logs()
+            message = (stdout + stderr).decode(errors="replace")
+            msg = f"Lakekeeper database migration failed:\n{message}"
+            raise RuntimeError(msg)
+    finally:
+        container.stop()
+
+
+@pytest.fixture(scope="module")
+def lakekeeper_container(
+    docker_network: Network,
+    seaweedfs_container: DockerContainer,  # noqa: ARG001
+    _lakekeeper_migration: None,
+) -> Generator[DockerContainer, None, None]:
+    """Start Lakekeeper after its database has been migrated."""
+    container = (
+        lakekeeper_command_container(docker_network, "serve")
+        .with_exposed_ports(8181)
+        .with_network_aliases("lakekeeper")
+        .waiting_for(LogMessageWaitStrategy("Starting server on 0.0.0.0:8181").with_startup_timeout(120))
+    )
+    container.start()
+    try:
+        yield container
+    finally:
+        container.stop()
+
+
+@pytest.fixture(scope="module")
+def lakekeeper_endpoint(lakekeeper_container: DockerContainer) -> str:
+    """Return a healthy host-accessible Lakekeeper endpoint."""
+    host = lakekeeper_container.get_container_host_ip()
+    port = lakekeeper_container.get_exposed_port(8181)
+    endpoint = f"http://{host}:{port}"
+
+    for attempt in range(30):
+        try:
+            response = requests.get(f"{endpoint}/health", timeout=10)
+            response.raise_for_status()
+        except requests.RequestException:
+            if attempt == 29:  # noqa: PLR2004
+                raise
+            time.sleep(1)
+        else:
+            return endpoint
+    msg = "unreachable"
+    raise AssertionError(msg)
+
+
+@pytest.fixture(scope="module")
+def lakekeeper_warehouse_id(
+    lakekeeper_endpoint: str,
+    seaweedfs_internal_endpoint: str,
+    _create_s3_bucket: None,
+) -> str:
+    """Bootstrap Lakekeeper and create the S3-compatible test warehouse."""
+    bootstrap = requests.post(
+        f"{lakekeeper_endpoint}/management/v1/bootstrap",
+        json={"accept-terms-of-use": True},
+        timeout=30,
+    )
+    bootstrap.raise_for_status()
+
+    warehouse = requests.post(
+        f"{lakekeeper_endpoint}/management/v1/warehouse",
+        json={
+            "warehouse-name": "demo",
+            "project-id": LAKEKEEPER_PROJECT_ID,
+            "storage-profile": {
+                "type": "s3",
+                "bucket": "icebergdata",
+                "key-prefix": "lakekeeper",
+                "endpoint": seaweedfs_internal_endpoint,
+                "region": "us-east-1",
+                "path-style-access": True,
+                "flavor": "s3-compat",
+                "sts-enabled": False,
+                "remote-signing-enabled": True,
+            },
+            "storage-credential": {
+                "type": "s3",
+                "credential-type": "access-key",
+                "access-key-id": "admin",
+                "secret-access-key": "password",
+            },
+        },
+        timeout=30,
+    )
+    warehouse.raise_for_status()
+    return warehouse.json()["warehouse-id"]
+
+
 @pytest.fixture(scope="module")
 def iceberg_rest_container(
     docker_network: Network,
diff --git a/python/pysail/tests/spark/catalog/iceberg_rest/test_lakekeeper.py b/python/pysail/tests/spark/catalog/iceberg_rest/test_lakekeeper.py
new file mode 100644
index 000000000..c8e107315
--- /dev/null
+++ b/python/pysail/tests/spark/catalog/iceberg_rest/test_lakekeeper.py
@@ -0,0 +1,78 @@
+from __future__ import annotations
+
+import urllib.parse
+from typing import TYPE_CHECKING
+
+import pytest
+import requests
+
+from pysail.testing.spark.session import spark_connect_server
+
+if TYPE_CHECKING:
+    from collections.abc import Generator
+
+    from pyspark.sql import SparkSession
+
+NAMESPACE = "lakekeeper_access_session_test"
+TABLE = f"sail.{NAMESPACE}.remote_signing_t"
+
+
+@pytest.fixture(scope="module")
+def remote(
+    lakekeeper_endpoint: str,
+    lakekeeper_warehouse_id: str,
+    seaweedfs_host_endpoint: str,
+) -> Generator[str, None, None]:
+    """Start Sail with the Lakekeeper-backed Iceberg REST catalog."""
+    del lakekeeper_warehouse_id
+    catalog_config = f'[{{name="sail", type="iceberg-rest", uri="{lakekeeper_endpoint}/catalog", warehouse="demo"}}]'
+    with spark_connect_server(
+        envs={
+            "SAIL_CATALOG__LIST": catalog_config,
+            "AWS_ACCESS_KEY_ID": "admin",
+            "AWS_SECRET_ACCESS_KEY": "password",
+            "AWS_REGION": "us-east-1",
+            "AWS_ENDPOINT": seaweedfs_host_endpoint,
+            "AWS_VIRTUAL_HOSTED_STYLE_REQUEST": "false",
+            "AWS_ALLOW_HTTP": "true",
+        },
+    ) as server:
+        yield server.remote
+
+
+@pytest.fixture(scope="module", autouse=True)
+def namespace(spark: SparkSession) -> Generator[None, None, None]:
+    spark.sql(f"CREATE NAMESPACE IF NOT EXISTS sail.{NAMESPACE}")
+    yield
+    spark.sql(f"DROP NAMESPACE IF EXISTS sail.{NAMESPACE} CASCADE")
+
+
+def _load_lakekeeper_table(
+    lakekeeper_endpoint: str,
+    lakekeeper_warehouse_id: str,
+) -> dict:
+    namespace = urllib.parse.quote(NAMESPACE, safe="")
+    table = urllib.parse.quote(TABLE.rsplit(".", 1)[1], safe="")
+    response = requests.get(
+        f"{lakekeeper_endpoint}/catalog/v1/{lakekeeper_warehouse_id}/namespaces/{namespace}/tables/{table}",
+        timeout=30,
+    )
+    response.raise_for_status()
+    return response.json()
+
+
+def test_create_write_uses_configured_credentials_with_lakekeeper_session_hints(
+    spark: SparkSession,
+    lakekeeper_endpoint: str,
+    lakekeeper_warehouse_id: str,
+) -> None:
+    spark.sql(f"DROP TABLE IF EXISTS {TABLE}")
+    source = spark.createDataFrame([(1, "a"), (2, "b")], ["id", "name"])
+    source.writeTo(TABLE).using("iceberg").create()
+
+    rows = spark.table(TABLE).orderBy("id").collect()
+    assert [(row["id"], row["name"]) for row in rows] == [(1, "a"), (2, "b")]
+
+    table = _load_lakekeeper_table(lakekeeper_endpoint, lakekeeper_warehouse_id)
+    assert table["config"]["s3.remote-signing-enabled"] == "true"
+    assert table["storage-credentials"]

From ed524b6455fc8c5bef2717103cdc3f3fcd8e94e1 Mon Sep 17 00:00:00 2001
From: Heran Lin 
Date: Thu, 30 Jul 2026 21:41:17 +0800
Subject: [PATCH 101/131] fix: stop driver server properly (#2326)

---
 crates/sail-cli/src/flight/server.rs          |  2 +-
 crates/sail-execution/src/driver/registry.rs  | 25 +++++++++++-
 crates/sail-flight/src/entrypoint.rs          | 38 +++++++++++++++++++
 crates/sail-flight/src/lib.rs                 | 33 +---------------
 crates/sail-python/src/flight/server.rs       |  2 +-
 .../src/session_manager/actor/core.rs         | 20 +++++++++-
 .../src/session_manager/actor/mod.rs          |  1 +
 .../sail-session/src/session_manager/event.rs |  7 +++-
 .../sail-session/src/session_manager/mod.rs   | 13 +++++++
 crates/sail-spark-connect/src/entrypoint.rs   | 33 +++++++++-------
 .../spark/execution/test_driver_server.py     | 19 ++++++++++
 11 files changed, 140 insertions(+), 53 deletions(-)
 create mode 100644 crates/sail-flight/src/entrypoint.rs

diff --git a/crates/sail-cli/src/flight/server.rs b/crates/sail-cli/src/flight/server.rs
index 22ac0fbe5..9b2a19cca 100644
--- a/crates/sail-cli/src/flight/server.rs
+++ b/crates/sail-cli/src/flight/server.rs
@@ -33,7 +33,7 @@ pub fn run_flight_server(ip: IpAddr, port: u16) -> Result<(), Box ExecutionResult<()> {
-        self.send(DriverEvent::Shutdown { history: None })
+        // A closed channel means that the driver actor has already stopped.
+        // Shutdown is intentionally idempotent, so this is still a success.
+        let _ = self.send(DriverEvent::Shutdown { history: None }).await;
+        Ok(())
+    }
+
+    /// Stop the driver and wait for its shutdown hook to complete.
+    pub async fn shutdown_and_wait(&self) -> ExecutionResult<()> {
+        let (tx, rx) = oneshot::channel();
+        if self
+            .send(DriverEvent::Shutdown { history: Some(tx) })
             .await
-            .map_err(ExecutionError::from)
+            .is_ok()
+        {
+            // A closed result channel means another shutdown request won the race.
+            // In either case, the driver actor is no longer running.
+            let _ = rx.await;
+        }
+        Ok(())
     }
 }
 
@@ -67,6 +84,10 @@ impl DriverRegistry {
             .cloned()
             .ok_or_else(|| ExecutionError::InvalidArgument(format!("driver {driver_id} not found")))
     }
+
+    pub fn drain(&mut self) -> impl Iterator + '_ {
+        self.drivers.drain()
+    }
 }
 
 #[async_trait]
diff --git a/crates/sail-flight/src/entrypoint.rs b/crates/sail-flight/src/entrypoint.rs
new file mode 100644
index 000000000..8cb8bd95a
--- /dev/null
+++ b/crates/sail-flight/src/entrypoint.rs
@@ -0,0 +1,38 @@
+use std::future::Future;
+use std::sync::Arc;
+
+use arrow_flight::flight_service_server::FlightServiceServer;
+use sail_common::config::AppConfig;
+use sail_common::runtime::RuntimeHandle;
+use sail_server::{ServerBuilder, ServerBuilderOptions};
+use tokio::net::TcpListener;
+
+use crate::service::SailFlightSqlService;
+use crate::session::create_flight_session_manager;
+
+pub async fn serve(
+    listener: TcpListener,
+    signal: F,
+    config: Arc,
+    runtime: RuntimeHandle,
+) -> Result<(), Box>
+where
+    F: Future,
+{
+    let session_manager = create_flight_session_manager(config, runtime).await?;
+    let result = {
+        let service = SailFlightSqlService::new(session_manager.clone());
+        let flight_service = FlightServiceServer::new(service);
+
+        let builder = ServerBuilder::new("flight_sql", ServerBuilderOptions::default())
+            .add_service(flight_service, None)
+            .await;
+
+        builder
+            .serve(listener, signal)
+            .await
+            .map_err(|e| std::io::Error::other(e.to_string()))
+    };
+    session_manager.shutdown().await?;
+    result.map_err(Into::into)
+}
diff --git a/crates/sail-flight/src/lib.rs b/crates/sail-flight/src/lib.rs
index 54bc439f0..55243bce0 100644
--- a/crates/sail-flight/src/lib.rs
+++ b/crates/sail-flight/src/lib.rs
@@ -1,37 +1,6 @@
+pub mod entrypoint;
 pub mod error;
 mod metrics;
 pub mod service;
 pub mod session;
 mod state;
-
-use std::future::Future;
-use std::sync::Arc;
-
-use arrow_flight::flight_service_server::FlightServiceServer;
-use sail_common::config::AppConfig;
-use sail_common::runtime::RuntimeHandle;
-use sail_server::{ServerBuilder, ServerBuilderOptions};
-use service::SailFlightSqlService;
-use session::create_flight_session_manager;
-use tokio::net::TcpListener;
-
-pub async fn serve(
-    listener: TcpListener,
-    signal: F,
-    config: Arc,
-    runtime: RuntimeHandle,
-) -> Result<(), Box>
-where
-    F: Future,
-{
-    let session_manager = create_flight_session_manager(config, runtime).await?;
-    let service = SailFlightSqlService::new(session_manager);
-    let flight_service = FlightServiceServer::new(service);
-
-    let builder = ServerBuilder::new("flight_sql", ServerBuilderOptions::default())
-        .add_service(flight_service, None)
-        .await;
-
-    builder.serve(listener, signal).await?;
-    Ok(())
-}
diff --git a/crates/sail-python/src/flight/server.rs b/crates/sail-python/src/flight/server.rs
index a68ad0d43..24504afc2 100644
--- a/crates/sail-python/src/flight/server.rs
+++ b/crates/sail-python/src/flight/server.rs
@@ -8,7 +8,7 @@ use pyo3::exceptions::{PyRuntimeError, PyValueError};
 use pyo3::prelude::*;
 use sail_common::config::AppConfig;
 use sail_common::runtime::RuntimeHandle;
-use sail_flight::serve;
+use sail_flight::entrypoint::serve;
 use tokio::net::TcpListener;
 use tokio::runtime::Handle;
 use tokio::sync::oneshot::{Receiver, Sender};
diff --git a/crates/sail-session/src/session_manager/actor/core.rs b/crates/sail-session/src/session_manager/actor/core.rs
index 65d32470b..ef4b1c439 100644
--- a/crates/sail-session/src/session_manager/actor/core.rs
+++ b/crates/sail-session/src/session_manager/actor/core.rs
@@ -1,7 +1,7 @@
 use std::sync::Arc;
 
 use indexmap::IndexMap;
-use log::info;
+use log::{info, warn};
 use sail_execution::driver::{DriverHandle, DriverRegistryAccessor};
 use sail_execution::error::{ExecutionError, ExecutionResult};
 use sail_execution::{DriverId, IdGenerator};
@@ -51,6 +51,7 @@ impl Actor for SessionManagerActor {
             drivers: Default::default(),
             gateway,
             driver_id_generator: IdGenerator::new(),
+            shutdown_notifier: None,
         }
     }
 
@@ -91,13 +92,28 @@ impl Actor for SessionManagerActor {
             SessionManagerEvent::GetDriver { driver_id, result } => {
                 self.handle_get_driver(driver_id, result)
             }
+            SessionManagerEvent::Shutdown { result } => {
+                self.shutdown_notifier = Some(result);
+                ActorAction::Stop
+            }
         }
     }
 
-    async fn stop(self, _ctx: &mut ActorContext) {
+    async fn stop(mut self, _ctx: &mut ActorContext) {
+        // Keep the gateway available while drivers stop. Graceful gateway shutdown waits for
+        // active task stream connections, which are owned by the drivers.
+        let drivers = self.drivers.drain().collect::>();
+        for (driver_id, driver) in drivers {
+            if let Err(e) = driver.shutdown_and_wait().await {
+                warn!("failed to shut down driver {driver_id}: {e}");
+            }
+        }
         if let Some(mut gateway) = self.gateway {
             gateway.stop().await;
             info!("driver server has stopped");
         }
+        if let Some(result) = self.shutdown_notifier {
+            let _ = result.send(());
+        }
     }
 }
diff --git a/crates/sail-session/src/session_manager/actor/mod.rs b/crates/sail-session/src/session_manager/actor/mod.rs
index ca9a71210..b1c772762 100644
--- a/crates/sail-session/src/session_manager/actor/mod.rs
+++ b/crates/sail-session/src/session_manager/actor/mod.rs
@@ -16,4 +16,5 @@ pub struct SessionManagerActor {
     drivers: DriverRegistry,
     gateway: Option,
     driver_id_generator: IdGenerator,
+    shutdown_notifier: Option>,
 }
diff --git a/crates/sail-session/src/session_manager/event.rs b/crates/sail-session/src/session_manager/event.rs
index 45267dcf9..f40b35739 100644
--- a/crates/sail-session/src/session_manager/event.rs
+++ b/crates/sail-session/src/session_manager/event.rs
@@ -41,6 +41,9 @@ pub enum SessionManagerEvent {
         driver_id: DriverId,
         result: oneshot::Sender>,
     },
+    Shutdown {
+        result: oneshot::Sender<()>,
+    },
 }
 
 pub struct SessionHistory {
@@ -57,6 +60,7 @@ impl SpanAssociation for SessionManagerEvent {
             SessionManagerEvent::SetSessionFailure { .. } => "SetSessionFailure",
             SessionManagerEvent::ObserveState { .. } => "ObserveState",
             SessionManagerEvent::GetDriver { .. } => "GetDriver",
+            SessionManagerEvent::Shutdown { .. } => "Shutdown",
         };
         name.into()
     }
@@ -90,7 +94,8 @@ impl SpanAssociation for SessionManagerEvent {
             } => {
                 p.push((SpanAttribute::CLUSTER_DRIVER_ID, driver_id.to_string()));
             }
-            SessionManagerEvent::ObserveState { observer: _ } => {}
+            SessionManagerEvent::ObserveState { observer: _ }
+            | SessionManagerEvent::Shutdown { .. } => {}
         }
         p.into_iter().map(|(k, v)| (k.into(), v.into()))
     }
diff --git a/crates/sail-session/src/session_manager/mod.rs b/crates/sail-session/src/session_manager/mod.rs
index a5f953f9c..ce5398718 100644
--- a/crates/sail-session/src/session_manager/mod.rs
+++ b/crates/sail-session/src/session_manager/mod.rs
@@ -25,6 +25,7 @@ pub use crate::session_manager::options::SessionManagerOptions;
 pub type ServerSessionFactoryFn =
     fn(Arc, RuntimeHandle) -> Box>;
 
+#[derive(Clone)]
 pub struct SessionManager {
     handle: ActorHandle,
 }
@@ -68,6 +69,18 @@ impl SessionManager {
         rx.await
             .map_err(|e| SessionError::internal(format!("failed to delete session: {e}")))?
     }
+
+    /// Shut down the session manager and all resources it owns.
+    pub async fn shutdown(&self) -> SessionResult<()> {
+        let (tx, rx) = oneshot::channel();
+        self.handle
+            .send(SessionManagerEvent::Shutdown { result: tx })
+            .await?;
+        rx.await.map_err(|e| {
+            SessionError::internal(format!("failed to shut down session manager: {e}"))
+        })?;
+        Ok(())
+    }
 }
 
 pub async fn create_session_manager(
diff --git a/crates/sail-spark-connect/src/entrypoint.rs b/crates/sail-spark-connect/src/entrypoint.rs
index e125db6da..d736d7aa1 100644
--- a/crates/sail-spark-connect/src/entrypoint.rs
+++ b/crates/sail-spark-connect/src/entrypoint.rs
@@ -23,18 +23,23 @@ where
     F: Future,
 {
     let session_manager = create_spark_session_manager(config, runtime).await?;
-    let server = SparkConnectServer::new(session_manager);
-    let service = SparkConnectServiceServer::new(server)
-        // The original Spark Connect server seems to have configuration for inbound (decoding) message size only.
-        // .max_encoding_message_size(GRPC_MAX_MESSAGE_LENGTH_DEFAULT)
-        .max_decoding_message_size(GRPC_MAX_MESSAGE_LENGTH_DEFAULT)
-        .accept_compressed(CompressionEncoding::Gzip)
-        .accept_compressed(CompressionEncoding::Zstd)
-        .send_compressed(CompressionEncoding::Gzip)
-        .send_compressed(CompressionEncoding::Zstd);
-    ServerBuilder::new("sail_spark_connect", Default::default())
-        .add_service(service, Some(crate::spark::connect::FILE_DESCRIPTOR_SET))
-        .await
-        .serve(listener, signal)
-        .await
+    let result = {
+        let server = SparkConnectServer::new(session_manager.clone());
+        let service = SparkConnectServiceServer::new(server)
+            // The original Spark Connect server seems to have configuration for inbound (decoding) message size only.
+            // .max_encoding_message_size(GRPC_MAX_MESSAGE_LENGTH_DEFAULT)
+            .max_decoding_message_size(GRPC_MAX_MESSAGE_LENGTH_DEFAULT)
+            .accept_compressed(CompressionEncoding::Gzip)
+            .accept_compressed(CompressionEncoding::Zstd)
+            .send_compressed(CompressionEncoding::Gzip)
+            .send_compressed(CompressionEncoding::Zstd);
+        ServerBuilder::new("sail_spark_connect", Default::default())
+            .add_service(service, Some(crate::spark::connect::FILE_DESCRIPTOR_SET))
+            .await
+            .serve(listener, signal)
+            .await
+            .map_err(|e| std::io::Error::other(e.to_string()))
+    };
+    session_manager.shutdown().await?;
+    result.map_err(Into::into)
 }
diff --git a/python/pysail/tests/spark/execution/test_driver_server.py b/python/pysail/tests/spark/execution/test_driver_server.py
index 62d75c2a4..54b9e9f42 100644
--- a/python/pysail/tests/spark/execution/test_driver_server.py
+++ b/python/pysail/tests/spark/execution/test_driver_server.py
@@ -40,3 +40,22 @@ def test_driver_gateway_startup_failure(monkeypatch):
         server = SparkConnectServer("127.0.0.1", 0)
         with pytest.raises(RuntimeError, match="failed to create driver gateway"):
             server.start(background=False)
+
+
+def test_session_manager_stops_with_server(monkeypatch):
+    with socket.socket() as sock:
+        sock.bind(("127.0.0.1", 0))
+        _, driver_port = sock.getsockname()
+    monkeypatch.setenv("SAIL_MODE", "local-cluster")
+    monkeypatch.setenv("SAIL_CLUSTER__DRIVER_LISTEN_PORT", str(driver_port))
+
+    server = SparkConnectServer("127.0.0.1", 0)
+    server.start(background=True)
+    _, port = server.listening_address
+    with spark_session_factory(f"sc://localhost:{port}") as registry:
+        spark = registry.create()
+        assert spark.sql("SELECT 1 AS value").first().value == 1
+    server.stop()
+
+    with socket.socket() as sock:
+        sock.bind(("127.0.0.1", driver_port))

From 41ffda5b37780c5bc57ef592b1f50cb4ae16a23c Mon Sep 17 00:00:00 2001
From: Rohan Dubey 
Date: Thu, 30 Jul 2026 20:40:02 +0530
Subject: [PATCH 102/131] refactor: consolidate schema type merging logic
 (#2312)

---
 .../src/aggregate/schema_of_variant_agg.rs    | 198 +---------------
 crates/sail-function/src/lib.rs               |   1 +
 .../src/scalar/json/schema_of_json.rs         |  76 ++----
 .../scalar/variant/spark_schema_of_variant.rs | 221 ++++++++++--------
 crates/sail-function/src/schema_inference.rs  |  78 +++++++
 .../variant/schema_of_variant.feature         |  16 ++
 6 files changed, 250 insertions(+), 340 deletions(-)
 create mode 100644 crates/sail-function/src/schema_inference.rs

diff --git a/crates/sail-function/src/aggregate/schema_of_variant_agg.rs b/crates/sail-function/src/aggregate/schema_of_variant_agg.rs
index 6d0126390..050ff91ee 100644
--- a/crates/sail-function/src/aggregate/schema_of_variant_agg.rs
+++ b/crates/sail-function/src/aggregate/schema_of_variant_agg.rs
@@ -1,4 +1,3 @@
-use std::collections::BTreeMap;
 use std::sync::Arc;
 
 use arrow::array::{Array, ArrayRef};
@@ -7,10 +6,13 @@ use datafusion::common::Result;
 use datafusion::logical_expr::function::{AccumulatorArgs, StateFieldsArgs};
 use datafusion::logical_expr::{Accumulator, AggregateUDFImpl, Signature, Volatility};
 use datafusion::scalar::ScalarValue;
-use parquet_variant::Variant;
 use parquet_variant_compute::VariantArray;
 
-use crate::scalar::variant::spark_schema_of_variant::variant_to_spark_type;
+use crate::scalar::variant::spark_schema_of_variant::{
+    merge_variant_types, variant_to_inferred_type, variant_type_from_spark_type,
+    variant_type_to_spark_type,
+};
+use crate::schema_inference::InferredType;
 
 /// Aggregate function that merges variant schemas across rows.
 ///
@@ -68,185 +70,9 @@ impl AggregateUDFImpl for SchemaOfVariantAggFunction {
     }
 }
 
-/// Internal representation of a merged variant schema.
-#[derive(Debug, Clone, PartialEq)]
-enum MergedType {
-    /// A concrete primitive type (e.g., "BIGINT", "STRING", "BOOLEAN")
-    Primitive(String),
-    /// VOID (from null values)
-    Void,
-    /// An object with merged fields (field_name → merged_type)
-    Object(BTreeMap),
-    /// An array with a merged element type
-    Array(Box),
-    /// Mixed types that cannot be unified → VARIANT
-    Variant,
-}
-
-impl MergedType {
-    /// Parse a Spark type string back into a MergedType.
-    fn from_spark_type(s: &str) -> Result {
-        if s == "VOID" {
-            return Ok(MergedType::Void);
-        }
-        if s == "VARIANT" {
-            return Ok(MergedType::Variant);
-        }
-        if let Some(inner) = s.strip_prefix("OBJECT<").and_then(|s| s.strip_suffix('>')) {
-            if inner.is_empty() {
-                return Ok(MergedType::Object(BTreeMap::new()));
-            }
-            let mut fields = BTreeMap::new();
-            for field_str in split_top_level(inner) {
-                let (name, type_str) = field_str.split_once(": ").ok_or_else(|| {
-                    datafusion::common::DataFusionError::Execution(format!(
-                        "schema_of_variant_agg: invalid object field entry '{field_str}'"
-                    ))
-                })?;
-                fields.insert(
-                    name.trim().to_string(),
-                    MergedType::from_spark_type(type_str.trim())?,
-                );
-            }
-            return Ok(MergedType::Object(fields));
-        }
-        if let Some(inner) = s.strip_prefix("ARRAY<").and_then(|s| s.strip_suffix('>')) {
-            return Ok(MergedType::Array(Box::new(MergedType::from_spark_type(
-                inner,
-            )?)));
-        }
-        Ok(MergedType::Primitive(s.to_string()))
-    }
-
-    /// Convert to Spark type string.
-    fn to_spark_type(&self) -> String {
-        match self {
-            MergedType::Void => "VOID".to_string(),
-            MergedType::Primitive(s) => s.clone(),
-            MergedType::Variant => "VARIANT".to_string(),
-            MergedType::Object(fields) => {
-                if fields.is_empty() {
-                    return "OBJECT<>".to_string();
-                }
-                let fields_str: Vec = fields
-                    .iter()
-                    .map(|(name, ty)| format!("{name}: {}", ty.to_spark_type()))
-                    .collect();
-                format!("OBJECT<{}>", fields_str.join(", "))
-            }
-            MergedType::Array(elem) => {
-                format!("ARRAY<{}>", elem.to_spark_type())
-            }
-        }
-    }
-
-    /// Estimate heap memory usage recursively.
-    fn estimated_size(&self) -> usize {
-        match self {
-            MergedType::Void | MergedType::Variant => 0,
-            MergedType::Primitive(s) => s.len(),
-            MergedType::Object(fields) => fields
-                .iter()
-                .map(|(k, v)| {
-                    k.len() + v.estimated_size() + std::mem::size_of::<(String, MergedType)>()
-                })
-                .sum(),
-            MergedType::Array(elem) => std::mem::size_of::() + elem.estimated_size(),
-        }
-    }
-
-    /// Merge two types together.
-    fn merge(self, other: MergedType) -> MergedType {
-        match (self, other) {
-            // VOID merges with anything → the other type
-            (MergedType::Void, other) | (other, MergedType::Void) => other,
-            // VARIANT absorbs everything
-            (MergedType::Variant, _) | (_, MergedType::Variant) => MergedType::Variant,
-            // Same primitive → keep
-            (MergedType::Primitive(a), MergedType::Primitive(b)) if a == b => {
-                MergedType::Primitive(a)
-            }
-            // Different primitives → VARIANT
-            (MergedType::Primitive(_), MergedType::Primitive(_)) => MergedType::Variant,
-            // Object + Object → merge fields
-            (MergedType::Object(mut a), MergedType::Object(b)) => {
-                for (key, val) in b {
-                    match a.entry(key) {
-                        std::collections::btree_map::Entry::Occupied(mut entry) => {
-                            let old = std::mem::replace(entry.get_mut(), MergedType::Void);
-                            *entry.get_mut() = old.merge(val);
-                        }
-                        std::collections::btree_map::Entry::Vacant(entry) => {
-                            entry.insert(val);
-                        }
-                    }
-                }
-                MergedType::Object(a)
-            }
-            // Array + Array → merge element types
-            (MergedType::Array(a), MergedType::Array(b)) => {
-                MergedType::Array(Box::new(a.merge(*b)))
-            }
-            // Incompatible types → VARIANT
-            _ => MergedType::Variant,
-        }
-    }
-}
-
-/// Split a string by top-level commas (respecting nested angle brackets and parentheses).
-/// Handles types like `DECIMAL(10,2)` inside `OBJECT<...>` fields.
-fn split_top_level(s: &str) -> Vec<&str> {
-    let mut results = Vec::new();
-    let mut angle_depth: usize = 0;
-    let mut paren_depth: usize = 0;
-    let mut start = 0;
-    for (i, ch) in s.char_indices() {
-        match ch {
-            '<' => angle_depth += 1,
-            '>' => angle_depth = angle_depth.saturating_sub(1),
-            '(' => paren_depth += 1,
-            ')' => paren_depth = paren_depth.saturating_sub(1),
-            ',' if angle_depth == 0 && paren_depth == 0 => {
-                results.push(s[start..i].trim());
-                start = i + 1;
-            }
-            _ => {}
-        }
-    }
-    if start < s.len() {
-        results.push(s[start..].trim());
-    }
-    results
-}
-
-/// Convert a Variant to a MergedType.
-fn variant_to_merged_type(variant: &Variant) -> MergedType {
-    match variant {
-        Variant::Null => MergedType::Void,
-        Variant::Object(obj) => {
-            let mut fields = BTreeMap::new();
-            for (name, value) in obj.iter() {
-                fields.insert(name.to_string(), variant_to_merged_type(&value));
-            }
-            MergedType::Object(fields)
-        }
-        Variant::List(list) => {
-            let mut elem_type = MergedType::Void;
-            for elem in list.iter() {
-                elem_type = elem_type.merge(variant_to_merged_type(&elem));
-            }
-            MergedType::Array(Box::new(elem_type))
-        }
-        _ => {
-            // Use the same type string as schema_of_variant for primitives
-            MergedType::Primitive(variant_to_spark_type(variant))
-        }
-    }
-}
-
 #[derive(Debug)]
 struct SchemaOfVariantAggAccumulator {
-    merged_schema: Option,
+    merged_schema: Option,
 }
 
 impl Accumulator for SchemaOfVariantAggAccumulator {
@@ -255,9 +81,9 @@ impl Accumulator for SchemaOfVariantAggAccumulator {
         let variant_array = VariantArray::try_new(arr.as_ref())?;
 
         for variant in variant_array.iter().flatten() {
-            let current_type = variant_to_merged_type(&variant);
+            let current_type = variant_to_inferred_type(&variant);
             self.merged_schema = Some(match self.merged_schema.take() {
-                Some(existing) => existing.merge(current_type),
+                Some(existing) => merge_variant_types(existing, current_type),
                 None => current_type,
             });
         }
@@ -268,7 +94,7 @@ impl Accumulator for SchemaOfVariantAggAccumulator {
         let schema_str = self
             .merged_schema
             .as_ref()
-            .map_or_else(|| "VOID".to_string(), |t| t.to_spark_type());
+            .map_or_else(|| "VOID".to_string(), variant_type_to_spark_type);
         Ok(vec![ScalarValue::Utf8(Some(schema_str))])
     }
 
@@ -286,9 +112,9 @@ impl Accumulator for SchemaOfVariantAggAccumulator {
             if schema_arr.is_null(i) {
                 continue;
             }
-            let other_schema = MergedType::from_spark_type(schema_arr.value(i))?;
+            let other_schema = variant_type_from_spark_type(schema_arr.value(i))?;
             self.merged_schema = Some(match self.merged_schema.take() {
-                Some(existing) => existing.merge(other_schema),
+                Some(existing) => merge_variant_types(existing, other_schema),
                 None => other_schema,
             });
         }
@@ -300,7 +126,7 @@ impl Accumulator for SchemaOfVariantAggAccumulator {
         let result = self
             .merged_schema
             .as_ref()
-            .map_or_else(|| "VOID".to_string(), |t| t.to_spark_type());
+            .map_or_else(|| "VOID".to_string(), variant_type_to_spark_type);
         Ok(ScalarValue::Utf8(Some(result)))
     }
 
diff --git a/crates/sail-function/src/lib.rs b/crates/sail-function/src/lib.rs
index b4ca0b0ce..507b248c0 100644
--- a/crates/sail-function/src/lib.rs
+++ b/crates/sail-function/src/lib.rs
@@ -4,6 +4,7 @@ mod functions_nested_utils;
 mod functions_utils;
 mod hll_sketch;
 pub mod scalar;
+mod schema_inference;
 pub mod sketch {
     pub use crate::hll_sketch::DEFAULT_LG_CONFIG_K as DEFAULT_HLL_LG_CONFIG_K;
     pub use crate::theta_sketch::DEFAULT_LG_NOM_ENTRIES as DEFAULT_THETA_LG_NOM_ENTRIES;
diff --git a/crates/sail-function/src/scalar/json/schema_of_json.rs b/crates/sail-function/src/scalar/json/schema_of_json.rs
index f2620770b..6688dd8b1 100644
--- a/crates/sail-function/src/scalar/json/schema_of_json.rs
+++ b/crates/sail-function/src/scalar/json/schema_of_json.rs
@@ -17,6 +17,8 @@ use datafusion_expr_common::signature::Volatility;
 use datafusion_functions::downcast_arg;
 use datafusion_functions::utils::make_scalar_function;
 
+use crate::schema_inference::{InferredType, TypeMerger};
+
 #[derive(Debug, PartialEq, Eq, Hash)]
 pub struct SparkSchemaOfJson {
     signature: Signature,
@@ -201,26 +203,6 @@ fn infer_json_schema_type(json_string: &str, options: &SparkSchemaOfJsonOptions)
 /// The maximum precision of Spark's `DecimalType`.
 const MAX_DECIMAL_PRECISION: usize = 38;
 
-/// A JSON type inferred from a literal JSON string, mirroring the types that
-/// Spark's `JsonInferSchema` can produce for `schema_of_json`.
-#[derive(Debug, Clone, PartialEq)]
-enum InferredType {
-    Null,
-    Boolean,
-    Long,
-    /// A number inferred as a decimal with the given precision and scale.
-    Decimal(u8, u8),
-    Double,
-    String,
-    /// A string inferred as a timestamp when the `inferTimestamp` option is
-    /// enabled and the value matches a recognized timestamp/date pattern.
-    Timestamp,
-    Array(Box),
-    /// Fields are sorted by name and duplicate names are preserved, matching
-    /// Spark's `JsonInferSchema`.
-    Struct(Vec<(String, InferredType)>),
-}
-
 /// A parser that infers the Spark type of a literal JSON string, mirroring
 /// the Jackson lexing behavior that Spark relies on for `schema_of_json`,
 /// including features that strict JSON parsers reject: single-quoted strings,
@@ -831,18 +813,22 @@ fn is_timestamp_string(s: &str) -> bool {
 /// Returns the most specific type that both types can be promoted to,
 /// mirroring Spark's `JsonInferSchema.compatibleType`.
 fn merge_types(left: InferredType, right: InferredType) -> InferredType {
-    use InferredType::*;
-    match (left, right) {
-        (Null, t) | (t, Null) => t,
-        (l, r) if l == r => l,
-        (Long, Double) | (Double, Long) => Double,
-        // A long is at most `DECIMAL(20, 0)` when promoted to a decimal.
-        (Long, Decimal(p, s)) | (Decimal(p, s), Long) => merge_decimals(p.max(20), s, 20, 0),
-        (Double, Decimal(_, _)) | (Decimal(_, _), Double) => Double,
-        (Decimal(p1, s1), Decimal(p2, s2)) => merge_decimals(p1, s1, p2, s2),
-        (Array(l), Array(r)) => Array(Box::new(merge_types(*l, *r))),
-        (Struct(l), Struct(r)) => Struct(merge_fields(l, r)),
-        _ => String,
+    left.merge_with(right, &JsonTypeMerger)
+}
+
+struct JsonTypeMerger;
+
+impl TypeMerger for JsonTypeMerger {
+    fn merge_atomic(&self, left: InferredType, right: InferredType) -> InferredType {
+        use InferredType::*;
+        match (left, right) {
+            (Long, Double) | (Double, Long) => Double,
+            // A long is at most `DECIMAL(20, 0)` when promoted to a decimal.
+            (Long, Decimal(p, s)) | (Decimal(p, s), Long) => merge_decimals(p.max(20), s, 20, 0),
+            (Double, Decimal(_, _)) | (Decimal(_, _), Double) => Double,
+            (Decimal(p1, s1), Decimal(p2, s2)) => merge_decimals(p1, s1, p2, s2),
+            _ => String,
+        }
     }
 }
 
@@ -859,27 +845,6 @@ fn merge_decimals(p1: u8, s1: u8, p2: u8, s2: u8) -> InferredType {
     }
 }
 
-/// Merges the fields of two structs by name, mirroring Spark's behavior of
-/// grouping fields by name and reducing each group with `compatibleType`.
-fn merge_fields(
-    left: Vec<(String, InferredType)>,
-    right: Vec<(String, InferredType)>,
-) -> Vec<(String, InferredType)> {
-    let mut fields = left;
-    fields.extend(right);
-    fields.sort_by(|(a, _), (b, _)| a.cmp(b));
-    let mut merged: Vec<(String, InferredType)> = Vec::new();
-    for (name, t) in fields {
-        match merged.last_mut() {
-            Some((last_name, last_type)) if *last_name == name => {
-                *last_type = merge_types(last_type.clone(), t);
-            }
-            _ => merged.push((name, t)),
-        }
-    }
-    merged
-}
-
 /// Writes the type as a Spark DDL string, returning `None` for types that
 /// Spark drops during canonicalization (empty structs, fields with empty
 /// names, and arrays of dropped types). `NULL` types become `STRING`.
@@ -888,10 +853,14 @@ fn canonicalize_ddl(t: &InferredType) -> Option {
         InferredType::Null => Some("STRING".to_string()),
         InferredType::Boolean => Some("BOOLEAN".to_string()),
         InferredType::Long => Some("BIGINT".to_string()),
+        InferredType::Float => Some("FLOAT".to_string()),
         InferredType::Decimal(p, s) => Some(format!("DECIMAL({p},{s})")),
         InferredType::Double => Some("DOUBLE".to_string()),
         InferredType::String => Some("STRING".to_string()),
+        InferredType::Binary => Some("BINARY".to_string()),
+        InferredType::Date => Some("DATE".to_string()),
         InferredType::Timestamp => Some("TIMESTAMP".to_string()),
+        InferredType::TimestampNtz => Some("TIMESTAMP_NTZ".to_string()),
         InferredType::Array(element) => canonicalize_ddl(element).map(|e| format!("ARRAY<{e}>")),
         InferredType::Struct(fields) => {
             let fields = fields
@@ -907,6 +876,7 @@ fn canonicalize_ddl(t: &InferredType) -> Option {
                 Some(format!("STRUCT<{}>", fields.join(", ")))
             }
         }
+        InferredType::Variant => Some("VARIANT".to_string()),
     }
 }
 
diff --git a/crates/sail-function/src/scalar/variant/spark_schema_of_variant.rs b/crates/sail-function/src/scalar/variant/spark_schema_of_variant.rs
index eb44cdd3e..200b55760 100644
--- a/crates/sail-function/src/scalar/variant/spark_schema_of_variant.rs
+++ b/crates/sail-function/src/scalar/variant/spark_schema_of_variant.rs
@@ -2,7 +2,7 @@ use std::sync::Arc;
 
 use arrow::array::ArrayRef;
 use arrow_schema::DataType;
-use datafusion::common::{exec_datafusion_err, exec_err};
+use datafusion::common::{DataFusionError, exec_datafusion_err, exec_err};
 use datafusion::error::Result;
 use datafusion::logical_expr::{
     ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility,
@@ -13,6 +13,7 @@ use parquet_variant_compute::VariantArray;
 
 use crate::error::invalid_arg_count_exec_err;
 use crate::scalar::variant::utils::helper::{try_field_as_variant_array, try_parse_variant_scalar};
+use crate::schema_inference::{InferredType, TypeMerger};
 
 /// Returns the schema (type string) of a variant value using Spark type names.
 ///
@@ -115,145 +116,163 @@ impl ScalarUDFImpl for SparkSchemaOfVariantUdf {
 
 /// Convert a Variant value to its Spark type string representation.
 pub(crate) fn variant_to_spark_type(variant: &Variant) -> String {
+    variant_type_to_spark_type(&variant_to_inferred_type(variant))
+}
+
+pub(crate) fn variant_to_inferred_type(variant: &Variant) -> InferredType {
     match variant {
-        Variant::Null => "VOID".to_string(),
-        Variant::BooleanTrue | Variant::BooleanFalse => "BOOLEAN".to_string(),
+        Variant::Null => InferredType::Null,
+        Variant::BooleanTrue | Variant::BooleanFalse => InferredType::Boolean,
         Variant::Int8(_) | Variant::Int16(_) | Variant::Int32(_) | Variant::Int64(_) => {
-            "BIGINT".to_string()
+            InferredType::Long
         }
-        Variant::Float(_) => "FLOAT".to_string(),
-        Variant::Double(_) => "DOUBLE".to_string(),
-        Variant::Decimal4(d) => decimal_type_string(d.integer() as i128, d.scale()),
-        Variant::Decimal8(d) => decimal_type_string(d.integer() as i128, d.scale()),
-        Variant::Decimal16(d) => decimal_type_string(d.integer(), d.scale()),
-        Variant::String(_) | Variant::ShortString(_) => "STRING".to_string(),
-        Variant::Binary(_) => "BINARY".to_string(),
-        Variant::Date(_) => "DATE".to_string(),
-        Variant::TimestampMicros(_) | Variant::TimestampNanos(_) => "TIMESTAMP".to_string(),
+        Variant::Float(_) => InferredType::Float,
+        Variant::Double(_) => InferredType::Double,
+        Variant::Decimal4(d) => decimal_type(d.integer() as i128, d.scale()),
+        Variant::Decimal8(d) => decimal_type(d.integer() as i128, d.scale()),
+        Variant::Decimal16(d) => decimal_type(d.integer(), d.scale()),
+        Variant::String(_) | Variant::ShortString(_) => InferredType::String,
+        Variant::Binary(_) => InferredType::Binary,
+        Variant::Date(_) => InferredType::Date,
+        Variant::TimestampMicros(_) | Variant::TimestampNanos(_) => InferredType::Timestamp,
         Variant::TimestampNtzMicros(_) | Variant::TimestampNtzNanos(_) => {
-            "TIMESTAMP_NTZ".to_string()
+            InferredType::TimestampNtz
         }
-        Variant::Time(_) => "STRING".to_string(),
-        Variant::Uuid(_) => "STRING".to_string(),
+        Variant::Time(_) | Variant::Uuid(_) => InferredType::String,
         Variant::Object(obj) => {
-            if obj.is_empty() {
-                return "OBJECT<>".to_string();
-            }
-            // Collect fields, sorted by field name (Spark sorts alphabetically)
-            let mut fields: Vec<(String, String)> = Vec::with_capacity(obj.len());
+            let mut fields = Vec::with_capacity(obj.len());
             for (name, value) in obj.iter() {
-                fields.push((name.to_string(), variant_to_spark_type(&value)));
+                fields.push((name.to_string(), variant_to_inferred_type(&value)));
             }
             fields.sort_by(|a, b| a.0.cmp(&b.0));
-            let fields_str: Vec = fields
-                .iter()
-                .map(|(name, ty)| format!("{name}: {ty}"))
-                .collect();
-            format!("OBJECT<{}>", fields_str.join(", "))
+            InferredType::Struct(fields)
         }
         Variant::List(list) => {
-            if list.is_empty() {
-                return "ARRAY".to_string();
+            let mut element_type = InferredType::Null;
+            for element in list.iter() {
+                element_type =
+                    merge_variant_types(element_type, variant_to_inferred_type(&element));
             }
-            let element_types: Vec =
-                list.iter().map(|e| variant_to_spark_type(&e)).collect();
-            let merged = merge_types(&element_types);
-            format!("ARRAY<{merged}>")
+            InferredType::Array(Box::new(element_type))
         }
     }
 }
 
-/// Merge a list of Spark type strings into a single type.
-///
-/// Rules (matching Spark behavior):
-/// - VOID is absorbed by any non-VOID type: `[VOID, BIGINT]` → `BIGINT`
-/// - OBJECT types with different fields are merged (field union): `[OBJECT, OBJECT]` → `OBJECT`
-/// - All other mismatches → `VARIANT`
-fn merge_types(types: &[String]) -> String {
-    // Filter out VOIDs
-    let non_void: Vec<&String> = types.iter().filter(|t| t.as_str() != "VOID").collect();
-
-    if non_void.is_empty() {
-        return "VOID".to_string();
-    }
+pub(crate) fn merge_variant_types(left: InferredType, right: InferredType) -> InferredType {
+    left.merge_with(right, &VariantTypeMerger)
+}
 
-    // Check if all non-void types are identical
-    if non_void.iter().all(|t| *t == non_void[0]) {
-        return non_void[0].clone();
-    }
+struct VariantTypeMerger;
 
-    // Try merging OBJECTs: if all non-void types are OBJECT<...>, union their fields
-    if non_void.iter().all(|t| t.starts_with("OBJECT<")) {
-        return merge_object_types(&non_void);
+impl TypeMerger for VariantTypeMerger {
+    fn merge_atomic(&self, _left: InferredType, _right: InferredType) -> InferredType {
+        InferredType::Variant
     }
-
-    "VARIANT".to_string()
 }
 
-/// Merge multiple OBJECT<...> type strings by unioning their fields.
-fn merge_object_types(types: &[&String]) -> String {
-    let mut merged_fields: std::collections::BTreeMap =
-        std::collections::BTreeMap::new();
-
-    for ty in types {
-        // Strip "OBJECT<" prefix and ">" suffix
-        let inner = ty
-            .strip_prefix("OBJECT<")
-            .and_then(|s| s.strip_suffix('>'))
-            .unwrap_or("");
-
-        if inner.is_empty() {
-            continue;
+pub(crate) fn variant_type_to_spark_type(inferred: &InferredType) -> String {
+    match inferred {
+        InferredType::Null => "VOID".to_string(),
+        InferredType::Boolean => "BOOLEAN".to_string(),
+        InferredType::Long => "BIGINT".to_string(),
+        InferredType::Float => "FLOAT".to_string(),
+        InferredType::Decimal(precision, scale) => format!("DECIMAL({precision},{scale})"),
+        InferredType::Double => "DOUBLE".to_string(),
+        InferredType::String => "STRING".to_string(),
+        InferredType::Binary => "BINARY".to_string(),
+        InferredType::Date => "DATE".to_string(),
+        InferredType::Timestamp => "TIMESTAMP".to_string(),
+        InferredType::TimestampNtz => "TIMESTAMP_NTZ".to_string(),
+        InferredType::Array(element) => {
+            format!("ARRAY<{}>", variant_type_to_spark_type(element))
         }
-
-        // Parse fields: "a: BIGINT, b: STRING" → [("a", "BIGINT"), ("b", "STRING")]
-        // Handle nested types by tracking angle bracket depth
-        for field_str in split_fields(inner) {
-            if let Some((name, field_type)) = field_str.split_once(':') {
-                let name = name.trim().to_string();
-                let field_type = field_type.trim().to_string();
-                merged_fields.entry(name).or_insert(field_type);
-            }
+        InferredType::Struct(fields) => {
+            let fields = fields
+                .iter()
+                .map(|(name, ty)| format!("{name}: {}", variant_type_to_spark_type(ty)))
+                .collect::>();
+            format!("OBJECT<{}>", fields.join(", "))
         }
+        InferredType::Variant => "VARIANT".to_string(),
     }
+}
 
-    if merged_fields.is_empty() {
-        return "OBJECT<>".to_string();
+pub(crate) fn variant_type_from_spark_type(s: &str) -> Result {
+    match s {
+        "VOID" => return Ok(InferredType::Null),
+        "BOOLEAN" => return Ok(InferredType::Boolean),
+        "BIGINT" => return Ok(InferredType::Long),
+        "FLOAT" => return Ok(InferredType::Float),
+        "DOUBLE" => return Ok(InferredType::Double),
+        "STRING" => return Ok(InferredType::String),
+        "BINARY" => return Ok(InferredType::Binary),
+        "DATE" => return Ok(InferredType::Date),
+        "TIMESTAMP" => return Ok(InferredType::Timestamp),
+        "TIMESTAMP_NTZ" => return Ok(InferredType::TimestampNtz),
+        "VARIANT" => return Ok(InferredType::Variant),
+        _ => {}
     }
-
-    let fields_str: Vec = merged_fields
-        .iter()
-        .map(|(name, ty)| format!("{name}: {ty}"))
-        .collect();
-    format!("OBJECT<{}>", fields_str.join(", "))
+    if let Some(inner) = s.strip_prefix("DECIMAL(").and_then(|s| s.strip_suffix(')')) {
+        let (precision, scale) = inner.split_once(',').ok_or_else(|| {
+            DataFusionError::Execution(format!("invalid inferred decimal type '{s}'"))
+        })?;
+        let precision = precision.parse::().map_err(|_| {
+            DataFusionError::Execution(format!("invalid inferred decimal precision '{precision}'"))
+        })?;
+        let scale = scale.parse::().map_err(|_| {
+            DataFusionError::Execution(format!("invalid inferred decimal scale '{scale}'"))
+        })?;
+        return Ok(InferredType::Decimal(precision, scale));
+    }
+    if let Some(inner) = s.strip_prefix("ARRAY<").and_then(|s| s.strip_suffix('>')) {
+        return Ok(InferredType::Array(Box::new(variant_type_from_spark_type(
+            inner,
+        )?)));
+    }
+    if let Some(inner) = s.strip_prefix("OBJECT<").and_then(|s| s.strip_suffix('>')) {
+        let mut fields = Vec::new();
+        for field in split_top_level(inner) {
+            let (name, ty) = field.split_once(": ").ok_or_else(|| {
+                DataFusionError::Execution(format!("invalid inferred object field '{field}'"))
+            })?;
+            fields.push((
+                name.trim().to_string(),
+                variant_type_from_spark_type(ty.trim())?,
+            ));
+        }
+        return Ok(InferredType::Struct(fields));
+    }
+    Err(DataFusionError::Execution(format!(
+        "invalid inferred variant type '{s}'"
+    )))
 }
 
-/// Split a comma-separated field list respecting nested angle brackets.
-/// e.g. "a: OBJECT, b: ARRAY" → ["a: OBJECT", "b: ARRAY"]
-fn split_fields(s: &str) -> Vec<&str> {
-    let mut result = Vec::new();
-    let mut depth = 0;
+fn split_top_level(s: &str) -> Vec<&str> {
+    let mut fields = Vec::new();
+    let mut angle_depth = 0usize;
+    let mut paren_depth = 0usize;
     let mut start = 0;
     for (i, ch) in s.char_indices() {
         match ch {
-            '<' => depth += 1,
-            '>' => depth -= 1,
-            ',' if depth == 0 => {
-                result.push(s[start..i].trim());
+            '<' => angle_depth += 1,
+            '>' => angle_depth = angle_depth.saturating_sub(1),
+            '(' => paren_depth += 1,
+            ')' => paren_depth = paren_depth.saturating_sub(1),
+            ',' if angle_depth == 0 && paren_depth == 0 => {
+                fields.push(s[start..i].trim());
                 start = i + 1;
             }
             _ => {}
         }
     }
-    let last = s[start..].trim();
-    if !last.is_empty() {
-        result.push(last);
+    if start < s.len() {
+        fields.push(s[start..].trim());
     }
-    result
+    fields
 }
 
 /// Compute the decimal precision from the integer value and scale.
-fn decimal_type_string(integer: i128, scale: u8) -> String {
+fn decimal_type(integer: i128, scale: u8) -> InferredType {
     let abs = integer.unsigned_abs();
     let int_digits = if abs == 0 {
         1u8
@@ -268,5 +287,5 @@ fn decimal_type_string(integer: i128, scale: u8) -> String {
         d
     };
     let precision = int_digits.max(scale);
-    format!("DECIMAL({precision},{scale})")
+    InferredType::Decimal(precision, scale)
 }
diff --git a/crates/sail-function/src/schema_inference.rs b/crates/sail-function/src/schema_inference.rs
new file mode 100644
index 000000000..5e163a04e
--- /dev/null
+++ b/crates/sail-function/src/schema_inference.rs
@@ -0,0 +1,78 @@
+/// A recursively inferred schema before it is rendered as a Spark type string.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub(crate) enum InferredType {
+    Null,
+    Boolean,
+    Long,
+    Float,
+    Decimal(u8, u8),
+    Double,
+    String,
+    Binary,
+    Date,
+    Timestamp,
+    TimestampNtz,
+    Array(Box),
+    Struct(Vec<(String, InferredType)>),
+    Variant,
+}
+
+/// Defines how format-specific incompatible scalar types are merged.
+pub(crate) trait TypeMerger {
+    fn merge_atomic(&self, left: InferredType, right: InferredType) -> InferredType;
+}
+
+impl InferredType {
+    /// Recursively merges two inferred types while delegating format-specific
+    /// scalar coercion and incompatible-type handling to the merger.
+    pub(crate) fn merge_with(self, other: InferredType, merger: &impl TypeMerger) -> InferredType {
+        match (self, other) {
+            (InferredType::Null, other) | (other, InferredType::Null) => other,
+            (left, right) if left == right => left,
+            (InferredType::Array(left), InferredType::Array(right)) => {
+                InferredType::Array(Box::new(left.merge_with(*right, merger)))
+            }
+            (InferredType::Struct(left), InferredType::Struct(right)) => {
+                InferredType::Struct(merge_fields(left, right, merger))
+            }
+            (left, right) => merger.merge_atomic(left, right),
+        }
+    }
+
+    pub(crate) fn estimated_size(&self) -> usize {
+        match self {
+            InferredType::Array(element) => {
+                std::mem::size_of::() + element.estimated_size()
+            }
+            InferredType::Struct(fields) => fields
+                .iter()
+                .map(|(name, ty)| {
+                    name.len() + ty.estimated_size() + std::mem::size_of::<(String, InferredType)>()
+                })
+                .sum(),
+            _ => 0,
+        }
+    }
+}
+
+fn merge_fields(
+    left: Vec<(String, InferredType)>,
+    right: Vec<(String, InferredType)>,
+    merger: &impl TypeMerger,
+) -> Vec<(String, InferredType)> {
+    let mut fields = left;
+    fields.extend(right);
+    fields.sort_by(|(left, _), (right, _)| left.cmp(right));
+
+    let mut merged: Vec<(String, InferredType)> = Vec::new();
+    for (name, ty) in fields {
+        match merged.last_mut() {
+            Some((last_name, last_type)) if *last_name == name => {
+                let previous = std::mem::replace(last_type, InferredType::Null);
+                *last_type = previous.merge_with(ty, merger);
+            }
+            _ => merged.push((name, ty)),
+        }
+    }
+    merged
+}
diff --git a/python/pysail/tests/spark/function/features/variant/schema_of_variant.feature b/python/pysail/tests/spark/function/features/variant/schema_of_variant.feature
index eb1d593c3..bb9c1ec51 100644
--- a/python/pysail/tests/spark/function/features/variant/schema_of_variant.feature
+++ b/python/pysail/tests/spark/function/features/variant/schema_of_variant.feature
@@ -80,6 +80,22 @@ Feature: schema_of_variant
         | result                      |
         | ARRAY>> |
 
+    Scenario: schema_of_variant merges object fields with decimal types
+      When query
+        """
+        SELECT schema_of_variant(
+          to_variant_object(
+            array(
+              map('a', CAST(1.23 AS DECIMAL(3,2))),
+              map('b', CAST(4.56 AS DECIMAL(3,2)))
+            )
+          )
+        ) AS result
+        """
+      Then query result
+        | result                                          |
+        | ARRAY> |
+
     Scenario Outline: Nested: 
       When query
         """

From 58d2efc806f82045fea6143f5eea747b5948b654 Mon Sep 17 00:00:00 2001
From: Heran Lin 
Date: Fri, 31 Jul 2026 09:39:35 +0800
Subject: [PATCH 103/131] fix: always report job runner history (#2327)

---
 .../sail-common-datafusion/src/session/job.rs |   8 +-
 .../sail-execution/src/driver/actor/core.rs   |  11 +-
 .../src/driver/actor/handler.rs               |   8 +-
 crates/sail-execution/src/driver/actor/mod.rs |   5 +-
 crates/sail-execution/src/driver/event.rs     |   3 +-
 crates/sail-execution/src/driver/options.rs   |   8 ++
 crates/sail-execution/src/driver/registry.rs  |   4 +-
 .../src/driver/worker_pool/core.rs            |   2 +-
 crates/sail-execution/src/job_runner.rs       |  44 ++++----
 .../src/session_factory/job_runner.rs         |   8 +-
 .../src/session_manager/actor/handler.rs      | 100 +++++++++++++-----
 .../src/session_manager/session.rs            |  18 ++--
 12 files changed, 142 insertions(+), 77 deletions(-)

diff --git a/crates/sail-common-datafusion/src/session/job.rs b/crates/sail-common-datafusion/src/session/job.rs
index c5d1b7f88..3fd457115 100644
--- a/crates/sail-common-datafusion/src/session/job.rs
+++ b/crates/sail-common-datafusion/src/session/job.rs
@@ -5,7 +5,6 @@ use datafusion::common::Result;
 use datafusion::execution::SendableRecordBatchStream;
 use datafusion::physical_plan::ExecutionPlan;
 use datafusion::prelude::SessionContext;
-use tokio::sync::oneshot;
 
 use crate::extension::SessionExtension;
 use crate::system::catalog::{JobRow, StageRow, TaskRow, WorkerRow};
@@ -25,7 +24,12 @@ pub trait JobRunner: StateObservable + Send + Sync + 'static
     /// Returns the distributed execution plan for EXPLAIN output.
     fn explain(&self, plan: Arc) -> Result;
 
-    async fn stop(&self, history: oneshot::Sender);
+    async fn stop(&self);
+}
+
+#[tonic::async_trait]
+pub trait JobRunnerHistoryReporter: Send + 'static {
+    async fn report(self: Box, history: JobRunnerHistory);
 }
 
 pub struct JobRunnerHistory {
diff --git a/crates/sail-execution/src/driver/actor/core.rs b/crates/sail-execution/src/driver/actor/core.rs
index 1c0caf24b..4f533d369 100644
--- a/crates/sail-execution/src/driver/actor/core.rs
+++ b/crates/sail-execution/src/driver/actor/core.rs
@@ -35,7 +35,7 @@ impl Actor for DriverActor {
             task_runner: TaskRunner::new(),
             stream_manager,
             task_sequences: HashMap::new(),
-            history: None,
+            shutdown_notifier: None,
         }
     }
 
@@ -109,7 +109,7 @@ impl Actor for DriverActor {
                 result,
             } => self.handle_fetch_remote_stream(ctx, key, schema, context, result),
             DriverEvent::ObserveState { observer } => self.handle_observe_state(ctx, observer),
-            DriverEvent::Shutdown { history } => self.handle_shutdown(ctx, history),
+            DriverEvent::Shutdown { result } => self.handle_shutdown(ctx, result),
         }
     }
 
@@ -119,8 +119,11 @@ impl Actor for DriverActor {
         if let Err(e) = self.worker_pool.close(ctx).await {
             error!("encountered error while stopping workers: {e}");
         }
-        if let Some(history) = self.history.take() {
-            let _ = history.send(self.build_history());
+        if let Some(history_reporter) = self.options.take_history_reporter() {
+            history_reporter.report(self.build_history()).await;
+        }
+        if let Some(result) = self.shutdown_notifier.take() {
+            let _ = result.send(());
         }
         info!("driver {} has stopped", self.options.driver_id);
     }
diff --git a/crates/sail-execution/src/driver/actor/handler.rs b/crates/sail-execution/src/driver/actor/handler.rs
index 5ec0164ca..41402ccd7 100644
--- a/crates/sail-execution/src/driver/actor/handler.rs
+++ b/crates/sail-execution/src/driver/actor/handler.rs
@@ -412,12 +412,12 @@ impl DriverActor {
     pub(super) fn handle_shutdown(
         &mut self,
         _ctx: &mut ActorContext,
-        history: Option>,
+        result: Option>,
     ) -> ActorAction {
-        if self.history.is_some() {
-            warn!("overriding existing history sender");
+        if self.shutdown_notifier.is_some() {
+            warn!("overriding existing shutdown notifier");
         }
-        self.history = history;
+        self.shutdown_notifier = result;
         ActorAction::Stop
     }
 
diff --git a/crates/sail-execution/src/driver/actor/mod.rs b/crates/sail-execution/src/driver/actor/mod.rs
index 923b2cea6..e95cd6878 100644
--- a/crates/sail-execution/src/driver/actor/mod.rs
+++ b/crates/sail-execution/src/driver/actor/mod.rs
@@ -3,7 +3,6 @@ mod handler;
 
 use std::collections::HashMap;
 
-use sail_common_datafusion::session::job::JobRunnerHistory;
 use tokio::sync::oneshot;
 
 use crate::driver::job_scheduler::JobScheduler;
@@ -22,6 +21,6 @@ pub struct DriverActor {
     /// The sequence number corresponding to the last task status update from the worker.
     /// A different sequence number is tracked for each attempt.
     task_sequences: HashMap,
-    /// An optional channel to send history when stopping the driver.
-    history: Option>,
+    /// An optional channel to signal that the driver has stopped.
+    shutdown_notifier: Option>,
 }
diff --git a/crates/sail-execution/src/driver/event.rs b/crates/sail-execution/src/driver/event.rs
index c61a16ce6..b6f24001c 100644
--- a/crates/sail-execution/src/driver/event.rs
+++ b/crates/sail-execution/src/driver/event.rs
@@ -7,7 +7,6 @@ use datafusion::arrow::datatypes::SchemaRef;
 use datafusion::execution::{SendableRecordBatchStream, TaskContext};
 use datafusion::physical_plan::ExecutionPlan;
 use sail_common_datafusion::error::CommonErrorCause;
-use sail_common_datafusion::session::job::JobRunnerHistory;
 use sail_common_datafusion::system::observable::JobRunnerObserver;
 use sail_telemetry::common::{SpanAssociation, SpanAttribute};
 use tokio::sync::oneshot;
@@ -100,7 +99,7 @@ pub enum DriverEvent {
         observer: JobRunnerObserver,
     },
     Shutdown {
-        history: Option>,
+        result: Option>,
     },
 }
 
diff --git a/crates/sail-execution/src/driver/options.rs b/crates/sail-execution/src/driver/options.rs
index 52beb4ccc..ef83a26bd 100644
--- a/crates/sail-execution/src/driver/options.rs
+++ b/crates/sail-execution/src/driver/options.rs
@@ -3,6 +3,7 @@ use std::time::Duration;
 
 use sail_common::config::AppConfig;
 use sail_common::runtime::RuntimeHandle;
+use sail_common_datafusion::session::job::JobRunnerHistoryReporter;
 use sail_server::RetryStrategy;
 
 use crate::id::DriverId;
@@ -31,6 +32,7 @@ pub struct DriverOptions {
     pub rpc_retry_strategy: RetryStrategy,
     pub runtime: RuntimeHandle,
     pub worker_manager: Arc,
+    pub history_reporter: Option>,
 }
 
 impl DriverOptions {
@@ -40,6 +42,7 @@ impl DriverOptions {
         driver_id: DriverId,
         driver_server_port: u16,
         worker_manager: Arc,
+        history_reporter: Box,
     ) -> Self {
         Self {
             enable_tls: config.cluster.enable_tls,
@@ -68,6 +71,11 @@ impl DriverOptions {
             shuffle_backend: (&config.cluster.shuffle_backend).into(),
             runtime,
             worker_manager,
+            history_reporter: Some(history_reporter),
         }
     }
+
+    pub(crate) fn take_history_reporter(&mut self) -> Option> {
+        self.history_reporter.take()
+    }
 }
diff --git a/crates/sail-execution/src/driver/registry.rs b/crates/sail-execution/src/driver/registry.rs
index 6d9aba2cc..a57aa5fba 100644
--- a/crates/sail-execution/src/driver/registry.rs
+++ b/crates/sail-execution/src/driver/registry.rs
@@ -38,7 +38,7 @@ impl DriverHandle {
     pub async fn shutdown(&self) -> ExecutionResult<()> {
         // A closed channel means that the driver actor has already stopped.
         // Shutdown is intentionally idempotent, so this is still a success.
-        let _ = self.send(DriverEvent::Shutdown { history: None }).await;
+        let _ = self.send(DriverEvent::Shutdown { result: None }).await;
         Ok(())
     }
 
@@ -46,7 +46,7 @@ impl DriverHandle {
     pub async fn shutdown_and_wait(&self) -> ExecutionResult<()> {
         let (tx, rx) = oneshot::channel();
         if self
-            .send(DriverEvent::Shutdown { history: Some(tx) })
+            .send(DriverEvent::Shutdown { result: Some(tx) })
             .await
             .is_ok()
         {
diff --git a/crates/sail-execution/src/driver/worker_pool/core.rs b/crates/sail-execution/src/driver/worker_pool/core.rs
index a824bed64..7f46badbc 100644
--- a/crates/sail-execution/src/driver/worker_pool/core.rs
+++ b/crates/sail-execution/src/driver/worker_pool/core.rs
@@ -40,7 +40,7 @@ impl WorkerPool {
     pub fn start_worker(&mut self, ctx: &mut ActorContext) {
         let Ok(worker_id) = self.worker_id_generator.generate() else {
             error!("failed to generate worker ID");
-            ctx.send(DriverEvent::Shutdown { history: None });
+            ctx.send(DriverEvent::Shutdown { result: None });
             return;
         };
         let descriptor = WorkerDescriptor {
diff --git a/crates/sail-execution/src/job_runner.rs b/crates/sail-execution/src/job_runner.rs
index 048daf5ef..6ba95873a 100644
--- a/crates/sail-execution/src/job_runner.rs
+++ b/crates/sail-execution/src/job_runner.rs
@@ -5,7 +5,7 @@ use datafusion::common::{DataFusionError, Result, internal_datafusion_err, inter
 use datafusion::execution::SendableRecordBatchStream;
 use datafusion::physical_plan::{ExecutionPlan, execute_stream};
 use datafusion::prelude::SessionContext;
-use sail_common_datafusion::session::job::{JobRunner, JobRunnerHistory};
+use sail_common_datafusion::session::job::{JobRunner, JobRunnerHistory, JobRunnerHistoryReporter};
 use sail_common_datafusion::system::observable::{JobRunnerObserver, Observer, StateObservable};
 use sail_server::actor::ActorSystem;
 use sail_telemetry::telemetry::global_metrics;
@@ -31,23 +31,19 @@ fn explain_job_graph(plan: Arc, use_blocking_shuffle: bool) -
 pub struct LocalJobRunner {
     next_job_id: AtomicU64,
     stopped: AtomicBool,
+    history_reporter: std::sync::Mutex>>,
 }
 
 impl LocalJobRunner {
-    pub fn new() -> Self {
+    pub fn new(history_reporter: Box) -> Self {
         Self {
             next_job_id: AtomicU64::new(1),
             stopped: AtomicBool::new(false),
+            history_reporter: std::sync::Mutex::new(Some(history_reporter)),
         }
     }
 }
 
-impl Default for LocalJobRunner {
-    fn default() -> Self {
-        Self::new()
-    }
-}
-
 #[tonic::async_trait]
 impl StateObservable for LocalJobRunner {
     async fn observe(&self, observer: JobRunnerObserver) {
@@ -81,14 +77,23 @@ impl JobRunner for LocalJobRunner {
         Ok(execute_stream(plan, ctx.task_ctx())?)
     }
 
-    async fn stop(&self, history: oneshot::Sender) {
+    async fn stop(&self) {
         self.stopped.store(true, Ordering::Relaxed);
-        let _ = history.send(JobRunnerHistory {
-            jobs: vec![],
-            stages: vec![],
-            tasks: vec![],
-            workers: vec![],
-        });
+        let history_reporter = self
+            .history_reporter
+            .lock()
+            .ok()
+            .and_then(|mut history_reporter| history_reporter.take());
+        if let Some(history_reporter) = history_reporter {
+            history_reporter
+                .report(JobRunnerHistory {
+                    jobs: vec![],
+                    stages: vec![],
+                    tasks: vec![],
+                    workers: vec![],
+                })
+                .await;
+        }
     }
 }
 
@@ -156,12 +161,7 @@ impl JobRunner for ClusterJobRunner {
             .map_err(|e| internal_datafusion_err!("{e}"))
     }
 
-    async fn stop(&self, history: oneshot::Sender) {
-        let _ = self
-            .driver
-            .send(DriverEvent::Shutdown {
-                history: Some(history),
-            })
-            .await;
+    async fn stop(&self) {
+        let _ = self.driver.shutdown_and_wait().await;
     }
 }
diff --git a/crates/sail-session/src/session_factory/job_runner.rs b/crates/sail-session/src/session_factory/job_runner.rs
index 3b63c003d..f04e25358 100644
--- a/crates/sail-session/src/session_factory/job_runner.rs
+++ b/crates/sail-session/src/session_factory/job_runner.rs
@@ -4,7 +4,7 @@ use std::sync::{Arc, Mutex};
 use datafusion::common::{Result, internal_err};
 use sail_common::config::{AppConfig, ExecutionMode};
 use sail_common::runtime::RuntimeHandle;
-use sail_common_datafusion::session::job::JobRunner;
+use sail_common_datafusion::session::job::{JobRunner, JobRunnerHistoryReporter};
 use sail_execution::DriverId;
 use sail_execution::driver::{DriverHandle, DriverOptions};
 use sail_execution::job_runner::{ClusterJobRunner, LocalJobRunner};
@@ -44,6 +44,7 @@ impl SessionJobRunner {
 pub struct SessionJobRunnerInfo {
     pub driver_id: DriverId,
     pub driver_server_port: Option,
+    pub history_reporter: Box,
 }
 
 pub trait SessionJobRunnerFactory: Send {
@@ -83,6 +84,7 @@ impl ServerSessionJobRunnerFactory {
             info.driver_id,
             port,
             worker_manager,
+            info.history_reporter,
         );
         let mut system = self
             .system
@@ -98,7 +100,9 @@ impl ServerSessionJobRunnerFactory {
 impl SessionJobRunnerFactory for ServerSessionJobRunnerFactory {
     fn create(&mut self, info: SessionJobRunnerInfo) -> Result {
         match self.config.mode {
-            ExecutionMode::Local => Ok(SessionJobRunner::local(LocalJobRunner::new())),
+            ExecutionMode::Local => Ok(SessionJobRunner::local(LocalJobRunner::new(
+                info.history_reporter,
+            ))),
             ExecutionMode::LocalCluster => {
                 let worker_session =
                     WorkerSessionFactory::new(self.config.clone(), self.runtime.clone())
diff --git a/crates/sail-session/src/session_manager/actor/handler.rs b/crates/sail-session/src/session_manager/actor/handler.rs
index 0ace99e33..5604c5d12 100644
--- a/crates/sail-session/src/session_manager/actor/handler.rs
+++ b/crates/sail-session/src/session_manager/actor/handler.rs
@@ -8,7 +8,9 @@ use log::{info, warn};
 use sail_cache::remote_checkpoint::RemoteCheckpointRegistry;
 use sail_common_datafusion::extension::SessionExtensionAccessor;
 use sail_common_datafusion::session::activity::ActivityTracker;
-use sail_common_datafusion::session::job::JobService;
+use sail_common_datafusion::session::job::{
+    JobRunnerHistory, JobRunnerHistoryReporter, JobService,
+};
 use sail_common_datafusion::system::catalog::{OptionRow, SessionRow};
 use sail_common_datafusion::system::observable::{JobRunnerObserver, SessionManagerObserver};
 use sail_common_datafusion::system::predicate::PredicateExt;
@@ -25,6 +27,26 @@ use crate::session_manager::actor::SessionManagerActor;
 use crate::session_manager::event::{SessionHistory, SessionManagerEvent};
 use crate::session_manager::session::{ServerSession, ServerSessionState};
 
+struct SessionJobRunnerHistoryReporter {
+    session_id: String,
+    session_manager: sail_server::actor::ActorHandle,
+}
+
+#[tonic::async_trait]
+impl JobRunnerHistoryReporter for SessionJobRunnerHistoryReporter {
+    async fn report(self: Box, history: JobRunnerHistory) {
+        let _ = self
+            .session_manager
+            .send(SessionManagerEvent::SetSessionHistory {
+                session_id: self.session_id,
+                history: SessionHistory {
+                    job_runner: history,
+                },
+            })
+            .await;
+    }
+}
+
 impl SessionManagerActor {
     pub(super) fn handle_get_driver(
         &self,
@@ -43,7 +65,7 @@ impl SessionManagerActor {
         result: oneshot::Sender>,
     ) -> ActorAction {
         let context = if let Some(session) = self.sessions.get(&session_id) {
-            if let ServerSessionState::Running { context } = &session.state {
+            if let ServerSessionState::Running { context, .. } = &session.state {
                 Ok(context.clone())
             } else {
                 Err(SessionError::invalid(format!(
@@ -69,6 +91,10 @@ impl SessionManagerActor {
             let runner = self.job_runner_factory.create(SessionJobRunnerInfo {
                 driver_id,
                 driver_server_port: self.gateway.as_ref().map(|x| x.port()),
+                history_reporter: Box::new(SessionJobRunnerHistoryReporter {
+                    session_id: session_id.clone(),
+                    session_manager: ctx.handle().clone(),
+                }),
             });
             match runner {
                 Ok(runner) => {
@@ -77,6 +103,13 @@ impl SessionManagerActor {
                     if let Some(driver) = &driver
                         && let Err(e) = self.drivers.insert(driver_id, driver.clone())
                     {
+                        let session = ServerSession {
+                            user_id,
+                            created_at: Utc::now(),
+                            deleted_at: None,
+                            state: ServerSessionState::Failed,
+                        };
+                        self.sessions.insert(session_id, session);
                         let driver = driver.clone();
                         ctx.spawn(async move {
                             if let Err(e) = driver.shutdown().await {
@@ -108,8 +141,8 @@ impl SessionManagerActor {
                                 deleted_at: None,
                                 state: ServerSessionState::Running {
                                     context: context.clone(),
+                                    driver_id: registered_driver_id,
                                 },
-                                driver_id: registered_driver_id,
                             };
                             self.sessions.insert(session_id, session);
                             Ok(context)
@@ -124,6 +157,13 @@ impl SessionManagerActor {
                                     }
                                 });
                             }
+                            let session = ServerSession {
+                                user_id,
+                                created_at: Utc::now(),
+                                deleted_at: None,
+                                state: ServerSessionState::Failed,
+                            };
+                            self.sessions.insert(session_id, session);
                             Err(e.into())
                         }
                     }
@@ -156,14 +196,16 @@ impl SessionManagerActor {
     ) -> ActorAction {
         let session = self.sessions.get_mut(&session_id);
         if let Some(session) = session
-            && let ServerSessionState::Running { context } = &mut session.state
+            && let ServerSessionState::Running { context, driver_id } = &mut session.state
             && let Ok(tracker) = context.extension::()
             && tracker.active_at().is_ok_and(|x| x <= instant)
         {
             info!("removing idle session {session_id}");
             Self::delete_session(ctx, session_id, context);
             session.deleted_at = Some(Utc::now());
-            session.state = ServerSessionState::Deleting;
+            session.state = ServerSessionState::Deleting {
+                driver_id: *driver_id,
+            };
         }
         ActorAction::Continue
     }
@@ -176,11 +218,13 @@ impl SessionManagerActor {
     ) -> ActorAction {
         let session = self.sessions.get_mut(&session_id);
         let output = if let Some(session) = session {
-            if let ServerSessionState::Running { context } = &mut session.state {
+            if let ServerSessionState::Running { context, driver_id } = &mut session.state {
                 info!("removing session {session_id}");
                 Self::delete_session(ctx, session_id, context);
                 session.deleted_at = Some(Utc::now());
-                session.state = ServerSessionState::Deleting;
+                session.state = ServerSessionState::Deleting {
+                    driver_id: *driver_id,
+                };
                 Ok(())
             } else {
                 Err(SessionError::invalid(format!(
@@ -206,15 +250,19 @@ impl SessionManagerActor {
             warn!("session not found: {session_id}");
             return ActorAction::Continue;
         };
-        if let Some(driver_id) = session.driver_id.take() {
-            self.drivers.remove(driver_id);
-        }
-        if matches!(session.state, ServerSessionState::Deleting) {
-            session.state = ServerSessionState::Deleted {
-                history: Arc::new(history),
-            };
-        } else {
-            warn!("session is not being deleted: {session_id}");
+        match &mut session.state {
+            ServerSessionState::Running { driver_id, .. }
+            | ServerSessionState::Deleting { driver_id } => {
+                if let Some(driver_id) = driver_id.take() {
+                    self.drivers.remove(driver_id);
+                }
+                session.state = ServerSessionState::Deleted {
+                    history: Arc::new(history),
+                };
+            }
+            ServerSessionState::Deleted { .. } | ServerSessionState::Failed => {
+                warn!("session is not being deleted: {session_id}");
+            }
         }
         ActorAction::Continue
     }
@@ -228,7 +276,12 @@ impl SessionManagerActor {
             warn!("session not found: {session_id}");
             return ActorAction::Continue;
         };
-        if let Some(driver_id) = session.driver_id.take()
+        let driver_id = match &session.state {
+            ServerSessionState::Running { driver_id, .. }
+            | ServerSessionState::Deleting { driver_id } => *driver_id,
+            ServerSessionState::Deleted { .. } | ServerSessionState::Failed => None,
+        };
+        if let Some(driver_id) = driver_id
             && let Some(driver) = self.drivers.remove(driver_id)
         {
             ctx.spawn(async move {
@@ -402,12 +455,9 @@ impl SessionManagerActor {
         };
         let checkpoint_registry = context.extension::().ok();
         let runtime_env = context.runtime_env();
-        let handle = ctx.handle().clone();
-        let (tx, rx) = oneshot::channel();
         ctx.spawn(async move {
             // Stop tasks before deleting the namespace so late attempts cannot recreate objects.
-            service.runner().stop(tx).await;
-            let history = rx.await;
+            service.runner().stop().await;
             if let Some(checkpoint_registry) = checkpoint_registry
                 && let Err(error) = checkpoint_registry
                     .cleanup_session(runtime_env.as_ref())
@@ -415,14 +465,6 @@ impl SessionManagerActor {
             {
                 warn!("failed to clean checkpoints for session {session_id}: {error}");
             }
-            let message = match history {
-                Ok(x) => SessionManagerEvent::SetSessionHistory {
-                    session_id,
-                    history: SessionHistory { job_runner: x },
-                },
-                Err(_) => SessionManagerEvent::SetSessionFailure { session_id },
-            };
-            let _ = handle.send(message).await;
         });
     }
 }
diff --git a/crates/sail-session/src/session_manager/session.rs b/crates/sail-session/src/session_manager/session.rs
index 8ac77937b..49ccb9970 100644
--- a/crates/sail-session/src/session_manager/session.rs
+++ b/crates/sail-session/src/session_manager/session.rs
@@ -20,7 +20,6 @@ pub struct ServerSession {
     pub created_at: DateTime,
     pub deleted_at: Option>,
     pub state: ServerSessionState,
-    pub driver_id: Option,
 }
 
 impl ServerSession {
@@ -34,7 +33,7 @@ impl ServerSession {
     {
         let (tx, rx) = oneshot::channel();
         let observer = observer(tx);
-        if let ServerSessionState::Running { context } = &self.state {
+        if let ServerSessionState::Running { context, .. } = &self.state {
             match context.extension::() {
                 Ok(service) => async move {
                     service.runner().observe(observer).await;
@@ -64,9 +63,16 @@ impl ServerSession {
 }
 
 pub enum ServerSessionState {
-    Running { context: SessionContext },
-    Deleting,
-    Deleted { history: Arc },
+    Running {
+        context: SessionContext,
+        driver_id: Option,
+    },
+    Deleting {
+        driver_id: Option,
+    },
+    Deleted {
+        history: Arc,
+    },
     Failed,
 }
 
@@ -74,7 +80,7 @@ impl ServerSessionState {
     pub fn status(&self) -> &'static str {
         match self {
             ServerSessionState::Running { .. } => "RUNNING",
-            ServerSessionState::Deleting => "DELETING",
+            ServerSessionState::Deleting { .. } => "DELETING",
             ServerSessionState::Deleted { .. } => "DELETED",
             ServerSessionState::Failed => "FAILED",
         }

From e2b130ea6e8fc750ae98c0375ef6e2c787e2b3b3 Mon Sep 17 00:00:00 2001
From: Heran Lin 
Date: Fri, 31 Jul 2026 11:09:58 +0800
Subject: [PATCH 104/131] refactor: specify components for actor creation
 (#2329)

---
 .../sail-execution/src/driver/actor/core.rs   | 22 ++++---
 crates/sail-execution/src/driver/actor/mod.rs |  2 +
 crates/sail-execution/src/driver/mod.rs       |  2 +-
 crates/sail-execution/src/driver/options.rs   | 16 ++---
 .../src/driver/worker_pool/mod.rs             |  4 +-
 crates/sail-execution/src/job_runner.rs       | 10 ++-
 .../src/session_factory/job_runner.rs         | 20 +++---
 .../src/session_manager/actor/core.rs         | 27 ++++----
 .../src/session_manager/actor/handler.rs      |  3 +-
 .../src/session_manager/actor/mod.rs          |  2 +-
 .../sail-session/src/session_manager/mod.rs   | 61 +++++++++----------
 .../src/session_manager/options.rs            | 28 +++------
 .../spark/execution/test_driver_server.py     |  3 +-
 13 files changed, 92 insertions(+), 108 deletions(-)

diff --git a/crates/sail-execution/src/driver/actor/core.rs b/crates/sail-execution/src/driver/actor/core.rs
index 4f533d369..06db2a1b6 100644
--- a/crates/sail-execution/src/driver/actor/core.rs
+++ b/crates/sail-execution/src/driver/actor/core.rs
@@ -6,29 +6,32 @@ use sail_server::actor::{Actor, ActorAction, ActorContext};
 use crate::driver::job_scheduler::{JobScheduler, JobSchedulerOptions};
 use crate::driver::task_assigner::{TaskAssigner, TaskAssignerOptions};
 use crate::driver::worker_pool::{WorkerPool, WorkerPoolOptions};
-use crate::driver::{DriverActor, DriverEvent, DriverOptions};
+use crate::driver::{DriverActor, DriverComponents, DriverEvent, DriverOptions};
 use crate::stream_manager::{StreamManager, StreamManagerOptions};
 use crate::task_runner::TaskRunner;
 
 #[tonic::async_trait]
 impl Actor for DriverActor {
     type Message = DriverEvent;
-    type Options = DriverOptions;
+    type Options = (DriverOptions, DriverComponents);
 
     fn name() -> &'static str {
         "DriverActor"
     }
 
-    fn new(options: DriverOptions) -> Self {
-        let worker_pool = WorkerPool::new(
-            options.worker_manager.clone(),
-            WorkerPoolOptions::from(&options),
-        );
+    fn new(options: Self::Options) -> Self {
+        let (options, components) = options;
+        let DriverComponents {
+            worker_manager,
+            history_reporter,
+        } = components;
+        let worker_pool = WorkerPool::new(worker_manager, WorkerPoolOptions::from(&options));
         let job_scheduler = JobScheduler::new(JobSchedulerOptions::from(&options));
         let task_assigner = TaskAssigner::new(TaskAssignerOptions::from(&options));
         let stream_manager = StreamManager::new(StreamManagerOptions::from(&options));
         Self {
             options,
+            history_reporter,
             worker_pool,
             job_scheduler,
             task_assigner,
@@ -119,9 +122,8 @@ impl Actor for DriverActor {
         if let Err(e) = self.worker_pool.close(ctx).await {
             error!("encountered error while stopping workers: {e}");
         }
-        if let Some(history_reporter) = self.options.take_history_reporter() {
-            history_reporter.report(self.build_history()).await;
-        }
+        let history = self.build_history();
+        self.history_reporter.report(history).await;
         if let Some(result) = self.shutdown_notifier.take() {
             let _ = result.send(());
         }
diff --git a/crates/sail-execution/src/driver/actor/mod.rs b/crates/sail-execution/src/driver/actor/mod.rs
index e95cd6878..49b0647eb 100644
--- a/crates/sail-execution/src/driver/actor/mod.rs
+++ b/crates/sail-execution/src/driver/actor/mod.rs
@@ -3,6 +3,7 @@ mod handler;
 
 use std::collections::HashMap;
 
+use sail_common_datafusion::session::job::JobRunnerHistoryReporter;
 use tokio::sync::oneshot;
 
 use crate::driver::job_scheduler::JobScheduler;
@@ -13,6 +14,7 @@ use crate::task_runner::TaskRunner;
 
 pub struct DriverActor {
     options: super::options::DriverOptions,
+    history_reporter: Box,
     worker_pool: super::worker_pool::WorkerPool,
     job_scheduler: JobScheduler,
     task_assigner: TaskAssigner,
diff --git a/crates/sail-execution/src/driver/mod.rs b/crates/sail-execution/src/driver/mod.rs
index a8728614a..a198fc475 100644
--- a/crates/sail-execution/src/driver/mod.rs
+++ b/crates/sail-execution/src/driver/mod.rs
@@ -23,5 +23,5 @@ pub(crate) use client::DriverClientSet;
 pub(crate) use event::{DriverEvent, TaskStatus};
 pub use gateway::{DriverGateway, DriverGatewayOptions};
 pub(crate) use r#gen::driver_service_client::DriverServiceClient;
-pub use options::DriverOptions;
+pub use options::{DriverComponents, DriverOptions};
 pub use registry::{DriverHandle, DriverRegistry, DriverRegistryAccessor};
diff --git a/crates/sail-execution/src/driver/options.rs b/crates/sail-execution/src/driver/options.rs
index ef83a26bd..86b4e359c 100644
--- a/crates/sail-execution/src/driver/options.rs
+++ b/crates/sail-execution/src/driver/options.rs
@@ -1,4 +1,3 @@
-use std::sync::Arc;
 use std::time::Duration;
 
 use sail_common::config::AppConfig;
@@ -31,8 +30,11 @@ pub struct DriverOptions {
     pub shuffle_backend: ShuffleBackendKind,
     pub rpc_retry_strategy: RetryStrategy,
     pub runtime: RuntimeHandle,
-    pub worker_manager: Arc,
-    pub history_reporter: Option>,
+}
+
+pub struct DriverComponents {
+    pub worker_manager: Box,
+    pub history_reporter: Box,
 }
 
 impl DriverOptions {
@@ -41,8 +43,6 @@ impl DriverOptions {
         runtime: RuntimeHandle,
         driver_id: DriverId,
         driver_server_port: u16,
-        worker_manager: Arc,
-        history_reporter: Box,
     ) -> Self {
         Self {
             enable_tls: config.cluster.enable_tls,
@@ -70,12 +70,6 @@ impl DriverOptions {
             task_max_attempts: config.cluster.task_max_attempts,
             shuffle_backend: (&config.cluster.shuffle_backend).into(),
             runtime,
-            worker_manager,
-            history_reporter: Some(history_reporter),
         }
     }
-
-    pub(crate) fn take_history_reporter(&mut self) -> Option> {
-        self.history_reporter.take()
-    }
 }
diff --git a/crates/sail-execution/src/driver/worker_pool/mod.rs b/crates/sail-execution/src/driver/worker_pool/mod.rs
index 51a8ff3a9..69dc2608a 100644
--- a/crates/sail-execution/src/driver/worker_pool/mod.rs
+++ b/crates/sail-execution/src/driver/worker_pool/mod.rs
@@ -20,10 +20,10 @@ pub struct WorkerPool {
 }
 
 impl WorkerPool {
-    pub fn new(worker_manager: Arc, options: WorkerPoolOptions) -> Self {
+    pub fn new(worker_manager: Box, options: WorkerPoolOptions) -> Self {
         Self {
             options,
-            worker_manager,
+            worker_manager: Arc::from(worker_manager),
             workers: IndexMap::new(),
             worker_id_generator: IdGenerator::new(),
         }
diff --git a/crates/sail-execution/src/job_runner.rs b/crates/sail-execution/src/job_runner.rs
index 6ba95873a..071bf425d 100644
--- a/crates/sail-execution/src/job_runner.rs
+++ b/crates/sail-execution/src/job_runner.rs
@@ -13,7 +13,7 @@ use sail_telemetry::{TracingExecOptions, trace_execution_plan};
 use tokio::sync::mpsc::error::SendError;
 use tokio::sync::oneshot;
 
-use crate::driver::{DriverActor, DriverEvent, DriverHandle, DriverOptions};
+use crate::driver::{DriverActor, DriverComponents, DriverEvent, DriverHandle, DriverOptions};
 use crate::job_graph::{JobGraph, JobGraphOptions};
 use crate::shuffle::ShuffleBackendKind;
 
@@ -103,9 +103,13 @@ pub struct ClusterJobRunner {
 }
 
 impl ClusterJobRunner {
-    pub fn new(system: &mut ActorSystem, options: DriverOptions) -> Self {
+    pub fn new(
+        system: &mut ActorSystem,
+        options: DriverOptions,
+        components: DriverComponents,
+    ) -> Self {
         let shuffle_backend = options.shuffle_backend.clone();
-        let driver = DriverHandle::new(system.spawn::(options));
+        let driver = DriverHandle::new(system.spawn::((options, components)));
         Self {
             driver,
             shuffle_backend,
diff --git a/crates/sail-session/src/session_factory/job_runner.rs b/crates/sail-session/src/session_factory/job_runner.rs
index f04e25358..492d6dca2 100644
--- a/crates/sail-session/src/session_factory/job_runner.rs
+++ b/crates/sail-session/src/session_factory/job_runner.rs
@@ -6,7 +6,7 @@ use sail_common::config::{AppConfig, ExecutionMode};
 use sail_common::runtime::RuntimeHandle;
 use sail_common_datafusion::session::job::{JobRunner, JobRunnerHistoryReporter};
 use sail_execution::DriverId;
-use sail_execution::driver::{DriverHandle, DriverOptions};
+use sail_execution::driver::{DriverComponents, DriverHandle, DriverOptions};
 use sail_execution::job_runner::{ClusterJobRunner, LocalJobRunner};
 use sail_execution::worker_manager::{
     KubernetesWorkerManager, KubernetesWorkerManagerOptions, LocalWorkerManager,
@@ -73,19 +73,16 @@ impl ServerSessionJobRunnerFactory {
     fn create_cluster_runner(
         &self,
         info: SessionJobRunnerInfo,
-        worker_manager: Arc,
+        worker_manager: Box,
     ) -> Result {
         let Some(port) = info.driver_server_port else {
             return internal_err!("driver gateway is not available");
         };
-        let options = DriverOptions::new(
-            &self.config,
-            self.runtime.clone(),
-            info.driver_id,
-            port,
+        let options = DriverOptions::new(&self.config, self.runtime.clone(), info.driver_id, port);
+        let components = DriverComponents {
             worker_manager,
-            info.history_reporter,
-        );
+            history_reporter: info.history_reporter,
+        };
         let mut system = self
             .system
             .lock()
@@ -93,6 +90,7 @@ impl ServerSessionJobRunnerFactory {
         Ok(SessionJobRunner::cluster(ClusterJobRunner::new(
             system.deref_mut(),
             options,
+            components,
         )))
     }
 }
@@ -109,7 +107,7 @@ impl SessionJobRunnerFactory for ServerSessionJobRunnerFactory {
                         .create(())?;
                 self.create_cluster_runner(
                     info,
-                    Arc::new(LocalWorkerManager::new(
+                    Box::new(LocalWorkerManager::new(
                         self.runtime.clone(),
                         worker_session,
                     )),
@@ -129,7 +127,7 @@ impl SessionJobRunnerFactory for ServerSessionJobRunnerFactory {
                         .clone(),
                     worker_pod_template: self.config.kubernetes.worker_pod_template.clone(),
                 };
-                self.create_cluster_runner(info, Arc::new(KubernetesWorkerManager::new(options)))
+                self.create_cluster_runner(info, Box::new(KubernetesWorkerManager::new(options)))
             }
         }
     }
diff --git a/crates/sail-session/src/session_manager/actor/core.rs b/crates/sail-session/src/session_manager/actor/core.rs
index ef4b1c439..b9eb960b2 100644
--- a/crates/sail-session/src/session_manager/actor/core.rs
+++ b/crates/sail-session/src/session_manager/actor/core.rs
@@ -9,7 +9,7 @@ use sail_server::actor::{Actor, ActorAction, ActorContext, ActorHandle};
 
 use crate::session_manager::actor::SessionManagerActor;
 use crate::session_manager::event::SessionManagerEvent;
-use crate::session_manager::options::SessionManagerOptions;
+use crate::session_manager::options::{SessionManagerComponents, SessionManagerOptions};
 
 struct SessionDriverRegistry {
     handle: ActorHandle,
@@ -33,36 +33,39 @@ impl DriverRegistryAccessor for SessionDriverRegistry {
 #[tonic::async_trait]
 impl Actor for SessionManagerActor {
     type Message = SessionManagerEvent;
-    type Options = SessionManagerOptions;
+    type Options = (SessionManagerOptions, SessionManagerComponents);
 
     fn name() -> &'static str {
         "SessionManagerActor"
     }
 
-    fn new(mut options: Self::Options) -> Self {
-        let session_factory = (options.session_factory)();
-        let job_runner_factory = (options.job_runner_factory)();
-        let gateway = options.take_driver_gateway();
+    fn new(options: Self::Options) -> Self {
+        let (options, components) = options;
+        let SessionManagerComponents {
+            session_factory,
+            job_runner_factory,
+            driver_gateway,
+        } = components;
         Self {
             options,
             session_factory,
             job_runner_factory,
             sessions: IndexMap::new(),
             drivers: Default::default(),
-            gateway,
+            driver_gateway,
             driver_id_generator: IdGenerator::new(),
             shutdown_notifier: None,
         }
     }
 
     async fn start(&mut self, ctx: &mut ActorContext) {
-        let Some(gateway) = &mut self.gateway else {
+        let Some(driver_gateway) = &mut self.driver_gateway else {
             return;
         };
-        gateway.start(Arc::new(SessionDriverRegistry {
+        driver_gateway.start(Arc::new(SessionDriverRegistry {
             handle: ctx.handle().clone(),
         }));
-        info!("driver server is ready on port {}", gateway.port());
+        info!("driver server is ready on port {}", driver_gateway.port());
     }
 
     fn receive(&mut self, ctx: &mut ActorContext, message: Self::Message) -> ActorAction {
@@ -108,8 +111,8 @@ impl Actor for SessionManagerActor {
                 warn!("failed to shut down driver {driver_id}: {e}");
             }
         }
-        if let Some(mut gateway) = self.gateway {
-            gateway.stop().await;
+        if let Some(mut driver_gateway) = self.driver_gateway {
+            driver_gateway.stop().await;
             info!("driver server has stopped");
         }
         if let Some(result) = self.shutdown_notifier {
diff --git a/crates/sail-session/src/session_manager/actor/handler.rs b/crates/sail-session/src/session_manager/actor/handler.rs
index 5604c5d12..cf622d828 100644
--- a/crates/sail-session/src/session_manager/actor/handler.rs
+++ b/crates/sail-session/src/session_manager/actor/handler.rs
@@ -90,7 +90,7 @@ impl SessionManagerActor {
             };
             let runner = self.job_runner_factory.create(SessionJobRunnerInfo {
                 driver_id,
-                driver_server_port: self.gateway.as_ref().map(|x| x.port()),
+                driver_server_port: self.driver_gateway.as_ref().map(|x| x.port()),
                 history_reporter: Box::new(SessionJobRunnerHistoryReporter {
                     session_id: session_id.clone(),
                     session_manager: ctx.handle().clone(),
@@ -256,6 +256,7 @@ impl SessionManagerActor {
                 if let Some(driver_id) = driver_id.take() {
                     self.drivers.remove(driver_id);
                 }
+                session.deleted_at.get_or_insert_with(Utc::now);
                 session.state = ServerSessionState::Deleted {
                     history: Arc::new(history),
                 };
diff --git a/crates/sail-session/src/session_manager/actor/mod.rs b/crates/sail-session/src/session_manager/actor/mod.rs
index b1c772762..b82d12744 100644
--- a/crates/sail-session/src/session_manager/actor/mod.rs
+++ b/crates/sail-session/src/session_manager/actor/mod.rs
@@ -14,7 +14,7 @@ pub struct SessionManagerActor {
     job_runner_factory: Box,
     sessions: IndexMap,
     drivers: DriverRegistry,
-    gateway: Option,
+    driver_gateway: Option,
     driver_id_generator: IdGenerator,
     shutdown_notifier: Option>,
 }
diff --git a/crates/sail-session/src/session_manager/mod.rs b/crates/sail-session/src/session_manager/mod.rs
index ce5398718..584a8e96e 100644
--- a/crates/sail-session/src/session_manager/mod.rs
+++ b/crates/sail-session/src/session_manager/mod.rs
@@ -20,7 +20,7 @@ use crate::session_factory::{
 };
 pub(crate) use crate::session_manager::actor::SessionManagerActor;
 pub(crate) use crate::session_manager::event::SessionManagerEvent;
-pub use crate::session_manager::options::SessionManagerOptions;
+pub use crate::session_manager::options::{SessionManagerComponents, SessionManagerOptions};
 
 pub type ServerSessionFactoryFn =
     fn(Arc, RuntimeHandle) -> Box>;
@@ -37,9 +37,14 @@ impl fmt::Debug for SessionManager {
 }
 
 impl SessionManager {
-    pub fn try_new(options: SessionManagerOptions) -> SessionResult {
+    pub fn try_new(
+        options: SessionManagerOptions,
+        components: SessionManagerComponents,
+    ) -> SessionResult {
         let system = options.system.clone();
-        let handle = system.lock()?.spawn::(options);
+        let handle = system
+            .lock()?
+            .spawn::((options, components));
         Ok(Self { handle })
     }
 
@@ -90,24 +95,13 @@ pub async fn create_session_manager(
     session_timeout: Duration,
 ) -> SessionResult {
     let system = Arc::new(Mutex::new(ActorSystem::new()));
-    let session_factory = {
-        let config = config.clone();
-        let runtime = runtime.clone();
-        Box::new(move || session_factory_fn(config.clone(), runtime.clone()))
-    };
-    let job_runner_factory = {
-        let config = config.clone();
-        let runtime = runtime.clone();
-        let system = system.clone();
-        Box::new(move || {
-            Box::new(ServerSessionJobRunnerFactory::new(
-                config.clone(),
-                runtime.clone(),
-                system.clone(),
-            )) as Box
-        })
-    };
-    let gateway = if matches!(&config.mode, ExecutionMode::Local) {
+    let session_factory = session_factory_fn(config.clone(), runtime.clone());
+    let job_runner_factory = Box::new(ServerSessionJobRunnerFactory::new(
+        config.clone(),
+        runtime.clone(),
+        system.clone(),
+    )) as Box;
+    let driver_gateway = if matches!(&config.mode, ExecutionMode::Local) {
         None
     } else {
         Some(
@@ -118,16 +112,17 @@ pub async fn create_session_manager(
                 })?,
         )
     };
-    let mut options =
-        SessionManagerOptions::new(runtime, system, session_factory, job_runner_factory)
-            .with_session_timeout(session_timeout)
-            .with_options(
-                config
-                    .raw()
-                    .map_err(|e| SessionError::internal(e.to_string()))?,
-            );
-    if let Some(gateway) = gateway {
-        options = options.with_driver_gateway(gateway);
-    }
-    SessionManager::try_new(options)
+    let options = SessionManagerOptions::new(runtime, system)
+        .with_session_timeout(session_timeout)
+        .with_options(
+            config
+                .raw()
+                .map_err(|e| SessionError::internal(e.to_string()))?,
+        );
+    let components = SessionManagerComponents {
+        session_factory,
+        job_runner_factory,
+        driver_gateway,
+    };
+    SessionManager::try_new(options, components)
 }
diff --git a/crates/sail-session/src/session_manager/options.rs b/crates/sail-session/src/session_manager/options.rs
index 80d98af8b..a9ab103c4 100644
--- a/crates/sail-session/src/session_manager/options.rs
+++ b/crates/sail-session/src/session_manager/options.rs
@@ -12,37 +12,27 @@ pub struct SessionManagerOptions {
     pub session_timeout: Duration,
     pub runtime: RuntimeHandle,
     pub system: Arc>,
-    pub session_factory: Box Box> + Send>,
-    pub job_runner_factory: Box Box + Send>,
-    pub driver_gateway: Option,
     /// The application configuration options as key-value pairs,
     /// used to populate the `system.session.options` table.
     pub options: Vec<(String, String)>,
 }
 
+pub struct SessionManagerComponents {
+    pub session_factory: Box>,
+    pub job_runner_factory: Box,
+    pub driver_gateway: Option,
+}
+
 impl SessionManagerOptions {
-    pub fn new(
-        runtime: RuntimeHandle,
-        system: Arc>,
-        session_factory: Box Box> + Send>,
-        job_runner_factory: Box Box + Send>,
-    ) -> Self {
+    pub fn new(runtime: RuntimeHandle, system: Arc>) -> Self {
         Self {
             session_timeout: Duration::MAX,
             runtime,
             system,
-            session_factory,
-            job_runner_factory,
-            driver_gateway: None,
             options: Vec::new(),
         }
     }
 
-    pub fn with_driver_gateway(mut self, gateway: DriverGateway) -> Self {
-        self.driver_gateway = Some(gateway);
-        self
-    }
-
     pub fn with_session_timeout(mut self, timeout: Duration) -> Self {
         self.session_timeout = timeout;
         self
@@ -52,8 +42,4 @@ impl SessionManagerOptions {
         self.options = options;
         self
     }
-
-    pub(crate) fn take_driver_gateway(&mut self) -> Option {
-        self.driver_gateway.take()
-    }
 }
diff --git a/python/pysail/tests/spark/execution/test_driver_server.py b/python/pysail/tests/spark/execution/test_driver_server.py
index 54b9e9f42..1aec34669 100644
--- a/python/pysail/tests/spark/execution/test_driver_server.py
+++ b/python/pysail/tests/spark/execution/test_driver_server.py
@@ -4,9 +4,8 @@
 
 from pysail.spark import SparkConnectServer
 from pysail.testing.spark.session import spark_connect_server, spark_session_factory
-from pysail.testing.spark.utils.common import is_jvm_spark
 
-pytestmark = pytest.mark.skipif(is_jvm_spark(), reason="Sail local-cluster mode only")
+pytestmark = pytest.mark.skip(reason="flaky tests that should be run manually")
 
 
 @pytest.fixture(scope="module")

From 1327bbb872fcc046f30f5fdc011881585acdf9d9 Mon Sep 17 00:00:00 2001
From: Heran Lin 
Date: Fri, 31 Jul 2026 12:22:59 +0800
Subject: [PATCH 105/131] feat!: remove the `extension` option for Parquet data
 source (#2330)

---
 .../data/options/parquet.yaml                 | 25 ---------------
 .../src/formats/parquet/options.rs            | 32 -------------------
 .../tests/spark/datasource/test_parquet.py    | 30 ++++-------------
 3 files changed, 6 insertions(+), 81 deletions(-)

diff --git a/crates/sail-data-source/data/options/parquet.yaml b/crates/sail-data-source/data/options/parquet.yaml
index ec89b44a9..9bbb55cd6 100644
--- a/crates/sail-data-source/data/options/parquet.yaml
+++ b/crates/sail-data-source/data/options/parquet.yaml
@@ -28,31 +28,6 @@
         - option.pathGlobFilter
       parser: crate::options::parsers::parse_optional_string
 
-- key: extension
-  description: |
-    (Reading) The file extension used to filter files when listing a
-    directory of Parquet files. Defaults to `.parquet`. Setting this to
-    an empty string disables extension filtering entirely, which allows
-    reading Parquet files written under non-standard suffixes (e.g. `.hive`).
-  default:
-    value: ".parquet"
-    parser: crate::options::parsers::parse_string
-  supported: true
-  rust_type: String
-  scopes:
-    - read
-  origins:
-    - type: option
-      keys:
-        - extension
-        - fileExtension
-      parser: crate::options::parsers::parse_string
-    - type: table_property
-      keys:
-        - option.extension
-        - option.fileExtension
-      parser: crate::options::parsers::parse_string
-
 - key: enable_page_index
   description: |
     (Reading) Whether to enable page index when reading Parquet files.
diff --git a/crates/sail-data-source/src/formats/parquet/options.rs b/crates/sail-data-source/src/formats/parquet/options.rs
index 263cc0888..c4f2830ec 100644
--- a/crates/sail-data-source/src/formats/parquet/options.rs
+++ b/crates/sail-data-source/src/formats/parquet/options.rs
@@ -27,7 +27,6 @@ fn check_parquet_level_is_none(codec: &str, level: &Option) -> DataSourceRe
 impl BuildPartialOptions for TableParquetOptions {
     fn build_partial_options(self) -> DataSourceResult {
         Ok(ParquetReadPartialOptions {
-            extension: None,
             enable_page_index: Some(self.global.enable_page_index),
             pruning: Some(self.global.pruning),
             skip_metadata: Some(self.global.skip_metadata),
@@ -48,7 +47,6 @@ impl BuildPartialOptions for TableParquetOptions {
 impl ParquetReadOptions {
     pub fn into_table_options(self) -> TableParquetOptions {
         let ParquetReadOptions {
-            extension: _,
             enable_page_index,
             pruning,
             skip_metadata,
@@ -369,36 +367,6 @@ mod tests {
         Ok(())
     }
 
-    #[test]
-    fn test_resolve_parquet_read_extension() -> datafusion_common::Result<()> {
-        let ctx = SessionContext::default();
-        let state = ctx.state();
-
-        // default: no option layer → ".parquet"
-        let opts = ParquetReadOptions::resolve(&state, vec![option_list(&[])])
-            .map_err(datafusion_common::DataFusionError::from)?;
-        assert_eq!(opts.extension, ".parquet");
-
-        // snake_case override
-        let opts =
-            ParquetReadOptions::resolve(&state, vec![option_list(&[("extension", ".hive")])])
-                .map_err(datafusion_common::DataFusionError::from)?;
-        assert_eq!(opts.extension, ".hive");
-
-        // camelCase alias
-        let opts =
-            ParquetReadOptions::resolve(&state, vec![option_list(&[("fileExtension", ".pq")])])
-                .map_err(datafusion_common::DataFusionError::from)?;
-        assert_eq!(opts.extension, ".pq");
-
-        // empty string disables filtering
-        let opts = ParquetReadOptions::resolve(&state, vec![option_list(&[("extension", "")])])
-            .map_err(datafusion_common::DataFusionError::from)?;
-        assert_eq!(opts.extension, "");
-
-        Ok(())
-    }
-
     #[test]
     fn test_resolve_parquet_read_options_with_global_default() -> datafusion_common::Result<()> {
         let ctx = SessionContext::default();
diff --git a/python/pysail/tests/spark/datasource/test_parquet.py b/python/pysail/tests/spark/datasource/test_parquet.py
index 8b37cb435..14bf81d88 100644
--- a/python/pysail/tests/spark/datasource/test_parquet.py
+++ b/python/pysail/tests/spark/datasource/test_parquet.py
@@ -206,15 +206,12 @@ def test_parquet_format_path(spark, sample_df, tmp_path):
 
 
 def test_parquet_read_with_custom_extension(spark, sample_pandas_df, tmp_path):
-    """Parquet files written under a non-standard suffix (e.g. `.hive`, as
-    emitted by Hive-managed tables) can be read via the `extension` /
-    `fileExtension` option."""
+    """`pathGlobFilter` selects Parquet files with a custom suffix."""
     directory = tmp_path / "parquet_custom_extension"
     directory.mkdir()
     file_path = directory / "data.hive"
     sample_pandas_df.to_parquet(str(file_path))
-    # TODO: add a file with another extension and the file should be ignored
-    #   when the filtering logic is implemented properly
+    sample_pandas_df.to_parquet(str(directory / "ignored.parquet"))
 
     expected_count = len(sample_pandas_df)
     expected_rows = sorted(sample_pandas_df.to_dict(orient="records"), key=safe_sort_key)
@@ -222,32 +219,17 @@ def test_parquet_read_with_custom_extension(spark, sample_pandas_df, tmp_path):
     def actual_rows(df):
         return sorted(df.toPandas().to_dict(orient="records"), key=safe_sort_key)
 
-    # Option key `extension`, directory path.
-    read_df = spark.read.option("extension", ".hive").parquet(str(directory))
+    read_df = spark.read.option("pathGlobFilter", "*.hive").parquet(str(directory))
     assert read_df.count() == expected_count
     assert actual_rows(read_df) == expected_rows
 
-    # Camel-case alias `fileExtension`, single-file path.
-    read_df = spark.read.option("fileExtension", ".hive").parquet(str(file_path))
-    assert read_df.count() == expected_count
-    assert actual_rows(read_df) == expected_rows
-
-    # Empty string disables extension filtering entirely.
-    read_df = spark.read.option("extension", "").parquet(str(directory))
-    assert read_df.count() == expected_count
-    assert actual_rows(read_df) == expected_rows
-
-    # SQL CREATE TABLE with OPTIONS (fileExtension '.hive').
-    # Use a separate directory so DROP TABLE side effects don't affect other cases.
-    sql_directory = tmp_path / "parquet_custom_extension_sql"
-    sql_directory.mkdir()
-    sample_pandas_df.to_parquet(str(sql_directory / "data.hive"))
+    # SQL CREATE TABLE with OPTIONS (pathGlobFilter '*.hive').
     table_name = "parquet_custom_extension_table"
     try:
         spark.sql(
             f"CREATE TABLE {table_name} USING parquet "
-            f"OPTIONS (fileExtension '.hive') "
-            f"LOCATION '{escape_sql_string_literal(str(sql_directory))}'"
+            f"OPTIONS (pathGlobFilter '*.hive') "
+            f"LOCATION '{escape_sql_string_literal(str(directory))}'"
         )
         read_df = spark.sql(f"SELECT * FROM {table_name}")  # noqa: S608
         assert read_df.count() == expected_count

From 7d646dd72dc5cd426b250b7c07941f9314b260cf Mon Sep 17 00:00:00 2001
From: Shehab Amin <11789402+shehabgamin@users.noreply.github.com>
Date: Fri, 31 Jul 2026 01:22:45 -0700
Subject: [PATCH 106/131] fix: preserve DDL parsing errors during schema
 fallback (#2332)

---
 crates/sail-spark-connect/src/error.rs        |  3 ++
 .../sail-spark-connect/src/proto/data_type.rs | 27 +++++++++---------
 .../tests/gold_data/data_type.json            | 28 +++++++++----------
 .../tests/gold_data/table_schema.json         | 10 +++----
 4 files changed, 36 insertions(+), 32 deletions(-)

diff --git a/crates/sail-spark-connect/src/error.rs b/crates/sail-spark-connect/src/error.rs
index 450a8ea86..4597f81f6 100644
--- a/crates/sail-spark-connect/src/error.rs
+++ b/crates/sail-spark-connect/src/error.rs
@@ -46,6 +46,8 @@ pub enum SparkError {
     InternalError(String),
     #[error("analysis error: {0}")]
     AnalysisError(String),
+    #[error("parse error: {0}")]
+    ParseError(String),
 }
 
 impl SparkError {
@@ -450,6 +452,7 @@ impl From for Status {
                 SparkThrowable::UnsupportedOperationException(s).into()
             }
             SparkError::AnalysisError(s) => SparkThrowable::AnalysisException(s).into(),
+            SparkError::ParseError(s) => SparkThrowable::ParseException(s).into(),
             e @ SparkError::SendError(_) => {
                 Status::cancelled(truncate_grpc_message(&e.to_string()))
             }
diff --git a/crates/sail-spark-connect/src/proto/data_type.rs b/crates/sail-spark-connect/src/proto/data_type.rs
index 677b3c5dd..903eb9a4d 100644
--- a/crates/sail-spark-connect/src/proto/data_type.rs
+++ b/crates/sail-spark-connect/src/proto/data_type.rs
@@ -27,20 +27,21 @@ pub(crate) const SPARK_DECIMAL_SYSTEM_DEFAULT_SCALE: i8 = 18;
 pub(crate) fn parse_spark_data_type(schema: &str) -> SparkResult {
     if let Ok(dt) = parse_data_type(schema).and_then(from_ast_data_type) {
         Ok(dt)
-    } else if let Ok(dt) =
-        parse_data_type(format!("struct<{schema}>").as_str()).and_then(from_ast_data_type)
-    {
-        match dt {
-            spec::DataType::Struct { fields } if fields.is_empty() => {
-                Err(SparkError::invalid("empty data type"))
-            }
-            // The SQL parser supports both `struct` and `struct` syntax.
-            // Therefore, by wrapping the input with `struct<...>`, we do not need separate logic
-            // to parse table schema input (`name type, ...`).
-            _ => Ok(dt),
-        }
     } else {
-        parse_spark_json_data_type(schema)?.try_into()
+        match parse_data_type(format!("struct<{schema}>").as_str()).and_then(from_ast_data_type) {
+            Ok(dt) => match dt {
+                spec::DataType::Struct { fields } if fields.is_empty() => {
+                    Err(SparkError::invalid("empty data type"))
+                }
+                // The SQL parser supports both `struct` and `struct` syntax.
+                // Therefore, by wrapping the input with `struct<...>`, we do not need separate logic
+                // to parse table schema input (`name type, ...`).
+                _ => Ok(dt),
+            },
+            Err(ddl_error) => parse_spark_json_data_type(schema)
+                .and_then(|dt| dt.try_into())
+                .map_err(|_| SparkError::ParseError(ddl_error.to_string())),
+        }
     }
 }
 
diff --git a/crates/sail-spark-connect/tests/gold_data/data_type.json b/crates/sail-spark-connect/tests/gold_data/data_type.json
index d2fbc42a4..ea7267a7d 100644
--- a/crates/sail-spark-connect/tests/gold_data/data_type.json
+++ b/crates/sail-spark-connect/tests/gold_data/data_type.json
@@ -362,7 +362,7 @@
     {
       "input": "TIME(6) WITHOUT TIME ZONE",
       "output": {
-        "failure": "error in JSON serde: expected value at line 1 column 1"
+        "failure": "parse error: error in SQL parser: found ( at 11:12 expected ':', or data type"
       }
     },
     {
@@ -400,7 +400,7 @@
     {
       "input": "TimE WiTHOUT TiME ZoNE",
       "output": {
-        "failure": "error in JSON serde: expected value at line 1 column 1"
+        "failure": "parse error: error in SQL parser: found WiTHOUT at 12:19 expected ':', or data type"
       }
     },
     {
@@ -541,7 +541,7 @@
       "input": "it is not a data type",
       "exception": "\n[PARSE_SYNTAX_ERROR] Syntax error at or near 'is'. SQLSTATE: 42601 (line 1, pos 3)\n\n== SQL ==\nit is not a data type\n---^^^\n",
       "output": {
-        "failure": "error in JSON serde: expected value at line 1 column 1"
+        "failure": "parse error: error in SQL parser: found is at 10:12 expected ':', or data type"
       }
     },
     {
@@ -703,28 +703,28 @@
       "input": "struct",
       "exception": "\n[PARSE_SYNTAX_ERROR] Syntax error at or near '<'. SQLSTATE: 42601 (line 1, pos 6)\n\n== SQL ==\nstruct\n------^^^\n",
       "output": {
-        "failure": "error in JSON serde: expected value at line 1 column 1"
+        "failure": "parse error: error in SQL parser: found < at 13:14 expected ':', or data type"
       }
     },
     {
       "input": "struct at 14:15 expected ':', or data type"
       }
     },
     {
       "input": "unknown(1,2,3)",
       "exception": "\n[UNSUPPORTED_DATATYPE] Unsupported data type \"UNKNOWN(1,2,3)\". SQLSTATE: 0A000 (line 1, pos 0)\n\n== SQL ==\nunknown(1,2,3)\n^^^\n",
       "output": {
-        "failure": "error in JSON serde: expected value at line 1 column 1"
+        "failure": "parse error: error in SQL parser: found ( at 14:15 expected ':', or data type"
       }
     },
     {
diff --git a/crates/sail-spark-connect/tests/gold_data/table_schema.json b/crates/sail-spark-connect/tests/gold_data/table_schema.json
index 9706c3411..d45d26539 100644
--- a/crates/sail-spark-connect/tests/gold_data/table_schema.json
+++ b/crates/sail-spark-connect/tests/gold_data/table_schema.json
@@ -45,7 +45,7 @@
       "input": "a",
       "exception": "\n[PARSE_SYNTAX_ERROR] Syntax error at or near end of input. SQLSTATE: 42601 (line 1, pos 1)\n\n== SQL ==\na\n-^^^\n",
       "output": {
-        "failure": "error in JSON serde: expected value at line 1 column 1"
+        "failure": "parse error: error in SQL parser: found > at 8:9 expected ':', or data type"
       }
     },
     {
@@ -67,28 +67,28 @@
       "input": "a INT b long",
       "exception": "\n[PARSE_SYNTAX_ERROR] Syntax error at or near 'b'. SQLSTATE: 42601 (line 1, pos 6)\n\n== SQL ==\na INT b long\n------^^^\n",
       "output": {
-        "failure": "error in JSON serde: expected value at line 1 column 1"
+        "failure": "parse error: error in SQL parser: found b at 13:14 expected 'NOT', 'COMMENT', ',', or '>'"
       }
     },
     {
       "input": "a INT, b long, c int,",
       "exception": "\n[PARSE_SYNTAX_ERROR] Syntax error at or near end of input. SQLSTATE: 42601 (line 1, pos 21)\n\n== SQL ==\na INT, b long, c int,\n---------------------^^^\n",
       "output": {
-        "failure": "error in JSON serde: expected value at line 1 column 1"
+        "failure": "parse error: error in SQL parser: found > at 28:29 expected identifier"
       }
     },
     {
       "input": "a INT, b long,,",
       "exception": "\n[PARSE_SYNTAX_ERROR] Syntax error at or near ','. SQLSTATE: 42601 (line 1, pos 14)\n\n== SQL ==\na INT, b long,,\n--------------^^^\n",
       "output": {
-        "failure": "error in JSON serde: expected value at line 1 column 1"
+        "failure": "parse error: error in SQL parser: found , at 21:22 expected identifier"
       }
     },
     {
       "input": "a INT,, b long",
       "exception": "\n[PARSE_SYNTAX_ERROR] Syntax error at or near ','. SQLSTATE: 42601 (line 1, pos 6)\n\n== SQL ==\na INT,, b long\n------^^^\n",
       "output": {
-        "failure": "error in JSON serde: expected value at line 1 column 1"
+        "failure": "parse error: error in SQL parser: found , at 13:14 expected identifier"
       }
     },
     {

From 72a43733fdfdad228615100129fc4694d84efc52 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?David=20L=C3=B3pez?= 
Date: Sat, 1 Aug 2026 07:59:22 +0200
Subject: [PATCH 107/131] fix: validate CSV expression function options like
 Spark (#2255)

---
 crates/sail-function/src/scalar/csv/mod.rs    |    1 +
 .../sail-function/src/scalar/csv/options.rs   |  330 ++++++
 .../src/scalar/csv/schema_of_csv.rs           |   65 +-
 .../src/scalar/csv/spark_from_csv.rs          |   67 +-
 .../src/scalar/csv/spark_to_csv.rs            |  117 +-
 .../function/features/csv/csv_options.feature | 1015 +++++++++++++++++
 .../function/features/csv/from_csv.feature    |    2 +-
 .../features/csv/schema_of_csv.feature        |    2 +-
 .../function/features/csv/to_csv.feature      |   36 +-
 9 files changed, 1454 insertions(+), 181 deletions(-)
 create mode 100644 crates/sail-function/src/scalar/csv/options.rs
 create mode 100644 python/pysail/tests/spark/function/features/csv/csv_options.feature

diff --git a/crates/sail-function/src/scalar/csv/mod.rs b/crates/sail-function/src/scalar/csv/mod.rs
index 08f5bc310..2695b8de9 100644
--- a/crates/sail-function/src/scalar/csv/mod.rs
+++ b/crates/sail-function/src/scalar/csv/mod.rs
@@ -1,3 +1,4 @@
+mod options;
 mod schema_of_csv;
 pub mod spark_from_csv;
 pub mod spark_to_csv;
diff --git a/crates/sail-function/src/scalar/csv/options.rs b/crates/sail-function/src/scalar/csv/options.rs
new file mode 100644
index 000000000..fc0773b6f
--- /dev/null
+++ b/crates/sail-function/src/scalar/csv/options.rs
@@ -0,0 +1,330 @@
+use std::ops::Range;
+
+use datafusion::arrow::array::{Array, MapArray, StringArray};
+use datafusion::error::Result;
+use datafusion_common::exec_err;
+use sail_common::spec::{SAIL_MAP_KEY_FIELD_NAME, SAIL_MAP_VALUE_FIELD_NAME};
+
+/// Which CSV expression function is reading the options.
+///
+/// Spark validates almost every option identically in all three, but not quite: only the writer
+/// rejects an empty `lineSep`, and only `from_csv` rejects the `DROPMALFORMED` parse mode.
+#[derive(Clone, Copy, PartialEq, Eq)]
+pub(super) enum CsvFunction {
+    From,
+    To,
+    SchemaOf,
+}
+
+impl CsvFunction {
+    fn name(self) -> &'static str {
+        match self {
+            Self::From => "from_csv",
+            Self::To => "to_csv",
+            Self::SchemaOf => "schema_of_csv",
+        }
+    }
+}
+
+/// The boolean options that Spark's `CSVOptions` parses with `getBool`, which reports
+/// `