From f9ad1c350c4d8c86cc873c13bcb8d2baa2678841 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20Heres?= Date: Sun, 30 Aug 2026 08:49:37 +0200 Subject: [PATCH] fix: Unnest must not keep its input's uniqueness `Unnest::try_new` copied the input's functional dependencies unchanged, with the comment "We can use the existing functional dependencies". That is not true for a list unnest, which turns one input row into several. A determinant that occurred once in the input can occur many times in the output. Optimizer rules that read those dependencies then produce wrong results. With a declared key: CREATE TABLE t_list (k INT, vals INT[], PRIMARY KEY (k)) AS VALUES (1, [10, 20, 30]), (2, [40]); CREATE TABLE t_join (k INT) AS VALUES (1), (2); SELECT u.k, u.v FROM (SELECT k, unnest(vals) AS v FROM t_list) u JOIN t_join j ON u.k = j.k; returns 2 rows instead of 4, because `eliminate_join` sees `u` as unique on `k` and rewrites the inner join into a semi join, which drops the repeated rows. Without the PRIMARY KEY the same query returns 4. The dependency still holds in the weaker sense: all rows produced from one input row share the determinant, so it determines the same columns. Downgrade it to `Dependency::Multi` rather than dropping it, which keeps it useful and stops it being read as a uniqueness guarantee. Unnesting a struct produces one row per input row, so those dependencies are left as they are. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01SgqwvctZdvR1ZCz2hbkEJC --- datafusion/expr/src/logical_plan/plan.rs | 11 ++++- .../test_files/functional_dependencies.slt | 46 +++++++++++++++++++ 2 files changed, 55 insertions(+), 2 deletions(-) diff --git a/datafusion/expr/src/logical_plan/plan.rs b/datafusion/expr/src/logical_plan/plan.rs index 1a8cd81aa74b..80a00dc8b6a2 100644 --- a/datafusion/expr/src/logical_plan/plan.rs +++ b/datafusion/expr/src/logical_plan/plan.rs @@ -4896,8 +4896,15 @@ impl Unnest { let metadata = input_schema.metadata().clone(); let df_schema = DFSchema::new_with_metadata(fields, metadata)?; - // We can use the existing functional dependencies: - let deps = input_schema.functional_dependencies().clone(); + // Unnesting a list turns one input row into several, so a determinant + // that occurred once in the input can now occur many times. It still + // determines the same columns, so downgrade the dependency instead of + // dropping it. Unnesting a struct keeps one row per input row, and so + // keeps the dependencies as they are. + let mut deps = input_schema.functional_dependencies().clone(); + if !list_columns.is_empty() { + deps = deps.with_dependency(Dependency::Multi); + } let schema = Arc::new(df_schema.with_functional_dependencies(deps)?); Ok(Unnest { diff --git a/datafusion/sqllogictest/test_files/functional_dependencies.slt b/datafusion/sqllogictest/test_files/functional_dependencies.slt index c49004190dc6..566dbe923f2f 100644 --- a/datafusion/sqllogictest/test_files/functional_dependencies.slt +++ b/datafusion/sqllogictest/test_files/functional_dependencies.slt @@ -296,6 +296,52 @@ drop table t_null; statement ok drop table t_probe; +########## +## Unnest +########## + +# Unnesting a list turns one row into several, so the key of the input no +# longer identifies a row of the output. Trusting it here made the join below +# a semi join, which dropped the repeated rows. +statement ok +CREATE TABLE t_list (k INT, vals INT[], PRIMARY KEY (k)) AS VALUES (1, [10, 20, 30]), (2, [40]); + +statement ok +CREATE TABLE t_join (k INT) AS VALUES (1), (2); + +query II +SELECT u.k, u.v FROM (SELECT k, unnest(vals) AS v FROM t_list) u +JOIN t_join j ON u.k = j.k +ORDER BY u.k, u.v; +---- +1 10 +1 20 +1 30 +2 40 + +# Unnesting a struct keeps one row per input row, so the key still holds. +statement ok +CREATE TABLE t_struct (k INT, s STRUCT, PRIMARY KEY (k)) AS VALUES (1, {'a': 1, 'b': 2}), (2, {'a': 3, 'b': 4}); + +query TT +EXPLAIN SELECT DISTINCT k FROM (SELECT k, unnest(s) FROM t_struct) t; +---- +logical_plan +01)SubqueryAlias: t +02)--Projection: t_struct.k +03)----Unnest: lists[] structs[__unnest_placeholder(t_struct.s)] +04)------Projection: t_struct.k, t_struct.s AS __unnest_placeholder(t_struct.s) +05)--------TableScan: t_struct projection=[k, s] + +statement ok +drop table t_list; + +statement ok +drop table t_join; + +statement ok +drop table t_struct; + ########## ## Cleanup ##########