diff --git a/CHANGELOG.md b/CHANGELOG.md index aeb61df4..9dba2c61 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,13 @@ The format is based on Keep a Changelog, and this project adheres to Semantic Ve `(a JOIN b) AS j`, registers only `j`. `ScopeVisitor::reference` reports the items inside the join at `depth + 1`, before the join itself, and subqueries in an unaliased join's conditions now get subquery scopes. +- `build_scope` now registers an aliased table function FROM item, such as + `my_func(x) AS g(y, z)`, as a virtual source, so `g.y` qualifies and its + lineage reaches the function's arguments under a `_N` virtual source name. + Without a schema, an unqualified column is resolved among all FROM items, + the function included, so `SELECT a FROM t, my_func(t.x) AS g` no longer + assumes `t.a`. An aliased `MATCH_RECOGNIZE` is registered as its alias, + which hides its input relation. - `build_scope` now gives a subquery scope to a query in an expression field the scope walk skipped: an aggregate's `FILTER (WHERE ...)`, `ORDER BY` and `LIMIT`, a window frame bound, a `CAST` `FORMAT` or `DEFAULT` expression, diff --git a/crates/polyglot-sql/src/lineage.rs b/crates/polyglot-sql/src/lineage.rs index 3519d446..5f67c0cb 100644 --- a/crates/polyglot-sql/src/lineage.rs +++ b/crates/polyglot-sql/src/lineage.rs @@ -5022,6 +5022,29 @@ FROM UNNEST(GENERATE_DATE_ARRAY('2024-01-01', '2024-12-31', INTERVAL 1 WEEK)) AS ); } + #[test] + fn test_lineage_aliased_table_function_is_a_virtual_source() { + let expr = parse_one( + "SELECT g.y AS out FROM t, my_func(t.x) AS g(y)", + DialectType::PostgreSQL, + ) + .expect("parse"); + + let node = lineage("out", &expr, Some(DialectType::PostgreSQL), false).expect("lineage"); + let child = node.downstream.first().expect("out should have lineage"); + + assert_eq!(child.name, "_0.y"); + assert_eq!(child.source_kind, SourceKind::Virtual); + assert_eq!(child.source_alias.as_deref(), Some("g")); + assert!( + child + .downstream + .iter() + .any(|argument| argument.name == "t.x"), + "the function's argument should be lineage of its output" + ); + } + #[test] fn test_lineage_real_table_named_like_unnest_alias_is_not_virtual() { let expr = diff --git a/crates/polyglot-sql/src/optimizer/qualify_columns.rs b/crates/polyglot-sql/src/optimizer/qualify_columns.rs index 51b6a5c9..0e4bc0c0 100644 --- a/crates/polyglot-sql/src/optimizer/qualify_columns.rs +++ b/crates/polyglot-sql/src/optimizer/qualify_columns.rs @@ -3216,6 +3216,31 @@ mod tests { ); } + #[test] + fn test_qualify_aliased_table_function_columns() { + let mut schema = MappingSchema::new(); + schema + .add_table("t", &[("x".into(), DataType::Text)], None) + .unwrap(); + let sql = "SELECT x, y FROM t, my_func(t.x) AS g(y)"; + let qualified = + qualify_columns(parse(sql), &schema, &QualifyColumnsOptions::new()).unwrap(); + assert_eq!( + gen(&qualified), + "SELECT t.x AS x, g.y AS y FROM t, MY_FUNC(t.x) AS g(y)" + ); + + // Without a column list the function's columns are unknown, but a + // reference qualified with its alias still resolves. + let sql = "SELECT x, g.z FROM t, my_func(t.x) AS g"; + let qualified = + qualify_columns(parse(sql), &schema, &QualifyColumnsOptions::new()).unwrap(); + assert_eq!( + gen(&qualified), + "SELECT t.x AS x, g.z AS z FROM t, MY_FUNC(t.x) AS g" + ); + } + #[test] fn test_get_scope_columns() { let expr = parse("SELECT a, b FROM t WHERE c = 1"); diff --git a/crates/polyglot-sql/src/resolver.rs b/crates/polyglot-sql/src/resolver.rs index 34ee4a45..0b647779 100644 --- a/crates/polyglot-sql/src/resolver.rs +++ b/crates/polyglot-sql/src/resolver.rs @@ -271,6 +271,9 @@ impl<'a> Resolver<'a> { Expression::Alias(alias) if matches!(&alias.this, Expression::Unnest(_)) => { alias_output_columns(alias) } + Expression::Alias(alias) if matches!(&alias.this, Expression::Function(_)) => { + table_function_output_columns(alias) + } Expression::Alias(alias) => { let columns = self.get_source_columns_for_expression(&alias.this)?; apply_alias_columns(columns, &alias.column_aliases) @@ -469,6 +472,9 @@ impl<'a> Resolver<'a> { Expression::Alias(alias) if matches!(&alias.this, Expression::Unnest(_)) => { alias_output_columns(alias) } + Expression::Alias(alias) if matches!(&alias.this, Expression::Function(_)) => { + table_function_output_columns(alias) + } Expression::Alias(alias) => { let columns = self.get_source_output_columns(&alias.this); apply_alias_columns(columns, &alias.column_aliases) @@ -697,15 +703,16 @@ fn unnest_output_columns(unnest: &crate::expressions::UnnestFunc) -> Vec .collect() } +/// A table function's output columns are unknown unless its alias lists them. +fn table_function_output_columns(alias: &crate::expressions::Alias) -> Vec { + identifier_names(&alias.column_aliases) +} + fn alias_output_columns(alias: &crate::expressions::Alias) -> Vec { if alias.column_aliases.is_empty() { vec![alias.alias.name.clone()] } else { - alias - .column_aliases - .iter() - .map(|column| column.name.clone()) - .collect() + identifier_names(&alias.column_aliases) } } @@ -718,10 +725,13 @@ fn lateral_output_columns(lateral: &crate::expressions::Lateral) -> Vec } fn lateral_view_output_columns(lateral_view: &crate::expressions::LateralView) -> Vec { - lateral_view - .column_aliases + identifier_names(&lateral_view.column_aliases) +} + +fn identifier_names(identifiers: &[Identifier]) -> Vec { + identifiers .iter() - .map(|column| column.name.clone()) + .map(|identifier| identifier.name.clone()) .collect() } diff --git a/crates/polyglot-sql/src/scope.rs b/crates/polyglot-sql/src/scope.rs index 192de910..17a479f2 100644 --- a/crates/polyglot-sql/src/scope.rs +++ b/crates/polyglot-sql/src/scope.rs @@ -774,7 +774,10 @@ pub trait ScopeVisitor { /// items inside it: they are still built and reported, with /// `registered_as` set to the name each binds inside the join, but they /// add no entry to [`Scope::sources`], so for them `registered_as` is not - /// a key into it. + /// a key into it. An aliased `MATCH_RECOGNIZE` is registered as its alias + /// and hides its input relation the same way: the input is reported at + /// `depth + 1` before the `MATCH_RECOGNIZE` itself. An unaliased one is + /// reported whole through `skipped`. fn reference( &mut self, _scope: ScopeId, @@ -789,9 +792,10 @@ pub trait ScopeVisitor { /// Called for each subtree of a FROM item that gets no scope and is not /// searched for subqueries, such as table-function and `UNNEST` /// arguments, `VALUES` row values, `LATERAL` calls, `PIVOT` clauses and - /// table sample or time-travel arguments, and the join conditions inside - /// an aliased parenthesized join. An item the builder does not handle at - /// all is reported whole. + /// table sample or time-travel arguments, the clauses of an aliased + /// `MATCH_RECOGNIZE`, and the join conditions behind an alias, such as + /// those inside an aliased parenthesized join. An item the builder does + /// not handle at all is reported whole. /// /// The values of a `VALUES` body that becomes a scope (a derived table, /// CTE body, root or set-operation branch) are reported against that @@ -1071,6 +1075,20 @@ fn add_from_item( builder, ); (name, None) + } else if let Expression::MatchRecognize(match_recognize) = item { + let name = add_match_recognize_to_scope( + item, + ( + match_recognize.this.as_deref(), + match_recognize.alias.as_ref(), + ), + depth, + next_index, + scope, + preceding_sources, + builder, + ); + (name, None) } else { let first_child = scope.derived_table_scopes.len(); let name = add_table_to_scope(item, scope, preceding_sources, builder); @@ -1121,11 +1139,33 @@ fn add_joined_table_to_scope( preceding_sources: &mut HashSet, builder: &mut ScopeBuilder, ) -> Option { - // The relations inside an aliased join still see each other, and the - // items before the join, as LATERAL inputs, so they are registered as - // usual and the enclosing names restored afterwards. - let enclosing = alias.map(|_| (scope.sources.clone(), preceding_sources.clone())); + let Some(alias) = alias else { + add_join_items(joined, depth, next_index, scope, preceding_sources, builder); + return None; + }; + let name = add_behind_alias( + item, + alias, + scope, + preceding_sources, + |scope, preceding_sources| { + add_join_items(joined, depth, next_index, scope, preceding_sources, builder); + skip_join_conditions(joined, scope, builder); + }, + ); + Some(name) +} +/// Registers the relations and lateral views of `joined` one depth below +/// `depth`. +fn add_join_items( + joined: &JoinedTable, + depth: usize, + next_index: &mut usize, + scope: &mut Scope, + preceding_sources: &mut HashSet, + builder: &mut ScopeBuilder, +) { for inner in std::iter::once(&joined.left).chain(joined.joins.iter().map(|join| &join.this)) { add_from_item( inner, @@ -1141,20 +1181,86 @@ fn add_joined_table_to_scope( preceding_sources.insert(name); } } +} - let (Some(alias), Some((sources, preceding))) = (alias, enclosing) else { +/// Registers a `MATCH_RECOGNIZE` item with the given input relation and +/// alias. +/// +/// An alias is registered as a source with no scope of its own, and it hides +/// the input relation, which is still built and reported one depth below the +/// item. Without an alias, the item is reported whole through `skipped`. +/// Returns the alias, if any. +fn add_match_recognize_to_scope( + item: &Expression, + (input, alias): (Option<&Expression>, Option<&Identifier>), + depth: usize, + next_index: &mut usize, + scope: &mut Scope, + preceding_sources: &mut HashSet, + builder: &mut ScopeBuilder, +) -> Option { + let Some(alias) = alias else { + builder.skipped(scope, item); return None; }; - // A subquery in these conditions may refer to the hidden names, which a - // child scope could not resolve, so the conditions are left unscoped. - for_each_join_condition(joined, &mut |condition| builder.skipped(scope, condition)); + let name = add_behind_alias( + item, + alias, + scope, + preceding_sources, + |scope, preceding_sources| { + let Some(input) = input else { + return; + }; + add_from_item( + input, + depth + 1, + next_index, + scope, + preceding_sources, + builder, + ); + if let Some((joined, None)) = parenthesized_join(input) { + skip_join_conditions(joined, scope, builder); + } + }, + ); + builder.skip_children(scope, item, &["this"]); + Some(name) +} + +/// Registers `item` in `scope` as a source named `alias` with no scope of its +/// own, hiding the relations `register` adds behind it. Returns the alias. +/// +/// The relations behind the alias are registered as usual first, so that they +/// see each other, and the items before them, as LATERAL inputs. Then +/// `scope.sources` and `preceding_sources` are restored to what they were +/// before, so those names are hidden again. +fn add_behind_alias( + item: &Expression, + alias: &Identifier, + scope: &mut Scope, + preceding_sources: &mut HashSet, + register: impl FnOnce(&mut Scope, &mut HashSet), +) -> String { + let sources = scope.sources.clone(); + let preceding = preceding_sources.clone(); + register(scope, preceding_sources); scope.sources = sources; *preceding_sources = preceding; scope.add_source_info( alias.name.clone(), SourceInfo::new(item.clone(), false, SourceKind::DerivedTable), ); - Some(alias.name.clone()) + alias.name.clone() +} + +/// Reports the join conditions of `joined` through `skipped`. +/// +/// A subquery in these conditions may refer to names an alias hides, which a +/// child scope could not resolve, so the conditions are left unscoped. +fn skip_join_conditions(joined: &JoinedTable, scope: &Scope, builder: &mut ScopeBuilder) { + for_each_join_condition(joined, &mut |condition| builder.skipped(scope, condition)); } /// Calls `f` on each join condition in `joined`, including those of unaliased @@ -1266,14 +1372,11 @@ fn add_table_to_scope( Some(alias.alias.name.clone()) } Expression::Alias(alias) => match &alias.this { - Expression::Unnest(_) => { + Expression::Unnest(_) | Expression::Function(_) => { builder.skip_children(scope, &alias.this, &[]); scope.add_virtual_source(alias.alias.name.clone(), expr.clone()); Some(alias.alias.name.clone()) } - Expression::Function(_) => { - add_table_to_scope(&alias.this, scope, preceding_sources, builder) - } _ => { builder.skipped(scope, expr); None @@ -2667,6 +2770,120 @@ mod tests { ); } + #[test] + fn test_aliased_table_function_is_a_virtual_source() { + use crate::DialectType::PostgreSQL; + + for sql in [ + "SELECT * FROM t, my_func(t.x) AS g", + "SELECT * FROM t, my_func(t.x) g", + "SELECT * FROM t, my_func(t.x) AS g(y, z)", + ] { + let (scope, recorder) = record(sql, PostgreSQL); + assert_eq!(sorted_keys(&scope.sources), ["g", "t"], "{sql}"); + assert_eq!(scope.sources["g"].kind, SourceKind::Virtual, "{sql}"); + assert_eq!( + from_items(&recorder), + [(0, 0, Some("t")), (1, 0, Some("g"))], + "{sql}" + ); + assert_eq!(recorder.skipped.len(), 1, "{sql}"); + } + } + + #[test] + fn test_aliased_match_recognize_hides_its_input() { + use crate::DialectType::{Oracle, Snowflake}; + + let sql = "SELECT * FROM t MATCH_RECOGNIZE (PARTITION BY k ORDER BY tstamp \ + MEASURES FIRST(A.tstamp) AS start_tstamp PATTERN (A B*) \ + DEFINE B AS B.v > PREV(B.v)) AS m"; + for dialect in [Snowflake, Oracle] { + let (scope, recorder) = record(sql, dialect); + assert_eq!(sorted_keys(&scope.sources), ["m"], "{sql}"); + let source = &scope.sources["m"]; + assert_eq!( + (source.kind, source.is_scope), + (SourceKind::DerivedTable, false), + "{sql}" + ); + assert_eq!( + from_items(&recorder), + [(1, 1, Some("t")), (0, 0, Some("m"))], + "{sql}" + ); + // PARTITION BY, ORDER BY, MEASURES and DEFINE, but not the input. + assert_eq!(recorder.skipped.len(), 4, "{sql}"); + assert!( + !recorder + .skipped + .iter() + .any(|skipped| matches!(skipped, Expression::Table(_))), + "{sql}" + ); + } + + // The input hides an earlier source of its name only inside the item. + let (scope, recorder) = record( + "SELECT * FROM s.x, x MATCH_RECOGNIZE (ORDER BY v PATTERN (A) \ + DEFINE A AS TRUE) AS m", + Snowflake, + ); + assert_eq!(sorted_keys(&scope.sources), ["m", "x"]); + assert!(matches!( + &scope.sources["x"].expression, + Expression::Table(table) if table.schema.is_some() + )); + assert_eq!( + from_items(&recorder), + [(0, 0, Some("x")), (2, 1, Some("x")), (1, 0, Some("m"))] + ); + + // A parenthesized join input: its condition may use the hidden names, + // so it is reported as skipped rather than scoped. + let (scope, recorder) = record( + "SELECT * FROM (a JOIN b ON a.k IN (SELECT k FROM s)) \ + MATCH_RECOGNIZE (ORDER BY v PATTERN (A) DEFINE A AS TRUE) AS m", + Snowflake, + ); + assert_eq!(sorted_keys(&scope.sources), ["m"]); + assert!(scope.subquery_scopes.is_empty()); + assert_eq!( + from_items(&recorder), + [ + (2, 2, Some("a")), + (3, 2, Some("b")), + (1, 1, None), + (0, 0, Some("m")), + ] + ); + assert!(recorder + .skipped + .iter() + .any(|skipped| matches!(skipped, Expression::In(_)))); + + // An unaliased MATCH_RECOGNIZE is still reported whole. + let (scope, recorder) = record( + "SELECT * FROM t MATCH_RECOGNIZE (ORDER BY v PATTERN (A) DEFINE A AS TRUE)", + Snowflake, + ); + assert!(scope.sources.is_empty()); + assert_eq!(from_items(&recorder), [(0, 0, None)]); + assert_eq!(recorder.skipped.len(), 1); + + // The input's name stays hidden from a LATERAL item after it. + let (scope, _) = record( + "SELECT * FROM t MATCH_RECOGNIZE (ORDER BY v PATTERN (A) DEFINE A AS TRUE) AS m, \ + LATERAL (SELECT m.v) AS l", + Snowflake, + ); + assert_eq!(sorted_keys(&scope.sources), ["l", "m"]); + assert_eq!( + sorted_keys(&scope.derived_table_scopes[0].lateral_sources), + ["m"] + ); + } + #[test] fn test_parenthesized_join_name_collision_keeps_last() { use crate::DialectType::Generic;