From 09e7fb6637073e9cd192b603ce15c9d425f3f8b6 Mon Sep 17 00:00:00 2001 From: Advit Arora Date: Tue, 25 Aug 2026 11:19:09 +0530 Subject: [PATCH 1/2] test(spark): fill the find_in_set slt from Spark 4.2.0 semantics --- .../test_files/spark/string/find_in_set.slt | 216 +++++++++++++++++- 1 file changed, 212 insertions(+), 4 deletions(-) diff --git a/datafusion/sqllogictest/test_files/spark/string/find_in_set.slt b/datafusion/sqllogictest/test_files/spark/string/find_in_set.slt index 690d03ffa475f..f1982ab7e9a0b 100644 --- a/datafusion/sqllogictest/test_files/spark/string/find_in_set.slt +++ b/datafusion/sqllogictest/test_files/spark/string/find_in_set.slt @@ -21,7 +21,215 @@ # For more information, please see: # https://github.com/apache/datafusion/issues/15914 -## Original Query: SELECT find_in_set('ab','abc,b,ab,c,def'); -## PySpark 3.5.5 Result: {'find_in_set(ab, abc,b,ab,c,def)': 3, 'typeof(find_in_set(ab, abc,b,ab,c,def))': 'int', 'typeof(ab)': 'string', 'typeof(abc,b,ab,c,def)': 'string'} -#query -#SELECT find_in_set('ab'::string, 'abc,b,ab,c,def'::string); +query I +SELECT find_in_set('ab'::string, 'abc,b,ab,c,def'::string); +---- +3 + +query I +SELECT find_in_set('abc'::string, 'abc,b,ab,c,def'::string); +---- +1 + +query I +SELECT find_in_set('def'::string, 'abc,b,ab,c,def'::string); +---- +5 + +query I +SELECT find_in_set('xyz'::string, 'abc,b,ab,c,def'::string); +---- +0 + +query I +SELECT find_in_set('a'::string, 'abc,b,ab,c,def'::string); +---- +0 + +query I +SELECT find_in_set('abcd'::string, 'abc,b,ab,c,def'::string); +---- +0 + +query I +SELECT find_in_set('Ab'::string, 'abc,b,ab,c,def'::string); +---- +0 + +query I +SELECT find_in_set('d,ef'::string, 'abc,b,ab,c,def'::string); +---- +0 + +query I +SELECT find_in_set('ab,'::string, 'abc,b,ab,c,def'::string); +---- +0 + +# by definition a needle containing a comma is never found +query I +SELECT find_in_set(','::string, 'a,,b'::string); +---- +0 + +# a list with no comma is one element, so an empty list has one empty element +query I +SELECT find_in_set(''::string, ''::string); +---- +1 + +query I +SELECT find_in_set('a'::string, ''::string); +---- +0 + +# one comma delimits two empty elements, and the first match wins +query I +SELECT find_in_set(''::string, ','::string); +---- +1 + +query I +SELECT find_in_set(''::string, 'a,,b'::string); +---- +2 + +query I +SELECT find_in_set(''::string, ',abc,b,ab,c,def'::string); +---- +1 + +# a trailing comma adds a final empty element +query I +SELECT find_in_set(''::string, 'abc,b,ab,c,def,'::string); +---- +6 + +query I +SELECT find_in_set(''::string, 'abc,b,ab,c,def'::string); +---- +0 + +query I +SELECT find_in_set(''::string, 'abc'::string); +---- +0 + +query I +SELECT find_in_set('ab'::string, ',,,ab,abc,b,ab,c,def'::string); +---- +4 + +query I +SELECT find_in_set('ab'::string, 'ab,abc,b,ab,c,def'::string); +---- +1 + +query I +SELECT find_in_set('b'::string, 'a, b'::string); +---- +0 + +query I +SELECT find_in_set(' b'::string, 'a, b'::string); +---- +2 + +query I +SELECT find_in_set('大'::string, 'test,大千,世,界X,大,千,世界'::string); +---- +5 + +query I +SELECT find_in_set('界x'::string, 'test,大千,世,界X,大,千,世界'::string); +---- +0 + +query I +SELECT find_in_set('𝔸'::string, 'a,𐐅,𝔸'::string); +---- +3 + +query I +SELECT find_in_set('🔥'::string, 'a,Д,🔥'::string); +---- +3 + +query I +SELECT find_in_set(NULL::string, 'abc,b,ab,c,def'::string); +---- +NULL + +query I +SELECT find_in_set('ab'::string, NULL::string); +---- +NULL + +query I +SELECT find_in_set(NULL::string, NULL::string); +---- +NULL + +query I +SELECT find_in_set(a, 'abc,b,ab,c,def'::string) FROM (VALUES ('abc'::string), ('def'::string), ('zzz'::string), (''::string), ('d,ef'::string), (NULL::string)) AS t(a); +---- +1 +5 +0 +0 +0 +NULL + +query I +SELECT find_in_set('ab'::string, a) FROM (VALUES ('abc,b,ab,c,def'::string), ('ab'::string), (''::string), (',ab'::string), (NULL::string)) AS t(a); +---- +3 +1 +0 +2 +NULL + +query I +SELECT find_in_set(a, b) FROM (VALUES ('ab'::string, 'abc,b,ab,c,def'::string), (''::string, 'a,,b'::string), ('c'::string, 'a,b'::string), ('a,b'::string, 'a,b'::string), (NULL::string, 'a,b'::string), ('a'::string, NULL::string)) AS t(a, b); +---- +3 +2 +0 +0 +NULL +NULL + +# 20 elements is past the size at which the set is looked up rather than scanned, and a5 repeats at 19 +query I +SELECT find_in_set(a, 'a0,a1,a2,a3,a4,a5,a6,a7,a8,a9,a10,a11,a12,a13,a14,a15,a16,a17,a5,'::string) FROM (VALUES ('a0'::string), ('a17'::string), ('a5'::string), (''::string), ('zz'::string), (NULL::string)) AS t(a); +---- +1 +18 +6 +20 +0 +NULL + +query I +SELECT find_in_set(arrow_cast('ab', 'Utf8View'), arrow_cast('abc,b,ab,c,def', 'Utf8View')); +---- +3 + +query I +SELECT find_in_set(arrow_cast(a, 'Utf8View'), arrow_cast('abc,b,ab,c,def', 'Utf8View')) FROM (VALUES ('ab'::string), (''::string), (NULL::string)) AS t(a); +---- +3 +0 +NULL + +query I +SELECT find_in_set(arrow_cast('ab', 'LargeUtf8'), arrow_cast('abc,b,ab,c,def', 'LargeUtf8')); +---- +3 + +query I +SELECT find_in_set(arrow_cast(a, 'LargeUtf8'), arrow_cast('abc,b,ab,c,def', 'LargeUtf8')) FROM (VALUES ('ab'::string), (''::string), (NULL::string)) AS t(a); +---- +3 +0 +NULL From 86f4a683aeece02448966bceb0eec00a439649c0 Mon Sep 17 00:00:00 2001 From: Advit Arora Date: Tue, 25 Aug 2026 11:19:09 +0530 Subject: [PATCH 2/2] fix(functions): build find_in_set scalar result at the promised int width --- .../functions/src/unicode/find_in_set.rs | 24 ++++++++++++------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/datafusion/functions/src/unicode/find_in_set.rs b/datafusion/functions/src/unicode/find_in_set.rs index fa23532406ce1..7746e4a34a7d6 100644 --- a/datafusion/functions/src/unicode/find_in_set.rs +++ b/datafusion/functions/src/unicode/find_in_set.rs @@ -110,18 +110,26 @@ impl ScalarUDFImpl for FindInSetFunc { | ScalarValue::LargeUtf8(str_list), ), ) => { - let res = match (string, str_list) { - (Some(string), Some(str_list)) => { - let position = str_list + let position = match (string, str_list) { + (Some(string), Some(str_list)) => Some( + str_list .split(',') .position(|s| s == string) - .map_or(0, |idx| idx + 1); - - Some(position as i32) - } + .map_or(0, |idx| idx + 1), + ), _ => None, }; - Ok(ColumnarValue::Scalar(ScalarValue::from(res))) + // Int64 when the argument is LargeUtf8, matching what `return_type` promised + let res = match return_field.data_type() { + DataType::Int32 => ScalarValue::Int32(position.map(|p| p as i32)), + DataType::Int64 => ScalarValue::Int64(position.map(|p| p as i64)), + other => { + return exec_err!( + "Unsupported return type {other:?} for function find_in_set" + ); + } + }; + Ok(ColumnarValue::Scalar(res)) } // `string` is an array, `str_list` is scalar