From 6bcea5e501f83672c6c72d1fba79548a4dac63ab Mon Sep 17 00:00:00 2001 From: peterxcli Date: Fri, 4 Sep 2026 21:10:06 +0800 Subject: [PATCH 1/5] fix: align string to timestamp parsing with Spark's segment rules Spark's SparkDateTimeUtils.parseTimestampString validates each segment with isValidDigits: month, day, hour, minute and second take 1-2 digits, the fraction after '.' may be empty, a timestamp year takes at most 6 digits (only stringToDate allows 7), and a zone id is only captured when the scanner is inside the seconds or fraction segment. Comet's regex table required exactly 2 digits and a non-empty fraction, allowed 7-digit years, and stripped a zone suffix from any shape, so '2020-1-1', '2020-01-01 12:34:5' and '2020-01-01 12:34:56.' returned NULL (or raised under ANSI) while '2020-10-01Z' and '0002020-01-01 00:00:00' were accepted. Relax the segment quantifiers, cap the year at 6 digits for timestamp shapes, allow an empty fraction, and only honour a stripped suffix when the remainder ends in a seconds or fraction segment, for both TIMESTAMP and TIMESTAMP_NTZ. Previously accepted inputs keep their exact values; CAST(... AS DATE) keeps its 7-digit years because date_parser is a separate port of stringToDate. Closes #5674 Co-Authored-By: Claude Fable 5.1 --- .../spark-expr/src/conversion_funcs/string.rs | 315 ++++++++++++++++-- .../apache/comet/CometNativeCastSuite.scala | 63 ++++ 2 files changed, 346 insertions(+), 32 deletions(-) diff --git a/native/spark-expr/src/conversion_funcs/string.rs b/native/spark-expr/src/conversion_funcs/string.rs index 9062c442081..39a9780324e 100644 --- a/native/spark-expr/src/conversion_funcs/string.rs +++ b/native/spark-expr/src/conversion_funcs/string.rs @@ -1470,7 +1470,12 @@ fn timestamp_parser( if !has_direct_match { if let Some((stripped, suffix_tz)) = extract_offset_suffix(value) { - return timestamp_parser_with_tz(stripped, eval_mode, &suffix_tz); + // A zone suffix is only meaningful after the seconds segment. Otherwise fall + // through with the unstripped value, which no base pattern matches, so it is + // reported as malformed (null, or CAST_INVALID_INPUT under ANSI) like Spark does. + if ends_with_seconds_segment(stripped) { + return timestamp_parser_with_tz(stripped, eval_mode, &suffix_tz); + } } } @@ -1636,34 +1641,52 @@ fn extract_offset_suffix(value: &str) -> Option<(&str, Tz)> { type TimestampParsePattern = (&'static Regex, fn(&str, &T) -> SparkResult>); -// RE_YEAR allows only 4-6 digits (not 7) because a bare 7-digit string like "0119704" -// is ambiguous and Spark rejects it. The other patterns (RE_MONTH, RE_DAY, etc.) keep -// \d{4,7} because the `-` separator disambiguates the year portion, so "0002020-01-01" -// is validly year 2020 with leading zeros. date_parser's is_valid_digits also allows up -// to 7 year digits for the same reason. +// These shapes transcribe the per-segment digit rules of Spark's +// `SparkDateTimeUtils.parseTimestampString` (`isValidDigits`): the year takes 4-6 digits +// (`maxDigitsYear = 6`, so "0002020-01-01" is malformed for a timestamp even though +// `stringToDate`, ported by `date_parser`, allows 7), month/day/hour/minute/second take 1-2 +// digits each, and the fraction takes any number of digits including none ("12:34:56." is +// valid), of which only the first six are kept. static RE_YEAR: LazyLock = LazyLock::new(|| Regex::new(r"^-?\d{4,6}$").unwrap()); -static RE_MONTH: LazyLock = LazyLock::new(|| Regex::new(r"^-?\d{4,7}-\d{2}$").unwrap()); -static RE_DAY: LazyLock = LazyLock::new(|| Regex::new(r"^-?\d{4,7}-\d{2}-\d{2}$").unwrap()); +static RE_MONTH: LazyLock = LazyLock::new(|| Regex::new(r"^-?\d{4,6}-\d{1,2}$").unwrap()); +static RE_DAY: LazyLock = + LazyLock::new(|| Regex::new(r"^-?\d{4,6}-\d{1,2}-\d{1,2}$").unwrap()); static RE_HOUR: LazyLock = - LazyLock::new(|| Regex::new(r"^-?\d{4,7}-\d{2}-\d{2}[T ]\d{1,2}$").unwrap()); + LazyLock::new(|| Regex::new(r"^-?\d{4,6}-\d{1,2}-\d{1,2}[T ]\d{1,2}$").unwrap()); static RE_MINUTE: LazyLock = - LazyLock::new(|| Regex::new(r"^-?\d{4,7}-\d{2}-\d{2}[T ]\d{2}:\d{2}$").unwrap()); -static RE_SECOND: LazyLock = - LazyLock::new(|| Regex::new(r"^-?\d{4,7}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}$").unwrap()); -static RE_MICROSECOND: LazyLock = - LazyLock::new(|| Regex::new(r"^-?\d{4,7}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}\.\d+$").unwrap()); + LazyLock::new(|| Regex::new(r"^-?\d{4,6}-\d{1,2}-\d{1,2}[T ]\d{1,2}:\d{1,2}$").unwrap()); +static RE_SECOND: LazyLock = LazyLock::new(|| { + Regex::new(r"^-?\d{4,6}-\d{1,2}-\d{1,2}[T ]\d{1,2}:\d{1,2}:\d{1,2}$").unwrap() +}); +static RE_MICROSECOND: LazyLock = LazyLock::new(|| { + Regex::new(r"^-?\d{4,6}-\d{1,2}-\d{1,2}[T ]\d{1,2}:\d{1,2}:\d{1,2}\.\d*$").unwrap() +}); static RE_TIME_ONLY_H: LazyLock = LazyLock::new(|| Regex::new(r"^T\d{1,2}$").unwrap()); static RE_TIME_ONLY_HM: LazyLock = LazyLock::new(|| Regex::new(r"^T\d{1,2}:\d{1,2}$").unwrap()); static RE_TIME_ONLY_HMS: LazyLock = LazyLock::new(|| Regex::new(r"^T\d{1,2}:\d{1,2}:\d{1,2}$").unwrap()); static RE_TIME_ONLY_HMSU: LazyLock = - LazyLock::new(|| Regex::new(r"^T\d{1,2}:\d{1,2}:\d{1,2}\.\d+$").unwrap()); + LazyLock::new(|| Regex::new(r"^T\d{1,2}:\d{1,2}:\d{1,2}\.\d*$").unwrap()); static RE_BARE_HM: LazyLock = LazyLock::new(|| Regex::new(r"^\d{1,2}:\d{1,2}$").unwrap()); static RE_BARE_HMS: LazyLock = LazyLock::new(|| Regex::new(r"^\d{1,2}:\d{1,2}:\d{1,2}$").unwrap()); static RE_BARE_HMSU: LazyLock = - LazyLock::new(|| Regex::new(r"^\d{1,2}:\d{1,2}:\d{1,2}\.\d+$").unwrap()); + LazyLock::new(|| Regex::new(r"^\d{1,2}:\d{1,2}:\d{1,2}\.\d*$").unwrap()); + +/// Whether `value` (a datetime with any zone suffix already stripped) ends in a seconds or +/// fraction segment. Spark's `parseTimestampString` only captures a zone id when its byte +/// scanner hits a non-digit while inside those two segments, so a suffix such as `Z`, `+05:30` +/// or ` UTC` is legal after `hh:mm:ss` or `hh:mm:ss.f*` but makes a date-only, hour-only or +/// hour:minute value malformed ("2020-10-01Z" and "2020-01-01T12:34Z" are both null). +fn ends_with_seconds_segment(value: &str) -> bool { + RE_SECOND.is_match(value) + || RE_MICROSECOND.is_match(value) + || RE_TIME_ONLY_HMS.is_match(value) + || RE_TIME_ONLY_HMSU.is_match(value) + || RE_BARE_HMS.is_match(value) + || RE_BARE_HMSU.is_match(value) +} fn timestamp_parser_with_tz( value: &str, @@ -1673,7 +1696,7 @@ fn timestamp_parser_with_tz( // Both T-separator and space-separator date-time forms are supported. // Negative years are handled by get_timestamp_values detecting a leading '-'. let patterns: &[TimestampParsePattern] = &[ - // Year only: 4-7 digits, optionally negative + // Year only: 4-6 digits, optionally negative ( &RE_YEAR, parse_str_to_year_timestamp as fn(&str, &T) -> SparkResult>, @@ -1787,23 +1810,26 @@ fn timestamp_ntz_parser( || RE_SECOND.is_match(value) || RE_MICROSECOND.is_match(value); - // If no direct match, try stripping a timezone suffix + // If no direct match, try stripping a timezone suffix. Spark only recognises a zone after + // the seconds segment; a suffix anywhere else leaves the unstripped value, which no base + // pattern matches, so the inner parser reports it as malformed. let value_to_parse = if !has_direct_match { - if let Some((stripped, _tz)) = extract_offset_suffix(value) { - if !allow_time_zone { - return if eval_mode == EvalMode::Ansi { - Err(SparkError::InvalidInputInCastToDatetime { - value: value.to_string(), - from_type: "STRING".to_string(), - to_type: "TIMESTAMP_NTZ".to_string(), - }) - } else { - Ok(None) - }; + match extract_offset_suffix(value) { + Some((stripped, _tz)) if ends_with_seconds_segment(stripped.trim_end()) => { + if !allow_time_zone { + return if eval_mode == EvalMode::Ansi { + Err(SparkError::InvalidInputInCastToDatetime { + value: value.to_string(), + from_type: "STRING".to_string(), + to_type: "TIMESTAMP_NTZ".to_string(), + }) + } else { + Ok(None) + }; + } + stripped.trim_end() } - stripped.trim_end() - } else { - value + _ => value, } } else { value @@ -2941,6 +2967,231 @@ mod tests { ); } + // 2020-01-01T00:00:00Z, 2020-01-01T12:34:56Z and the same wall clock at +05:30, in micros. + const JAN1_2020: i64 = 1577836800000000; + const JAN1_2020_123456: i64 = 1577882096000000; + const JAN1_2020_123456_PLUS_0530: i64 = 1577862296000000; + + /// Inputs Spark's `parseTimestampString` accepts that the fixed 2-digit shapes rejected: + /// 1-2 digit month/day/hour/minute/second, an empty fraction (also before a zone), and + /// 6-digit years (issue #5674). Values are UTC micros, identical for TIMESTAMP_NTZ. + const SPARK_SEGMENT_RULE_VALID: &[(&str, i64)] = &[ + ("2020-1", JAN1_2020), + ("2020-1-1", JAN1_2020), + ("2020-1-1T1", JAN1_2020 + 3600 * 1_000_000), + ("2020-1-1 1:2", JAN1_2020 + 3720 * 1_000_000), + ("2020-01-01 12:34:5", JAN1_2020 + 45245 * 1_000_000), + ("2020-1-1T1:2:3.4", JAN1_2020 + 3723 * 1_000_000 + 400_000), + ("2020-01-01 12:34:56.", JAN1_2020_123456), + ("002020-01-01 00:00:00", JAN1_2020), + ]; + + /// Inputs Spark rejects: a zone suffix anywhere but after the seconds segment, more than + /// six year digits, and more than two digits in any other segment. + const SPARK_SEGMENT_RULE_INVALID: &[&str] = &[ + "2020Z", + "2020-10-01Z", + "2020-01-01+05:30", + "2020-01-01-08:00", + "2020-10-01 UTC", + "2020-01-01T12Z", + "2020-01-01 12 UTC", + "2020-01-01T12:34Z", + "2020-01-01 12:34 UTC", + "2020-01-01T12:34:Z", + "0002020-01-01", + "0002020-01-01 00:00:00", + "-0002020-01-01", + "2020-001-01", + "2020-01-001", + "2020-01-01T123", + "2020-01-01T12:345", + "2020-01-01T12:34:567", + ]; + + #[test] + #[cfg_attr(miri, ignore)] + fn timestamp_parser_spark_segment_rules_test() { + let tz = &Tz::from_str("UTC").unwrap(); + for &(input, expected) in SPARK_SEGMENT_RULE_VALID { + for eval_mode in [EvalMode::Legacy, EvalMode::Try, EvalMode::Ansi] { + assert_eq!( + timestamp_parser(input, eval_mode, tz, true).unwrap(), + Some(expected), + "{input:?} in {eval_mode:?}" + ); + } + } + // An empty fraction may still be followed by a zone. + assert_eq!( + timestamp_parser("2020-01-01 12:34:56.Z", EvalMode::Legacy, tz, true).unwrap(), + Some(JAN1_2020_123456) + ); + assert_eq!( + timestamp_parser("2020-01-01 12:34:56.+05:30", EvalMode::Legacy, tz, true).unwrap(), + Some(JAN1_2020_123456_PLUS_0530) + ); + + // Time-only shapes may carry a zone after their seconds segment but not before it. + for input in ["T12:34:56Z", "12:34:56+05:30", "T1:2:3.Z"] { + assert!( + timestamp_parser(input, EvalMode::Ansi, tz, true) + .unwrap() + .is_some(), + "{input:?}" + ); + } + for input in + SPARK_SEGMENT_RULE_INVALID + .iter() + .copied() + .chain(["T12Z", "12:34Z", "T12:34 UTC"]) + { + for eval_mode in [EvalMode::Legacy, EvalMode::Try] { + assert_eq!( + timestamp_parser(input, eval_mode, tz, true).unwrap(), + None, + "{input:?} in {eval_mode:?}" + ); + } + assert!( + timestamp_parser(input, EvalMode::Ansi, tz, true).is_err(), + "{input:?} in Ansi" + ); + } + + // Shapes that were already accepted keep their exact values. + let la_offset = 8 * 3600 * 1_000_000; // America/Los_Angeles is UTC-8 in January + for (input, expected) in [ + ("2020-01-01", JAN1_2020), + ("2020-01-01 12:34:56", JAN1_2020_123456), + ("2020-01-01T12:34:56.123456", JAN1_2020_123456 + 123456), + ("2020-01-01T12:34:56Z", JAN1_2020_123456), + ("2020-01-01T12:34:56.123Z", JAN1_2020_123456 + 123000), + ("2020-01-01T12:34:56+05:30", JAN1_2020_123456_PLUS_0530), + ("2020-01-01T12:34:56 UTC", JAN1_2020_123456), + ("2020-01-01T12:34:56 UTC+5:30", JAN1_2020_123456_PLUS_0530), + ( + "2020-01-01T12:34:56 America/Los_Angeles", + JAN1_2020_123456 + la_offset, + ), + ("-0001-01-01T12:34:56", -62198709904000000), + ] { + for eval_mode in [EvalMode::Legacy, EvalMode::Try, EvalMode::Ansi] { + assert_eq!( + timestamp_parser(input, eval_mode, tz, true).unwrap(), + Some(expected), + "{input:?} in {eval_mode:?}" + ); + } + } + + // `date_parser` ports `stringToDate`, whose `maxDigitsYear` is 7, so a date cast keeps + // accepting the 7-digit year that a timestamp cast rejects. + for eval_mode in [EvalMode::Legacy, EvalMode::Try, EvalMode::Ansi] { + assert_eq!( + date_parser("0002020-01-01", eval_mode).unwrap(), + Some(18262) + ); + } + } + + #[test] + #[cfg_attr(miri, ignore)] + fn timestamp_ntz_parser_spark_segment_rules_test() { + for allow_time_zone in [true, false] { + for &(input, expected) in SPARK_SEGMENT_RULE_VALID { + for eval_mode in [EvalMode::Legacy, EvalMode::Try, EvalMode::Ansi] { + assert_eq!( + timestamp_ntz_parser(input, eval_mode, allow_time_zone, false).unwrap(), + Some(expected), + "{input:?} in {eval_mode:?}, allow_time_zone={allow_time_zone}" + ); + } + } + for &input in SPARK_SEGMENT_RULE_INVALID { + for eval_mode in [EvalMode::Legacy, EvalMode::Try] { + assert_eq!( + timestamp_ntz_parser(input, eval_mode, allow_time_zone, false).unwrap(), + None, + "{input:?} in {eval_mode:?}, allow_time_zone={allow_time_zone}" + ); + } + assert!( + timestamp_ntz_parser(input, EvalMode::Ansi, allow_time_zone, false).is_err(), + "{input:?} in Ansi, allow_time_zone={allow_time_zone}" + ); + } + } + // A zone after an empty fraction is discarded when allowed and rejected otherwise. + assert_eq!( + timestamp_ntz_parser("2020-01-01 12:34:56.Z", EvalMode::Legacy, true, false).unwrap(), + Some(JAN1_2020_123456) + ); + assert_eq!( + timestamp_ntz_parser("2020-01-01 12:34:56.Z", EvalMode::Legacy, false, false).unwrap(), + None + ); + assert!( + timestamp_ntz_parser("2020-01-01 12:34:56.Z", EvalMode::Ansi, false, false).is_err() + ); + } + + #[test] + #[cfg_attr(miri, ignore)] + fn test_cast_string_to_timestamp_spark_segment_rules_array() { + // The reproducer from issue #5674, through the batch entry points. + let inputs = vec![ + Some("2020-1-1"), + Some("2020-01-01 12:34:5"), + Some("2020-01-01 12:34:56."), + Some("2020-10-01Z"), + Some("0002020-01-01 00:00:00"), + ]; + let expected = [ + Some(JAN1_2020), + Some(JAN1_2020 + 45245 * 1_000_000), + Some(JAN1_2020_123456), + None, + None, + ]; + let array: ArrayRef = Arc::new(StringArray::from(inputs)); + let to_type = DataType::Timestamp(TimeUnit::Microsecond, Some("UTC".into())); + + let tz_result = + cast_string_to_timestamp(&array, &to_type, EvalMode::Legacy, "UTC", true).unwrap(); + let ntz_result = + cast_string_to_timestamp_ntz(&array, EvalMode::Legacy, true, false).unwrap(); + for result in [&tz_result, &ntz_result] { + let result = result + .as_any() + .downcast_ref::>() + .unwrap(); + let actual: Vec> = result.iter().collect(); + assert_eq!(actual, expected); + } + + // Under ANSI the first malformed row fails the batch and names the raw input. + let tz_err = + cast_string_to_timestamp(&array, &to_type, EvalMode::Ansi, "UTC", true).unwrap_err(); + let ntz_err = + cast_string_to_timestamp_ntz(&array, EvalMode::Ansi, true, false).unwrap_err(); + for (err, expected_type) in [(tz_err, "TIMESTAMP"), (ntz_err, "TIMESTAMP_NTZ")] { + match err { + SparkError::InvalidInputInCastToDatetime { + value, + from_type, + to_type, + } => { + assert_eq!(value, "2020-10-01Z"); + assert_eq!(from_type, "STRING"); + assert_eq!(to_type, expected_type); + } + other => panic!("Expected InvalidInputInCastToDatetime, got {other:?}"), + } + } + } + /// Asserts every date parses to null in legacy and try mode. When `expect_ansi_error` is set, /// ANSI mode must raise CAST_INVALID_INPUT; otherwise ANSI mode must also return null. fn assert_dates(dates: &[&str], expect_ansi_error: bool) { diff --git a/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala b/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala index 9bfa8f63774..8c607235080 100644 --- a/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala @@ -1398,6 +1398,69 @@ class CometNativeCastSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } + // Spark's SparkDateTimeUtils.parseTimestampString validates each segment with isValidDigits: + // month/day/hour/minute/second take 1-2 digits, the fraction may be empty, a timestamp year + // takes at most 6 digits (only a date takes 7), and a zone id is only recognised after the + // seconds segment. The fuzz alphabet in timestampPattern contains neither '-' nor '.', so + // these shapes have to be listed explicitly (https://github.com/apache/datafusion-comet/issues/5674). + private val sparkSegmentRuleTimestamps = Seq( + // 1-2 digit segments + "2020-1", + "2020-1-1", + "2020-1-1T1", + "2020-1-1 1:2", + "2020-01-01 12:34:5", + "2020-1-1T1:2:3.4", + // empty fraction, alone and before a zone + "2020-01-01 12:34:56.", + "2020-01-01 12:34:56.Z", + // 6-digit year is the timestamp maximum + "002020-01-01 00:00:00") + + private val sparkSegmentRuleMalformedTimestamps = Seq( + // zone suffix before the seconds segment + "2020Z", + "2020-10-01Z", + "2020-01-01+05:30", + "2020-01-01-08:00", + "2020-10-01 UTC", + "2020-01-01T12Z", + "2020-01-01T12:34Z", + "2020-01-01 12:34 UTC", + "2020-01-01T12:34:Z", + // 7-digit year + "0002020-01-01", + "0002020-01-01 00:00:00", + // 3-digit segments + "2020-001-01", + "2020-01-01T12:345") + + test("cast StringType to TimestampType - Spark segment rules") { + withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> "UTC") { + castTimestampTest( + sparkSegmentRuleTimestamps.toDF("a"), + DataTypes.TimestampType, + assertNative = true) + // One row per query so that every malformed value is checked under ANSI mode rather + // than only the first row that fails a batch. + sparkSegmentRuleMalformedTimestamps.foreach { value => + castTimestampTest(Seq(value).toDF("a"), DataTypes.TimestampType, assertNative = true) + } + } + } + + test("cast StringType to TimestampNTZType - Spark segment rules") { + withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> "UTC") { + castTimestampTest( + sparkSegmentRuleTimestamps.toDF("a"), + DataTypes.TimestampNTZType, + assertNative = true) + sparkSegmentRuleMalformedTimestamps.foreach { value => + castTimestampTest(Seq(value).toDF("a"), DataTypes.TimestampNTZType, assertNative = true) + } + } + } + // CAST from BinaryType test("cast BinaryType to StringType") { From bcd3803717ec3e6d24d7a30637251d2669307591 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Sat, 5 Sep 2026 03:37:35 +0800 Subject: [PATCH 2/5] fix: restrict timestamp segments to ASCII digits --- .../spark-expr/src/conversion_funcs/string.rs | 24 ++++++++++++------- .../apache/comet/CometNativeCastSuite.scala | 5 ++++ 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/native/spark-expr/src/conversion_funcs/string.rs b/native/spark-expr/src/conversion_funcs/string.rs index 39a9780324e..bc3c6e9571a 100644 --- a/native/spark-expr/src/conversion_funcs/string.rs +++ b/native/spark-expr/src/conversion_funcs/string.rs @@ -1648,18 +1648,20 @@ type TimestampParsePattern = (&'static Regex, fn(&str, &T) -> SparkResult = LazyLock::new(|| Regex::new(r"^-?\d{4,6}$").unwrap()); -static RE_MONTH: LazyLock = LazyLock::new(|| Regex::new(r"^-?\d{4,6}-\d{1,2}$").unwrap()); +static RE_MONTH: LazyLock = LazyLock::new(|| Regex::new(r"^-?\d{4,6}-[0-9]{1,2}$").unwrap()); static RE_DAY: LazyLock = - LazyLock::new(|| Regex::new(r"^-?\d{4,6}-\d{1,2}-\d{1,2}$").unwrap()); + LazyLock::new(|| Regex::new(r"^-?\d{4,6}-[0-9]{1,2}-[0-9]{1,2}$").unwrap()); static RE_HOUR: LazyLock = - LazyLock::new(|| Regex::new(r"^-?\d{4,6}-\d{1,2}-\d{1,2}[T ]\d{1,2}$").unwrap()); -static RE_MINUTE: LazyLock = - LazyLock::new(|| Regex::new(r"^-?\d{4,6}-\d{1,2}-\d{1,2}[T ]\d{1,2}:\d{1,2}$").unwrap()); + LazyLock::new(|| Regex::new(r"^-?\d{4,6}-[0-9]{1,2}-[0-9]{1,2}[T ]\d{1,2}$").unwrap()); +static RE_MINUTE: LazyLock = LazyLock::new(|| { + Regex::new(r"^-?\d{4,6}-[0-9]{1,2}-[0-9]{1,2}[T ][0-9]{1,2}:[0-9]{1,2}$").unwrap() +}); static RE_SECOND: LazyLock = LazyLock::new(|| { - Regex::new(r"^-?\d{4,6}-\d{1,2}-\d{1,2}[T ]\d{1,2}:\d{1,2}:\d{1,2}$").unwrap() + Regex::new(r"^-?\d{4,6}-[0-9]{1,2}-[0-9]{1,2}[T ][0-9]{1,2}:[0-9]{1,2}:[0-9]{1,2}$").unwrap() }); static RE_MICROSECOND: LazyLock = LazyLock::new(|| { - Regex::new(r"^-?\d{4,6}-\d{1,2}-\d{1,2}[T ]\d{1,2}:\d{1,2}:\d{1,2}\.\d*$").unwrap() + Regex::new(r"^-?\d{4,6}-[0-9]{1,2}-[0-9]{1,2}[T ][0-9]{1,2}:[0-9]{1,2}:[0-9]{1,2}\.\d*$") + .unwrap() }); static RE_TIME_ONLY_H: LazyLock = LazyLock::new(|| Regex::new(r"^T\d{1,2}$").unwrap()); static RE_TIME_ONLY_HM: LazyLock = @@ -2986,9 +2988,13 @@ mod tests { ("002020-01-01 00:00:00", JAN1_2020), ]; - /// Inputs Spark rejects: a zone suffix anywhere but after the seconds segment, more than - /// six year digits, and more than two digits in any other segment. + /// Inputs Spark rejects: non-ASCII segment digits, a zone suffix anywhere but after the + /// seconds segment, more than six year digits, and more than two digits in any other segment. const SPARK_SEGMENT_RULE_INVALID: &[&str] = &[ + "2020-٢", + "2020-01-٢", + "2020-01-01T1:٢", + "2020-01-01T1:2:٣", "2020Z", "2020-10-01Z", "2020-01-01+05:30", diff --git a/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala b/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala index 8c607235080..5abe27bcbd9 100644 --- a/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala @@ -1418,6 +1418,11 @@ class CometNativeCastSuite extends CometTestBase with AdaptiveSparkPlanHelper { "002020-01-01 00:00:00") private val sparkSegmentRuleMalformedTimestamps = Seq( + // Spark's scanner only accepts ASCII digits in timestamp segments + "2020-٢", + "2020-01-٢", + "2020-01-01T1:٢", + "2020-01-01T1:2:٣", // zone suffix before the seconds segment "2020Z", "2020-10-01Z", From 0a1fc76a02ae14228a6c521d6dda1949f8afc312 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Sat, 5 Sep 2026 09:29:06 +0800 Subject: [PATCH 3/5] fix: use ASCII digits throughout timestamp patterns --- .../spark-expr/src/conversion_funcs/string.rs | 44 ++++++++++++------- .../apache/comet/CometNativeCastSuite.scala | 11 +++++ 2 files changed, 40 insertions(+), 15 deletions(-) diff --git a/native/spark-expr/src/conversion_funcs/string.rs b/native/spark-expr/src/conversion_funcs/string.rs index bc3c6e9571a..577dfe24400 100644 --- a/native/spark-expr/src/conversion_funcs/string.rs +++ b/native/spark-expr/src/conversion_funcs/string.rs @@ -1646,35 +1646,38 @@ type TimestampParsePattern = (&'static Regex, fn(&str, &T) -> SparkResult = LazyLock::new(|| Regex::new(r"^-?\d{4,6}$").unwrap()); -static RE_MONTH: LazyLock = LazyLock::new(|| Regex::new(r"^-?\d{4,6}-[0-9]{1,2}$").unwrap()); +// valid), of which only the first six are kept. All digits must be ASCII, matching Spark's +// byte scanner and the numeric parsers used after shape recognition. +static RE_YEAR: LazyLock = LazyLock::new(|| Regex::new(r"^-?[0-9]{4,6}$").unwrap()); +static RE_MONTH: LazyLock = + LazyLock::new(|| Regex::new(r"^-?[0-9]{4,6}-[0-9]{1,2}$").unwrap()); static RE_DAY: LazyLock = - LazyLock::new(|| Regex::new(r"^-?\d{4,6}-[0-9]{1,2}-[0-9]{1,2}$").unwrap()); + LazyLock::new(|| Regex::new(r"^-?[0-9]{4,6}-[0-9]{1,2}-[0-9]{1,2}$").unwrap()); static RE_HOUR: LazyLock = - LazyLock::new(|| Regex::new(r"^-?\d{4,6}-[0-9]{1,2}-[0-9]{1,2}[T ]\d{1,2}$").unwrap()); + LazyLock::new(|| Regex::new(r"^-?[0-9]{4,6}-[0-9]{1,2}-[0-9]{1,2}[T ][0-9]{1,2}$").unwrap()); static RE_MINUTE: LazyLock = LazyLock::new(|| { - Regex::new(r"^-?\d{4,6}-[0-9]{1,2}-[0-9]{1,2}[T ][0-9]{1,2}:[0-9]{1,2}$").unwrap() + Regex::new(r"^-?[0-9]{4,6}-[0-9]{1,2}-[0-9]{1,2}[T ][0-9]{1,2}:[0-9]{1,2}$").unwrap() }); static RE_SECOND: LazyLock = LazyLock::new(|| { - Regex::new(r"^-?\d{4,6}-[0-9]{1,2}-[0-9]{1,2}[T ][0-9]{1,2}:[0-9]{1,2}:[0-9]{1,2}$").unwrap() + Regex::new(r"^-?[0-9]{4,6}-[0-9]{1,2}-[0-9]{1,2}[T ][0-9]{1,2}:[0-9]{1,2}:[0-9]{1,2}$").unwrap() }); static RE_MICROSECOND: LazyLock = LazyLock::new(|| { - Regex::new(r"^-?\d{4,6}-[0-9]{1,2}-[0-9]{1,2}[T ][0-9]{1,2}:[0-9]{1,2}:[0-9]{1,2}\.\d*$") + Regex::new(r"^-?[0-9]{4,6}-[0-9]{1,2}-[0-9]{1,2}[T ][0-9]{1,2}:[0-9]{1,2}:[0-9]{1,2}\.[0-9]*$") .unwrap() }); -static RE_TIME_ONLY_H: LazyLock = LazyLock::new(|| Regex::new(r"^T\d{1,2}$").unwrap()); +static RE_TIME_ONLY_H: LazyLock = LazyLock::new(|| Regex::new(r"^T[0-9]{1,2}$").unwrap()); static RE_TIME_ONLY_HM: LazyLock = - LazyLock::new(|| Regex::new(r"^T\d{1,2}:\d{1,2}$").unwrap()); + LazyLock::new(|| Regex::new(r"^T[0-9]{1,2}:[0-9]{1,2}$").unwrap()); static RE_TIME_ONLY_HMS: LazyLock = - LazyLock::new(|| Regex::new(r"^T\d{1,2}:\d{1,2}:\d{1,2}$").unwrap()); + LazyLock::new(|| Regex::new(r"^T[0-9]{1,2}:[0-9]{1,2}:[0-9]{1,2}$").unwrap()); static RE_TIME_ONLY_HMSU: LazyLock = - LazyLock::new(|| Regex::new(r"^T\d{1,2}:\d{1,2}:\d{1,2}\.\d*$").unwrap()); -static RE_BARE_HM: LazyLock = LazyLock::new(|| Regex::new(r"^\d{1,2}:\d{1,2}$").unwrap()); + LazyLock::new(|| Regex::new(r"^T[0-9]{1,2}:[0-9]{1,2}:[0-9]{1,2}\.[0-9]*$").unwrap()); +static RE_BARE_HM: LazyLock = + LazyLock::new(|| Regex::new(r"^[0-9]{1,2}:[0-9]{1,2}$").unwrap()); static RE_BARE_HMS: LazyLock = - LazyLock::new(|| Regex::new(r"^\d{1,2}:\d{1,2}:\d{1,2}$").unwrap()); + LazyLock::new(|| Regex::new(r"^[0-9]{1,2}:[0-9]{1,2}:[0-9]{1,2}$").unwrap()); static RE_BARE_HMSU: LazyLock = - LazyLock::new(|| Regex::new(r"^\d{1,2}:\d{1,2}:\d{1,2}\.\d*$").unwrap()); + LazyLock::new(|| Regex::new(r"^[0-9]{1,2}:[0-9]{1,2}:[0-9]{1,2}\.[0-9]*$").unwrap()); /// Whether `value` (a datetime with any zone suffix already stripped) ends in a seconds or /// fraction segment. Spark's `parseTimestampString` only captures a zone id when its byte @@ -2991,10 +2994,21 @@ mod tests { /// Inputs Spark rejects: non-ASCII segment digits, a zone suffix anywhere but after the /// seconds segment, more than six year digits, and more than two digits in any other segment. const SPARK_SEGMENT_RULE_INVALID: &[&str] = &[ + "2020-1-1T٢", + "2020-1-1T1:2:3.٢", + "٢020-1-1", "2020-٢", "2020-01-٢", "2020-01-01T1:٢", "2020-01-01T1:2:٣", + "2020-1-1T1:2:3.٢Z", + "T٢", + "T1:٢", + "T1:2:٣", + "T1:2:3.٢", + "1:٢", + "1:2:٣", + "1:2:3.٢", "2020Z", "2020-10-01Z", "2020-01-01+05:30", diff --git a/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala b/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala index 5abe27bcbd9..071fa778fe5 100644 --- a/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala @@ -1419,10 +1419,21 @@ class CometNativeCastSuite extends CometTestBase with AdaptiveSparkPlanHelper { private val sparkSegmentRuleMalformedTimestamps = Seq( // Spark's scanner only accepts ASCII digits in timestamp segments + "٢020-1-1", "2020-٢", "2020-01-٢", + "2020-1-1T٢", "2020-01-01T1:٢", "2020-01-01T1:2:٣", + "2020-1-1T1:2:3.٢", + "2020-1-1T1:2:3.٢Z", + "T٢", + "T1:٢", + "T1:2:٣", + "T1:2:3.٢", + "1:٢", + "1:2:٣", + "1:2:3.٢", // zone suffix before the seconds segment "2020Z", "2020-10-01Z", From 7c449fd9cfcead1f3b88269a8bbeffa7671724ac Mon Sep 17 00:00:00 2001 From: peterxcli Date: Sun, 6 Sep 2026 00:47:47 +0800 Subject: [PATCH 4/5] fix: address timestamp parser review regressions and fuzz coverage --- .../user-guide/latest/compatibility/index.md | 4 ++ .../benches/cast_string_to_timestamp.rs | 14 ++++++- .../spark-expr/src/conversion_funcs/string.rs | 41 +++++++++++++++++-- .../apache/comet/CometNativeCastSuite.scala | 22 ++++++---- 4 files changed, 69 insertions(+), 12 deletions(-) diff --git a/docs/source/user-guide/latest/compatibility/index.md b/docs/source/user-guide/latest/compatibility/index.md index 001ec876061..c206c26178a 100644 --- a/docs/source/user-guide/latest/compatibility/index.md +++ b/docs/source/user-guide/latest/compatibility/index.md @@ -126,6 +126,10 @@ divergence: ## Known result-value divergences +- **Explicit positive timestamp years:** Spark accepts strings such as `+7528` as the start + of that year, while Comet's native string-to-timestamp cast returns NULL in non-ANSI mode + ([#5716](https://github.com/apache/datafusion-comet/issues/5716)). + The following native paths silently return values that differ from Spark for edge-case inputs. Most also have entries in the per-category expression pages linked above; they are collected here so users hunting an unexpected value have a single place to check: diff --git a/native/spark-expr/benches/cast_string_to_timestamp.rs b/native/spark-expr/benches/cast_string_to_timestamp.rs index 3e83d78756d..43eca362b45 100644 --- a/native/spark-expr/benches/cast_string_to_timestamp.rs +++ b/native/spark-expr/benches/cast_string_to_timestamp.rs @@ -35,6 +35,18 @@ fn criterion_benchmark(c: &mut Criterion) { // path), a date-only string, whitespace padding (the trim), and a mix that includes // invalid values so the null path is measured too. let batches = [ + ( + "single_digit_segments", + create_batch(|i| format!("2020-{:02}-{}T1:2:3", i % 12 + 1, i % 9 + 1)), + ), + ( + "empty_fraction", + create_batch(|i| format!("2020-01-{:02}T12:34:56.", i % 28 + 1)), + ), + ( + "date_only_zone", + create_batch(|i| format!("2020-01-{:02}Z", i % 28 + 1)), + ), ( "canonical", create_batch(|i| { @@ -151,7 +163,7 @@ fn criterion_benchmark(c: &mut Criterion) { for (name, batch) in &batches { // ANSI raises on the first invalid value, so timing it against a batch that is // mostly invalid would measure the error path rather than the parser. - if mode == EvalMode::Ansi && *name == "mixed" { + if mode == EvalMode::Ansi && matches!(*name, "mixed" | "date_only_zone") { continue; } let cast = Cast::new( diff --git a/native/spark-expr/src/conversion_funcs/string.rs b/native/spark-expr/src/conversion_funcs/string.rs index 577dfe24400..c8b7461791d 100644 --- a/native/spark-expr/src/conversion_funcs/string.rs +++ b/native/spark-expr/src/conversion_funcs/string.rs @@ -1153,11 +1153,15 @@ fn parse_to_timestamp_info( let hour = parts.next().map_or(0, |h| h.parse::().unwrap_or(0)); let minute = parts.next().map_or(0, |m| m.parse::().unwrap_or(0)); let second = parts.next().map_or(0, |s| s.parse::().unwrap_or(0)); - let microsecond = parts.next().map_or(0, |ms| { - let ms = &ms[..ms.len().min(6)]; + let microsecond = if let Some(ms) = parts.next() { + let Some(ms) = ms.get(..ms.len().min(6)) else { + return Ok(None); + }; let n = ms.len(); ms.parse::().unwrap_or(0) * 10u32.pow((6 - n) as u32) - }); + } else { + 0 + }; let mut timestamp_info = TimeStampInfo::default(); @@ -1470,6 +1474,7 @@ fn timestamp_parser( if !has_direct_match { if let Some((stripped, suffix_tz)) = extract_offset_suffix(value) { + let stripped = stripped.trim_end(); // A zone suffix is only meaningful after the seconds segment. Otherwise fall // through with the unstripped value, which no base pattern matches, so it is // reported as malformed (null, or CAST_INVALID_INPUT under ANSI) like Spark does. @@ -1648,6 +1653,7 @@ type TimestampParsePattern = (&'static Regex, fn(&str, &T) -> SparkResult = LazyLock::new(|| Regex::new(r"^-?[0-9]{4,6}$").unwrap()); static RE_MONTH: LazyLock = LazyLock::new(|| Regex::new(r"^-?[0-9]{4,6}-[0-9]{1,2}$").unwrap()); @@ -1901,7 +1907,9 @@ fn parse_str_to_time_only_timestamp(value: &str, tz: &T) -> SparkRe let ns: u32 = if let Some(dot) = dot_idx { let frac = &sec_frac[dot + 1..]; // Interpret up to 6 digits as microseconds, padding with trailing zeros. - let trimmed = &frac[..frac.len().min(6)]; + let Some(trimmed) = frac.get(..frac.len().min(6)) else { + return Ok(None); + }; let padded = format!("{:0<6}", trimmed); padded.parse::().unwrap_or(0) * 1000 } else { @@ -2981,6 +2989,8 @@ mod tests { /// 1-2 digit month/day/hour/minute/second, an empty fraction (also before a zone), and /// 6-digit years (issue #5674). Values are UTC micros, identical for TIMESTAMP_NTZ. const SPARK_SEGMENT_RULE_VALID: &[(&str, i64)] = &[ + ("2020-10-1", 1_601_510_400_000_000), + ("2020-12-1", 1_606_780_800_000_000), ("2020-1", JAN1_2020), ("2020-1-1", JAN1_2020), ("2020-1-1T1", JAN1_2020 + 3600 * 1_000_000), @@ -2994,6 +3004,8 @@ mod tests { /// Inputs Spark rejects: non-ASCII segment digits, a zone suffix anywhere but after the /// seconds segment, more than six year digits, and more than two digits in any other segment. const SPARK_SEGMENT_RULE_INVALID: &[&str] = &[ + "2020-01-01 12:34:56.1٢٢٢", + "T1:2:3.1٢٢٢", "2020-1-1T٢", "2020-1-1T1:2:3.٢", "٢020-1-1", @@ -3033,6 +3045,27 @@ mod tests { #[cfg_attr(miri, ignore)] fn timestamp_parser_spark_segment_rules_test() { let tz = &Tz::from_str("UTC").unwrap(); + // Exercise the decoders without their regex gates: malformed UTF-8 boundaries + // must not panic even if the accepted patterns change later. + assert!( + parse_to_timestamp_info("2020-01-01 12:34:56.1٢٢٢", "microsecond") + .unwrap() + .is_none() + ); + assert_eq!( + parse_str_to_time_only_timestamp("T1:2:3.1٢٢٢", tz).unwrap(), + None + ); + for mode in [EvalMode::Legacy, EvalMode::Try, EvalMode::Ansi] { + assert_eq!( + timestamp_parser("2021-11-22 10:54:27 +08:00", mode, tz, true).unwrap(), + Some(1_637_549_667_000_000) + ); + assert_eq!( + timestamp_ntz_parser("2021-11-22 10:54:27 +08:00", mode, true, true).unwrap(), + Some(1_637_578_467_000_000) + ); + } for &(input, expected) in SPARK_SEGMENT_RULE_VALID { for eval_mode in [EvalMode::Legacy, EvalMode::Try, EvalMode::Ansi] { assert_eq!( diff --git a/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala b/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala index 071fa778fe5..5f244df2d7f 100644 --- a/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala @@ -89,7 +89,7 @@ class CometNativeCastSuite extends CometTestBase with AdaptiveSparkPlanHelper { private val datePattern = "0123456789/" + whitespaceChars - private val timestampPattern = "0123456789/:T" + whitespaceChars + private val timestampPattern = "0123456789/:T-.+Z" + whitespaceChars lazy val usingParquetExecWithIncompatTypes: Boolean = hasUnsignedSmallIntSafetyCheck(conf) @@ -1267,10 +1267,12 @@ class CometNativeCastSuite extends CometTestBase with AdaptiveSparkPlanHelper { test("cast StringType to TimestampType") { withSQLConf((SQLConf.SESSION_LOCAL_TIMEZONE.key, "UTC")) { - val values = Seq("2020-01-01T12:34:56.123456", "T2") ++ gen.generateStrings( - dataSize, - timestampPattern, - 8) + // Spark accepts explicit positive years; Comet does not yet (#5716). + // Keep the wider alphabet, excluding only the known bare-year mismatch. + val fuzzValues = gen + .generateStrings(dataSize, timestampPattern, 8) + .filterNot(_.trim.matches("\\+[0-9]{4,6}")) + val values = Seq("2020-01-01T12:34:56.123456", "T2") ++ fuzzValues castTest(values.toDF("a"), DataTypes.TimestampType) } } @@ -1401,12 +1403,15 @@ class CometNativeCastSuite extends CometTestBase with AdaptiveSparkPlanHelper { // Spark's SparkDateTimeUtils.parseTimestampString validates each segment with isValidDigits: // month/day/hour/minute/second take 1-2 digits, the fraction may be empty, a timestamp year // takes at most 6 digits (only a date takes 7), and a zone id is only recognised after the - // seconds segment. The fuzz alphabet in timestampPattern contains neither '-' nor '.', so - // these shapes have to be listed explicitly (https://github.com/apache/datafusion-comet/issues/5674). + // seconds segment. Keep explicit cases as well as fuzz coverage for these segment boundaries. private val sparkSegmentRuleTimestamps = Seq( // 1-2 digit segments "2020-1", "2020-1-1", + "2020-10-1", + "2020-12-1", + "-0001-01-01T12:34:56", + "2021-11-22 10:54:27 +08:00", "2020-1-1T1", "2020-1-1 1:2", "2020-01-01 12:34:5", @@ -1418,6 +1423,9 @@ class CometNativeCastSuite extends CometTestBase with AdaptiveSparkPlanHelper { "002020-01-01 00:00:00") private val sparkSegmentRuleMalformedTimestamps = Seq( + "-0002020-01-01", + "2020-01-01 12:34:56.1٢٢٢", + "T1:2:3.1٢٢٢", // Spark's scanner only accepts ASCII digits in timestamp segments "٢020-1-1", "2020-٢", From 89e5ff5cce65a5745757f6dfcc05f096d8d937ea Mon Sep 17 00:00:00 2001 From: peterxcli Date: Sun, 6 Sep 2026 05:38:00 +0800 Subject: [PATCH 5/5] Preserve Spark 4 leading-whitespace rejection with timestamp offsets --- .../spark-expr/src/conversion_funcs/string.rs | 37 +++++++++++-------- .../apache/comet/CometNativeCastSuite.scala | 11 ++++++ 2 files changed, 32 insertions(+), 16 deletions(-) diff --git a/native/spark-expr/src/conversion_funcs/string.rs b/native/spark-expr/src/conversion_funcs/string.rs index c8b7461791d..99b7c6f856f 100644 --- a/native/spark-expr/src/conversion_funcs/string.rs +++ b/native/spark-expr/src/conversion_funcs/string.rs @@ -1421,14 +1421,8 @@ fn timestamp_parser( // Spark 4.0+ rejects leading whitespace for ALL T-prefixed time-only strings // (T, T:, T::, T::.), but accepts trailing whitespace. // Spark 3.x trims all whitespace first, so leading whitespace is accepted there. - // Check the raw (pre-trim) value for leading whitespace before any T-time-only match. - if is_spark4_plus - && value.len() > value.trim_start().len() - && (RE_TIME_ONLY_H.is_match(trimmed) - || RE_TIME_ONLY_HM.is_match(trimmed) - || RE_TIME_ONLY_HMS.is_match(trimmed) - || RE_TIME_ONLY_HMSU.is_match(trimmed)) - { + // Check the prefix, not the base patterns: a zone suffix can hide a time-only match. + if is_spark4_plus && value.len() > value.trim_start().len() && trimmed.starts_with('T') { return if eval_mode == EvalMode::Ansi { Err(SparkError::InvalidInputInCastToDatetime { value: value.to_string(), @@ -2573,13 +2567,24 @@ mod tests { fn test_leading_whitespace_t_hm() { let tz = &Tz::from_str("UTC").unwrap(); // Spark 4.0+ rejects leading whitespace for ALL T-prefixed time-only patterns. - for ws_input in &[" T2:30", "\tT2:30", "\nT2:30", " T2", "\tT2", "\nT2"] { - assert!( - timestamp_parser(ws_input, EvalMode::Legacy, tz, true) - .unwrap() - .is_none(), - "'{ws_input}' should be null in Legacy mode on Spark 4.0+" - ); + for ws_input in &[ + " T2:30", + "\tT2:30", + "\nT2:30", + " T2", + "\tT2", + "\nT2", + "\tT1:2:3 +08:00", + " T1:2:3.4 +08:00", + ] { + for mode in [EvalMode::Legacy, EvalMode::Try] { + assert!( + timestamp_parser(ws_input, mode, tz, true) + .unwrap() + .is_none(), + "'{ws_input}' should be null in {mode:?} mode on Spark 4.0+" + ); + } // In ANSI mode the same inputs must raise an error (not silently return null). assert!( timestamp_parser(ws_input, EvalMode::Ansi, tz, true).is_err(), @@ -2594,7 +2599,7 @@ mod tests { ); } // Without leading whitespace, these must be valid on all versions. - for ok_input in &["T2:30", "T2"] { + for ok_input in &["T2:30", "T2", "T1:2:3 +08:00", "T1:2:3.4 +08:00"] { assert!( timestamp_parser(ok_input, EvalMode::Legacy, tz, true) .unwrap() diff --git a/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala b/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala index 5f244df2d7f..29226f91374 100644 --- a/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala @@ -1372,6 +1372,17 @@ class CometNativeCastSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } + test("cast StringType to TimestampType - time-only offset leading whitespace") { + withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> "UTC") { + // One Parquet-backed column value per query exercises each input in Legacy, TRY and ANSI. + // Spark 4 rejects the leading whitespace; Spark 3.5 accepts it. NTZ rejects time-only input. + Seq("\tT1:2:3 +08:00", " T1:2:3.4 +08:00", "T1:2:3 +08:00").foreach { value => + castTimestampTest(Seq(value).toDF("a"), DataTypes.TimestampType, assertNative = true) + castTimestampTest(Seq(value).toDF("a"), DataTypes.TimestampNTZType, assertNative = true) + } + } + } + test("cast StringType to TimestampNTZType") { representativeTimezones.foreach { tz => withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> tz) {