diff --git a/native/spark-expr/benches/cast_numeric.rs b/native/spark-expr/benches/cast_numeric.rs index 989cbf4d2cf..5153fb7d011 100644 --- a/native/spark-expr/benches/cast_numeric.rs +++ b/native/spark-expr/benches/cast_numeric.rs @@ -15,13 +15,16 @@ // specific language governing permissions and limitations // under the License. -use arrow::array::{builder::Int32Builder, RecordBatch}; +use arrow::array::{builder::Int32Builder, Decimal128Array, RecordBatch}; use arrow::datatypes::{DataType, Field, Schema}; use criterion::{criterion_group, criterion_main, Criterion}; use datafusion::physical_expr::{expressions::Column, PhysicalExpr}; use datafusion_comet_spark_expr::{Cast, EvalMode, SparkCastOptions}; +use std::hint::black_box; use std::sync::Arc; +const NUM_ROWS: usize = 8192; + fn criterion_benchmark(c: &mut Criterion) { let batch = create_int32_batch(); let expr = Arc::new(Column::new("a", 0)); @@ -52,6 +55,59 @@ fn criterion_benchmark(c: &mut Criterion) { group.bench_function("cast_i32_to_i64", |b| { b.iter(|| cast_i32_to_i64.evaluate(&batch).unwrap()); }); + group.finish(); + + let decimal_cast = |data_type| { + Cast::new( + Arc::new(Column::new("a", 0)), + data_type, + SparkCastOptions::new_without_timezone(EvalMode::Legacy, false), + None, + None, + ) + }; + let decimal_to_f64 = decimal_cast(DataType::Float64); + let decimal_to_f32 = decimal_cast(DataType::Float32); + let cases = [ + ( + "decimal18_to_f64", + create_decimal128_batch(18, 0, 1_i128 << 53), + &decimal_to_f64, + ), + ( + "decimal18_to_f64_nulls", + create_decimal128_batch(18, 5, 1_i128 << 53), + &decimal_to_f64, + ), + ( + "decimal38_to_f64", + create_decimal128_batch(38, 0, 10_i128.pow(37)), + &decimal_to_f64, + ), + ( + "decimal38_to_f64_nulls", + create_decimal128_batch(38, 5, 10_i128.pow(37)), + &decimal_to_f64, + ), + ( + "decimal12_to_f32", + create_decimal128_batch(12, 0, 1_i128 << 24), + &decimal_to_f32, + ), + ( + "decimal12_to_f32_nulls", + create_decimal128_batch(12, 5, 1_i128 << 24), + &decimal_to_f32, + ), + ]; + + let mut group = c.benchmark_group("cast_decimal_scale_zero"); + for (name, batch, cast) in cases { + group.bench_function(name, |b| { + b.iter(|| black_box(cast.evaluate(black_box(&batch)).unwrap())) + }); + } + group.finish(); } fn create_int32_batch() -> RecordBatch { @@ -69,6 +125,27 @@ fn create_int32_batch() -> RecordBatch { RecordBatch::try_new(schema.clone(), vec![Arc::new(array)]).unwrap() } +fn create_decimal128_batch(precision: u8, null_every: usize, base: i128) -> RecordBatch { + let array: Decimal128Array = (0..NUM_ROWS) + .map(|i| { + if null_every != 0 && i % null_every == 0 { + None + } else { + let magnitude = base + i as i128; + Some(if i % 2 == 0 { magnitude } else { -magnitude }) + } + }) + .collect::() + .with_precision_and_scale(precision, 0) + .unwrap(); + let schema = Arc::new(Schema::new(vec![Field::new( + "a", + DataType::Decimal128(precision, 0), + true, + )])); + RecordBatch::try_new(schema, vec![Arc::new(array)]).unwrap() +} + fn config() -> Criterion { Criterion::default() } diff --git a/native/spark-expr/src/conversion_funcs/cast.rs b/native/spark-expr/src/conversion_funcs/cast.rs index b1b71e8267d..7a9b93ff163 100644 --- a/native/spark-expr/src/conversion_funcs/cast.rs +++ b/native/spark-expr/src/conversion_funcs/cast.rs @@ -19,12 +19,13 @@ use crate::conversion_funcs::boolean::{ cast_boolean_to_timestamp, is_df_cast_from_bool_spark_compatible, }; use crate::conversion_funcs::numeric::{ - cast_decimal128_to_utf8, cast_decimal_to_timestamp, cast_float32_to_decimal128, - cast_float64_to_decimal128, cast_float_to_timestamp, cast_int_to_decimal128, - cast_int_to_timestamp, is_df_cast_from_decimal_spark_compatible, - is_df_cast_from_float_spark_compatible, is_df_cast_from_int_spark_compatible, - spark_cast_decimal_to_boolean, spark_cast_float32_to_utf8, spark_cast_float64_to_utf8, - spark_cast_int_to_int, spark_cast_nonintegral_numeric_to_integral, + cast_decimal128_to_float32, cast_decimal128_to_float64, cast_decimal128_to_utf8, + cast_decimal_to_timestamp, cast_float32_to_decimal128, cast_float64_to_decimal128, + cast_float_to_timestamp, cast_int_to_decimal128, cast_int_to_timestamp, + is_df_cast_from_decimal_spark_compatible, is_df_cast_from_float_spark_compatible, + is_df_cast_from_int_spark_compatible, spark_cast_decimal_to_boolean, + spark_cast_float32_to_utf8, spark_cast_float64_to_utf8, spark_cast_int_to_int, + spark_cast_nonintegral_numeric_to_integral, }; use crate::conversion_funcs::string::{ cast_string_to_date, cast_string_to_decimal, cast_string_to_float, cast_string_to_int, @@ -327,6 +328,11 @@ pub(crate) fn cast_array( spark_cast_nonintegral_numeric_to_integral(&array, eval_mode, &from_type, to_type) } (Decimal128(_p, _s), Boolean) => spark_cast_decimal_to_boolean(&array), + // Spark rounds the exact decimal value once (BigDecimal.doubleValue / floatValue); + // DataFusion's `(unscaled as f64) / 10^scale` rounds twice and can be off by one ulp. + // The conversion cannot fail, so it is the same in every eval mode. + (Decimal128(_, scale), Float64) => cast_decimal128_to_float64(&array, *scale), + (Decimal128(_, scale), Float32) => cast_decimal128_to_float32(&array, *scale), // Spark LEGACY cast uses Java BigDecimal.toString() which produces scientific notation // when adjusted_exponent < -6 (e.g. "0E-18" for zero with scale=18). // TRY and ANSI use plain notation ("0.000000000000000000") so DataFusion handles those. diff --git a/native/spark-expr/src/conversion_funcs/numeric.rs b/native/spark-expr/src/conversion_funcs/numeric.rs index 029a38e3e7f..79bdfe5b246 100644 --- a/native/spark-expr/src/conversion_funcs/numeric.rs +++ b/native/spark-expr/src/conversion_funcs/numeric.rs @@ -68,9 +68,11 @@ pub(crate) fn is_df_cast_from_decimal_spark_compatible(to_type: &DataType) -> bo | DataType::Int16 | DataType::Int32 | DataType::Int64 - | DataType::Float32 // DataFusion divides i128 by 10^scale in f64, then narrows to - | DataType::Float64 // f32 if needed; empirically matches Spark's BigDecimal.doubleValue - // / floatValue for all tested values + // Float32 / Float64 are intentionally absent: DataFusion computes + // `(unscaled as f64) / 10^scale`, which rounds twice (three times for f32, via the + // f64 intermediate) and can differ from Spark's correctly rounded + // BigDecimal.doubleValue() / floatValue() by one ulp once |unscaled| > 2^53. + // cast.rs routes Decimal128 sources to cast_decimal128_to_float64 / _float32. | DataType::Decimal128(_, _) | DataType::Decimal256(_, _) // DataFusion's Decimal128→Utf8 cast uses plain notation (toPlainString semantics), @@ -863,6 +865,109 @@ pub(crate) fn spark_cast_decimal_to_boolean(array: &dyn Array) -> SparkResult 2^53` (any `DECIMAL(38,18)` value of +/// magnitude `>= 0.01`) the two roundings can land one ulp away from the correctly rounded +/// result, e.g. `12345.6789` becomes `12345.678899999999`. +pub(crate) fn decimal128_to_f64(unscaled: i128, scale: i8) -> f64 { + if scale == 0 { + return unscaled as f64; + } + + // Fast path (also Java's): when the unscaled value and the power of ten are both exact + // doubles, a single IEEE division or multiplication is correctly rounded. + if unscaled.unsigned_abs() <= F64_EXACT_INT_LIMIT { + let m = unscaled as f64; + match scale { + 1..=22 => return m / F64_EXACT_POW10[scale as usize], + -22..=-1 => return m * F64_EXACT_POW10[scale.unsigned_abs() as usize], + _ => {} + } + } + parse_exact_decimal(unscaled, scale) +} + +/// Converts a Decimal128 value to the `f32` nearest to its exact decimal value (ties to even), +/// matching Spark's `Decimal.toFloat`, i.e. `java.math.BigDecimal.floatValue()`. +/// +/// The exact value is rounded straight to `f32`, never through an `f64` intermediate, because +/// rounding twice is observable: `16777217.0000000001` is `16777218` as a float, but narrowing +/// the nearest `f64` (`16777217.0`, a tie) gives `16777216`. +pub(crate) fn decimal128_to_f32(unscaled: i128, scale: i8) -> f32 { + if scale == 0 { + return unscaled as f32; + } + + if unscaled.unsigned_abs() <= F32_EXACT_INT_LIMIT { + let m = unscaled as f32; + match scale { + 1..=10 => return m / F32_EXACT_POW10[scale as usize], + -10..=-1 => return m * F32_EXACT_POW10[scale.unsigned_abs() as usize], + _ => {} + } + } + parse_exact_decimal(unscaled, scale) +} + +/// Rounds the exact value `unscaled * 10^-scale` once, by formatting it as `e<-scale>` +/// into a stack buffer and parsing it with Rust's correctly rounded float parser. This mirrors +/// Java's fallback of `Double.parseDouble(BigDecimal.toString())` / +/// `Float.parseFloat(BigDecimal.toString())`, which likewise round the exact decimal directly to +/// the target width; a value beyond the target's range parses to an infinity, as in Java. +fn parse_exact_decimal(unscaled: i128, scale: i8) -> F +where + F: std::str::FromStr, + F::Err: std::fmt::Debug, +{ + use std::io::Write; + // Sign and up to 39 digits, 'e', and an exponent of up to 4 characters (`-127`). + let mut buf = [0u8; 48]; + let mut cursor = &mut buf[..]; + write!(cursor, "{unscaled}e{}", -i32::from(scale)).expect("buffer holds any i128 and i8"); + let remaining = cursor.len(); + let text = std::str::from_utf8(&buf[..buf.len() - remaining]).expect("ascii"); + text.parse::().expect("well-formed float literal") +} + +/// Casts a Decimal128 array to Float64 with `BigDecimal.doubleValue()` semantics; see +/// [`decimal128_to_f64`]. The conversion is total and cannot overflow, so every eval mode +/// behaves alike and the input null buffer carries over unchanged. +pub(crate) fn cast_decimal128_to_float64(array: &dyn Array, scale: i8) -> SparkResult { + let input = array.as_primitive::(); + let result: Float64Array = input.unary(|v| decimal128_to_f64(v, scale)); + Ok(Arc::new(result)) +} + +/// Casts a Decimal128 array to Float32 with `BigDecimal.floatValue()` semantics; see +/// [`decimal128_to_f32`]. Every Decimal128 magnitude (`< 2^127`) is below `f32::MAX`, so for +/// the non-negative scales Spark produces the conversion cannot overflow and every eval mode +/// behaves alike. +pub(crate) fn cast_decimal128_to_float32(array: &dyn Array, scale: i8) -> SparkResult { + let input = array.as_primitive::(); + let result: Float32Array = input.unary(|v| decimal128_to_f32(v, scale)); + Ok(Arc::new(result)) +} + pub(crate) fn cast_float64_to_decimal128( array: &dyn Array, precision: u8, @@ -1492,6 +1597,242 @@ mod tests { assert_eq!(ts_array.timezone(), tz.as_ref().map(|s| s.as_ref())); } } + + #[test] + fn test_decimal128_to_f64_matches_bigdecimal_double_value() { + // Expected values were checked against Java's + // `new BigDecimal(BigInteger(unscaled), scale).doubleValue()`. + let cases: Vec<(i128, i8, f64)> = vec![ + // https://github.com/apache/datafusion-comet/issues/5670 + (12345678900000000000000, 18, 12345.6789), + (123456789012000000000000, 18, 123456.789012), + (76543210000000000000000, 18, 76543.21), + (577312285583388355022308, 18, 577312.2855833884), + (-12345678900000000000000, 18, -12345.6789), + (-577312285583388355022308, 18, -577312.2855833884), + // zero and the smallest magnitudes + (0, 18, 0.0), + (0, 38, 0.0), + (1, 38, 1e-38), + (-1, 38, -1e-38), + // 38-digit extremes + (99999999999999999999999999999999999999, 0, 1e38), + (-99999999999999999999999999999999999999, 0, -1e38), + (99999999999999999999999999999999999999, 38, 1.0), + (-99999999999999999999999999999999999999, 38, -1.0), + (i128::MAX, 0, 2f64.powi(127)), + (i128::MIN, 0, -(2f64.powi(127))), + // 2^53 + 1 is halfway between two doubles and rounds to even + (9007199254740993, 0, 9007199254740992.0), + // exact-operand fast path + (1, 18, 1e-18), + (-15, 1, -1.5), + (123456789, 3, 123456.789), + (9007199254740992, 22, 9007199254740992e-22), + // negative scales, which arrow permits but Spark never produces + (123456789, -3, 123456789e3), + (9007199254740992, -22, 9007199254740992e22), + (9007199254740993, -22, 9007199254740993e22), + ]; + for (unscaled, scale, expected) in cases { + let actual = decimal128_to_f64(unscaled, scale); + assert_eq!( + actual.to_bits(), + expected.to_bits(), + "{unscaled}e{}: got {actual:?}, expected {expected:?}", + -i32::from(scale) + ); + } + // The double-rounding formula the kernel replaces gets the issue's example wrong. + assert_ne!((12345678900000000000000_i128 as f64) / 1e18, 12345.6789); + } + + #[test] + fn test_decimal128_to_f32_matches_bigdecimal_float_value() { + // Expected values were checked against Java's + // `new BigDecimal(BigInteger(unscaled), scale).floatValue()`. + let cases: Vec<(i128, i8, f32)> = vec![ + // https://github.com/apache/datafusion-comet/issues/5670: 16777217.0000000001 lies + // above the midpoint of the floats 16777216 and 16777218, but its nearest double + // (16777217.0) is exactly that midpoint and would narrow to 16777216. + (167772170000000001, 10, 16777218.0), + (-167772170000000001, 10, -16777218.0), + (12345678900000000000000, 18, 12345.679), + (76543210000000000000000, 18, 76543.21), + (-12345678900000000000000, 18, -12345.679), + // zero and the smallest magnitudes (1e-38 is subnormal in f32) + (0, 18, 0.0), + (0, 38, 0.0), + (1, 38, 1e-38), + (-1, 38, -1e-38), + // 38-digit extremes + (99999999999999999999999999999999999999, 0, 1e38), + (-99999999999999999999999999999999999999, 0, -1e38), + (99999999999999999999999999999999999999, 38, 1.0), + (-99999999999999999999999999999999999999, 38, -1.0), + (i128::MAX, 0, 2f32.powi(127)), + (i128::MIN, 0, -(2f32.powi(127))), + // 2^24 + 1 is halfway between two floats and rounds to even + (16777217, 0, 16777216.0), + // exact-operand fast path + (1, 10, 1e-10), + (-15, 1, -1.5), + (123456789, 3, 123456.79), + (16777216, 10, 16777216e-10), + // negative scales, which arrow permits but Spark never produces; beyond the f32 + // range the result is an infinity, like Float.parseFloat + (123456789, -3, 1.2345679e11), + (16777216, -10, 16777216e10), + (1, -39, f32::INFINITY), + (-1, -39, f32::NEG_INFINITY), + ]; + for (unscaled, scale, expected) in cases { + let actual = decimal128_to_f32(unscaled, scale); + assert_eq!( + actual.to_bits(), + expected.to_bits(), + "{unscaled}e{}: got {actual:?}, expected {expected:?}", + -i32::from(scale) + ); + } + // Narrowing the (already correctly rounded) double gets the issue's example wrong. + assert_ne!( + decimal128_to_f64(167772170000000001, 10) as f32, + 16777218.0_f32 + ); + } + + #[test] + fn test_decimal128_to_float_fast_path_matches_exact_path() { + use rand::rngs::StdRng; + use rand::{RngExt, SeedableRng}; + let mut rng = StdRng::seed_from_u64(5670); + + let mut cases: Vec<(i128, i8)> = Vec::new(); + // Magnitudes spread over the whole i128 range, so both the exact-operand fast paths + // (|unscaled| <= 2^24 for f32, <= 2^53 for f64) and the parse fallback are exercised. + for _ in 0..10_000 { + let bits = rng.random_range(0_u32..=127); + let magnitude = if bits == 0 { + 0 + } else { + rng.random::() >> (128 - bits) + }; + let unscaled = if rng.random::() { + magnitude as i128 + } else { + -(magnitude as i128) + }; + cases.push((unscaled, rng.random_range(-30_i8..=38))); + } + // Values around the fast-path limits, where a wrong bound would show up. + for _ in 0..10_000 { + let limit = if rng.random::() { + 1_i128 << 53 + } else { + 1 << 24 + }; + let unscaled = limit + rng.random_range(-1000_i128..=1000); + let unscaled = if rng.random::() { + unscaled + } else { + -unscaled + }; + cases.push((unscaled, rng.random_range(-23_i8..=23))); + } + // Decimal-looking values with few digits, all on the fast paths. + for _ in 0..10_000 { + let unscaled = rng.random_range(-10_000_000_i128..=10_000_000); + cases.push((unscaled, rng.random_range(0_i8..=10))); + } + + for (unscaled, scale) in cases { + // Rust's float parser rounds the exact decimal value once, like BigDecimal does. + let text = format!("{unscaled}e{}", -i32::from(scale)); + let expected_f64: f64 = text.parse().unwrap(); + let expected_f32: f32 = text.parse().unwrap(); + let actual_f64 = decimal128_to_f64(unscaled, scale); + let actual_f32 = decimal128_to_f32(unscaled, scale); + assert_eq!( + actual_f64.to_bits(), + expected_f64.to_bits(), + "{text}: got {actual_f64:?}, expected {expected_f64:?}" + ); + assert_eq!( + actual_f32.to_bits(), + expected_f32.to_bits(), + "{text}: got {actual_f32:?}, expected {expected_f32:?}" + ); + } + } + + #[test] + fn test_cast_decimal128_to_float_arrays_keep_nulls_in_every_eval_mode() { + use crate::conversion_funcs::cast::cast_array; + use crate::SparkCastOptions; + + let decimals_38_18: ArrayRef = Arc::new( + Decimal128Array::from(vec![ + Some(12345678900000000000000_i128), + None, + Some(-76543210000000000000000), + Some(0), + Some(577312285583388355022308), + None, + ]) + .with_precision_and_scale(38, 18) + .unwrap(), + ); + let decimals_38_10: ArrayRef = Arc::new( + Decimal128Array::from(vec![Some(167772170000000001_i128), None, Some(-123456789)]) + .with_precision_and_scale(38, 10) + .unwrap(), + ); + + for eval_mode in [EvalMode::Legacy, EvalMode::Ansi, EvalMode::Try] { + let options = SparkCastOptions::new(eval_mode, "UTC", false); + + let doubles = + cast_array(Arc::clone(&decimals_38_18), &DataType::Float64, &options).unwrap(); + let doubles = doubles.as_primitive::(); + assert_eq!(doubles.len(), 6); + assert_eq!(doubles.null_count(), 2); + assert_eq!(doubles.value(0), 12345.6789); + assert!(doubles.is_null(1)); + assert_eq!(doubles.value(2), -76543.21); + assert_eq!(doubles.value(3).to_bits(), 0.0_f64.to_bits()); + assert_eq!(doubles.value(4), 577312.2855833884); + assert!(doubles.is_null(5)); + + let floats = + cast_array(Arc::clone(&decimals_38_18), &DataType::Float32, &options).unwrap(); + let floats = floats.as_primitive::(); + assert_eq!(floats.len(), 6); + assert_eq!(floats.null_count(), 2); + assert_eq!(floats.value(0), 12345.679_f32); + assert!(floats.is_null(1)); + assert_eq!(floats.value(2), -76543.21_f32); + assert_eq!(floats.value(3).to_bits(), 0.0_f32.to_bits()); + assert_eq!(floats.value(4), 577312.3_f32); + assert!(floats.is_null(5)); + + let floats = + cast_array(Arc::clone(&decimals_38_10), &DataType::Float32, &options).unwrap(); + let floats = floats.as_primitive::(); + assert_eq!(floats.value(0), 16777218.0_f32); + assert!(floats.is_null(1)); + assert_eq!(floats.value(2), -0.012345679_f32); + + let doubles = + cast_array(Arc::clone(&decimals_38_10), &DataType::Float64, &options).unwrap(); + let doubles = doubles.as_primitive::(); + // 16777217.0000000001 rounds to the double 16777217.0 + assert_eq!(doubles.value(0), 16777217.0); + assert!(doubles.is_null(1)); + assert_eq!(doubles.value(2), -0.0123456789); + } + } + #[test] fn test_cast_float_to_decimal() { let a: ArrayRef = Arc::new(Float64Array::from(vec![ diff --git a/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala b/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala index 9bfa8f63774..3ed73619409 100644 --- a/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala @@ -751,6 +751,7 @@ class CometNativeCastSuite extends CometTestBase with AdaptiveSparkPlanHelper { BigDecimal("1.500000000000000000"), BigDecimal("123456789.123456789"))), DataTypes.FloatType) + castTest(generateDecimalsWithLargeUnscaledValues(), DataTypes.FloatType) } test("cast DecimalType(38,18) to DoubleType") { @@ -764,6 +765,7 @@ class CometNativeCastSuite extends CometTestBase with AdaptiveSparkPlanHelper { BigDecimal("1.500000000000000000"), BigDecimal("123456789.123456789"))), DataTypes.DoubleType) + castTest(generateDecimalsWithLargeUnscaledValues(), DataTypes.DoubleType) } test("cast DecimalType(38,18) to BooleanType") { @@ -783,6 +785,14 @@ class CometNativeCastSuite extends CometTestBase with AdaptiveSparkPlanHelper { castTest(generateDecimalsPrecision38Scale18(), DataTypes.StringType) } + test("cast DecimalType(38,10) to FloatType") { + castTest(generateDecimalsPrecision38Scale10(), DataTypes.FloatType) + } + + test("cast DecimalType(38,10) to DoubleType") { + castTest(generateDecimalsPrecision38Scale10(), DataTypes.DoubleType) + } + test("cast DecimalType with negative scale to StringType") { // Negative-scale decimals are a legacy Spark feature gated on // spark.sql.legacy.allowNegativeScaleOfDecimal=true. Spark LEGACY cast uses Java's @@ -1802,13 +1812,7 @@ class CometNativeCastSuite extends CometTestBase with AdaptiveSparkPlanHelper { FloatType, DoubleType, DecimalType(10, 2), - // DecimalType(38, 18) is excluded here: random data exposes a ~1 ULP difference between - // DataFusion's (i128 as f64) / 10^scale path and Spark's BigDecimal.doubleValue() for - // float/double casts; and extreme boundary values that would avoid the ULP issue overflow - // byte/short/int in ANSI mode, causing non-deterministic exception-message differences - // between Spark's row-at-a-time and Comet's vectorized execution. The individual scalar - // tests (cast DecimalType(38,18) to FloatType / DoubleType / BooleanType / etc.) already - // cover this type fully. + DecimalType(38, 18), DateType, TimestampType, DataTypes.TimestampNTZType, @@ -1930,8 +1934,7 @@ class CometNativeCastSuite extends CometTestBase with AdaptiveSparkPlanHelper { FloatType, DoubleType, DecimalType(10, 2), - // DecimalType(38, 18) is excluded for the same reason as the one-dimensional array - // matrix: decimal-to-float/double casts can differ by ~1 ULP from Spark. + DecimalType(38, 18), DateType, TimestampType, BinaryType) @@ -2248,6 +2251,38 @@ class CometNativeCastSuite extends CometTestBase with AdaptiveSparkPlanHelper { withNulls(values).toDF("a") } + // https://github.com/apache/datafusion-comet/issues/5670: with 18 fractional digits, every + // value of magnitude >= 0.01 has an unscaled representation above 2^53, so rounding the + // unscaled value to a double before dividing by 10^18 lands one ulp away from Spark's + // BigDecimal.doubleValue() / floatValue() for ordinary values such as these. + private def generateDecimalsWithLargeUnscaledValues(): DataFrame = { + generateDecimalsPrecision38Scale18( + Seq( + BigDecimal("12345.6789"), + BigDecimal("-12345.6789"), + BigDecimal("123456.789012"), + BigDecimal("76543.21"), + BigDecimal("-76543.21"), + BigDecimal("577312.285583388355022308"))) + } + + private def generateDecimalsPrecision38Scale10(): DataFrame = { + val values = Seq( + // https://github.com/apache/datafusion-comet/issues/5670: just above the midpoint of the + // floats 16777216 and 16777218. Its nearest double (16777217.0) is exactly that midpoint + // and narrows to 16777216, whereas BigDecimal.floatValue() rounds once, to 16777218. + BigDecimal("16777217.0000000001"), + BigDecimal("-16777217.0000000001"), + BigDecimal("16777216.9999999999"), + BigDecimal("12345.6789"), + BigDecimal("-12345.6789"), + // unscaled value above 2^53 + BigDecimal("1234567.8901234567"), + BigDecimal("0.0000000001"), + BigDecimal("0")) + withNulls(values).toDF("b").withColumn("a", col("b").cast(DecimalType(38, 10))).drop("b") + } + private def generateDateLiterals(): Seq[String] = { // add 1st, 10th, 20th of each month from epoch to 2027 val sampledDates = (1970 to 2027).flatMap { year =>