diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index 794c935b7..0e7433a00 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -128,6 +128,7 @@ set(PAIMON_COMMON_SRCS common/predicate/multi_literals_leaf_function.cpp common/predicate/not_equal.cpp common/predicate/not_in.cpp + common/predicate/null_false_leaf_binary_function.cpp common/predicate/or.cpp common/predicate/predicate.cpp common/predicate/predicate_builder.cpp @@ -627,6 +628,7 @@ if(PAIMON_BUILD_TESTS) common/predicate/literal_converter_test.cpp common/predicate/literal_test.cpp common/predicate/multi_literals_leaf_function_test.cpp + common/predicate/null_false_leaf_binary_function_test.cpp common/predicate/predicate_test.cpp common/predicate/predicate_utils_test.cpp common/predicate/predicate_validator_test.cpp diff --git a/src/paimon/common/predicate/null_false_leaf_binary_function.cpp b/src/paimon/common/predicate/null_false_leaf_binary_function.cpp new file mode 100644 index 000000000..a53c06101 --- /dev/null +++ b/src/paimon/common/predicate/null_false_leaf_binary_function.cpp @@ -0,0 +1,248 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/common/predicate/null_false_leaf_binary_function.h" + +#include +#include +#include +#include + +#include "arrow/array/array_base.h" +#include "arrow/array/array_primitive.h" +#include "arrow/compute/exec.h" +#include "arrow/datum.h" +#include "arrow/memory_pool.h" +#include "arrow/result.h" +#include "arrow/scalar.h" +#include "arrow/type.h" +#include "paimon/common/predicate/literal_converter.h" +#include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/common/utils/checked_cast.h" +#include "paimon/common/utils/field_type_utils.h" +#include "paimon/defs.h" +#include "paimon/status.h" + +namespace paimon { +namespace { +/// The `arrow::compute` kernel that compares a column against a scalar the way `Test(field, +/// literal)` compares two `Literal`s, or `nullptr` for a function that is not a comparison. +/// +/// `STARTS_WITH`, `ENDS_WITH`, `CONTAINS` and `LIKE` are `NullFalseLeafBinaryFunction`s too, but +/// they match a pattern instead of ordering two values, so they are not here and keep the row by +/// row path. +const char* ComparisonKernel(Function::Type type) { + switch (type) { + case Function::Type::EQUAL: + return "equal"; + case Function::Type::NOT_EQUAL: + return "not_equal"; + case Function::Type::LESS_THAN: + return "less"; + case Function::Type::LESS_OR_EQUAL: + return "less_equal"; + case Function::Type::GREATER_THAN: + return "greater"; + case Function::Type::GREATER_OR_EQUAL: + return "greater_equal"; + default: + return nullptr; + } +} + +/// The field types one comparison kernel can stand in for `Literal::CompareTo`. +/// +/// `FLOAT` and `DOUBLE` are not here: `FieldsComparator::CompareFloatingPoint` orders +/// `-0.0 < +0.0` and makes every NaN equal to every NaN, where a kernel follows IEEE 754 and says +/// `-0.0 == +0.0` and that no NaN compares to anything. Unlike a NaN literal, which `IN` keeps off +/// its own fast path by looking at the literals, that divergence is a property of the column, so no +/// literal can be checked for it and a float column keeps the row by row path. +bool CanCompareWithKernel(FieldType field_type) { + switch (field_type) { + case FieldType::BOOLEAN: + case FieldType::TINYINT: + case FieldType::SMALLINT: + case FieldType::INT: + case FieldType::BIGINT: + case FieldType::DATE: + case FieldType::STRING: + case FieldType::BINARY: + case FieldType::DECIMAL: + case FieldType::TIMESTAMP: + return true; + default: + return false; + } +} + +/// The `FieldType` the row by row path reads the values of `array` as, which is the one +/// `LiteralConverter::ConvertLiteralsFromArray` gives them, or `std::nullopt` for a layout that +/// conversion rejects and this therefore has to leave to it to report. +std::optional ArrayFieldType(const arrow::Array& array) { + const std::shared_ptr& type = array.type(); + if (type->id() == arrow::Type::DICTIONARY) { + const auto& dict_type = checked_cast(*type); + const bool is_string = dict_type.value_type()->id() == arrow::Type::STRING && + dict_type.index_type()->id() == arrow::Type::INT32; + const bool is_large_string = dict_type.value_type()->id() == arrow::Type::LARGE_STRING && + dict_type.index_type()->id() == arrow::Type::INT64; + if (is_string || is_large_string) { + return FieldType::STRING; + } + return std::nullopt; + } + Result field_type = FieldTypeUtils::ConvertToFieldType(type->id()); + if (!field_type.ok()) { + return std::nullopt; + } + return field_type.value(); +} + +/// Whether the column and the scalar the literal was written to can be compared without arrow +/// casting one side to a type that `Literal::CompareTo` does not compare by. +/// +/// A kernel resolves a type difference by casting, and a cast fails on a value the target type does +/// not keep, where `Decimal::CompareTo` rescales two decimals and compares them by value and +/// `Literal::CompareTo` compares two timestamps by the instant they name and merely finds the +/// answer. A decimal column of another scale than the literal and a timestamp column of another +/// unit or with a time zone therefore keep the row by row path. +/// +/// Every other difference is one arrow resolves losslessly: a kernel decodes a dictionary column to +/// its values, a string column of another width compares byte by byte either way, and a decimal +/// column of another precision carrying the scale of the literal is widened without losing a digit. +/// The two `FieldType`s are already guarded to be the same one, so no numeric column reaches a +/// kernel of another width. +bool AgreesWithScalar(const arrow::DataType& column_type, const arrow::DataType& scalar_type) { + switch (column_type.id()) { + case arrow::Type::DECIMAL128: + return scalar_type.id() == arrow::Type::DECIMAL128 && + checked_cast(column_type).scale() == + checked_cast(scalar_type).scale(); + case arrow::Type::TIMESTAMP: + return scalar_type.id() == arrow::Type::TIMESTAMP && + checked_cast(column_type).unit() == + checked_cast(scalar_type).unit() && + checked_cast(column_type).timezone().empty(); + default: + return true; + } +} + +/// Writes `literal` to the one element scalar a comparison kernel compares the column against. +/// +/// @param field_type The field type of the literal, which picks the arrow type it is written with. +/// @param column_type The arrow type of the column the predicate is evaluated against, which +/// settles whether a decimal or a timestamp scalar can be compared against it. +/// @param pool The pool the scalar is allocated from. +/// @return `nullptr` when the literal cannot be compared by a kernel, which covers a decimal column +/// of another scale and a timestamp column of another unit or with a time zone. This never +/// fails. +std::shared_ptr MakeComparisonScalar(const Literal& literal, FieldType field_type, + const arrow::DataType& column_type, + arrow::MemoryPool* pool) { + // A `NullFalseLeafBinaryFunction` compares one literal at a time, so there is exactly one to + // write. `ConvertLiteralsToArray` is what `IN` writes its value set with, and it settles the + // arrow type of the scalar the same way for both. + Result> one = + LiteralConverter::ConvertLiteralsToArray(field_type, {literal}, pool); + // A failure only says the scalar is not there, and the row by row path still is. + if (!one.ok()) { + return nullptr; + } + arrow::Result> scalar = one.value()->GetScalar(0); + if (!scalar.ok()) { + return nullptr; + } + const std::shared_ptr& value = scalar.ValueOrDie(); + if (!AgreesWithScalar(column_type, *value->type)) { + return nullptr; + } + return value; +} + +/// Compares every row of `array` against `literal` with the kernel `kernel_name`. +/// +/// @param pool The pool the boolean bitmap the kernel writes is allocated from. +/// @return One entry per row, with the null rows left at 0 because a comparison is false on null. +/// +/// The kernel resolves the comparison itself: it decodes a dictionary column and emits a null for a +/// null row. It fails when the two types have no common type at all, which only happens when the +/// field type disagrees with the column the predicate is evaluated against. +Result> ProbeComparison(const arrow::Array& array, const char* kernel_name, + const std::shared_ptr& literal, + arrow::MemoryPool* pool) { + arrow::compute::ExecContext exec_context(pool); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( + arrow::Datum matches, + arrow::compute::CallFunction(kernel_name, {arrow::Datum(array), arrow::Datum(literal)}, + /*options=*/nullptr, &exec_context)); + // `make_array` hands out a new `shared_ptr` that owns the array, so it has to be held for as + // long as the array is read. Binding a reference straight to what it points at would drop the + // last owner at the end of the statement and leave that reference dangling. + std::shared_ptr matches_array = matches.make_array(); + const auto& matched = checked_cast(*matches_array); + std::vector is_valid(matched.length(), 0); + for (int64_t i = 0; i < matched.length(); i++) { + // A kernel emits a null for a null row, and every `NullFalseLeafBinaryFunction` is false on + // one, so a null row is left at 0 rather than read as a value. + is_valid[i] = static_cast(!matched.IsNull(i) && matched.Value(i)); + } + return is_valid; +} +} // namespace + +Result> NullFalseLeafBinaryFunction::Test(const arrow::Array& array, + const std::vector& literals, + arrow::MemoryPool* pool) const { + if (literals.size() < LITERAL_LIMIT) { + return Status::Invalid("NullFalseLeafBinaryFunction needs single literal for field"); + } + std::vector is_valid(array.length(), false); + if (literals[0].IsNull()) { + return is_valid; + } + + const char* kernel_name = ComparisonKernel(GetType()); + if (kernel_name != nullptr) { + const std::optional field_type = ArrayFieldType(array); + // A literal typed differently from the column makes `Literal::CompareTo` fail, so it keeps + // the row by row path that reports it instead of reaching a kernel that would cast one side + // to the other and compare by chance. + if (field_type != std::nullopt && *field_type == literals[0].GetType() && + CanCompareWithKernel(*field_type)) { + std::shared_ptr scalar = + MakeComparisonScalar(literals[0], *field_type, *array.type(), pool); + if (scalar != nullptr) { + return ProbeComparison(array, kernel_name, scalar, pool); + } + } + } + + // Materializing the column into `Literal` objects costs one heap allocation and one hash of the + // value per row, so this only runs when no kernel can compare the column against the literal. + PAIMON_ASSIGN_OR_RAISE(std::vector array_values, + LiteralConverter::ConvertLiteralsFromArray(array, /*own_data=*/false)); + for (int64_t i = 0; i < array.length(); i++) { + if (!array.IsNull(i)) { + PAIMON_ASSIGN_OR_RAISE(is_valid[i], Test(array_values[i], literals[0])); + } + } + return is_valid; +} + +} // namespace paimon diff --git a/src/paimon/common/predicate/null_false_leaf_binary_function.h b/src/paimon/common/predicate/null_false_leaf_binary_function.h index b9b6850b3..43fdef7db 100644 --- a/src/paimon/common/predicate/null_false_leaf_binary_function.h +++ b/src/paimon/common/predicate/null_false_leaf_binary_function.h @@ -18,38 +18,27 @@ #pragma once +#include +#include #include -#include "arrow/array/array_nested.h" -#include "arrow/c/bridge.h" -#include "fmt/format.h" +#include "arrow/array/array_base.h" +#include "arrow/type_fwd.h" #include "paimon/common/predicate/leaf_function.h" -#include "paimon/common/predicate/literal_converter.h" -#include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/predicate/literal.h" +#include "paimon/result.h" #include "paimon/status.h" namespace paimon { class NullFalseLeafBinaryFunction : public LeafFunction { public: + /// Compares the whole batch against the literal with one `arrow::compute` comparison kernel + /// when the literal and the column allow it, and falls back to materializing every row into a + /// `Literal` and comparing one at a time otherwise. Every `LeafFunction` is a shared stateless + /// singleton, so the scalar the kernel compares against is built per batch; that costs `O(1)` + /// and buys an `O(rows)` compare with no `Literal` per row in between. Result> Test(const arrow::Array& array, const std::vector& literals, - arrow::MemoryPool* pool) const override { - if (literals.size() < LITERAL_LIMIT) { - return Status::Invalid("NullFalseLeafBinaryFunction needs single literal for field"); - } - std::vector is_valid(array.length(), false); - if (literals[0].IsNull()) { - return is_valid; - } - PAIMON_ASSIGN_OR_RAISE( - std::vector array_values, - LiteralConverter::ConvertLiteralsFromArray(array, /*own_data=*/false)); - for (int64_t i = 0; i < array.length(); i++) { - if (!array.IsNull(i)) { - PAIMON_ASSIGN_OR_RAISE(is_valid[i], Test(array_values[i], literals[0])); - } - } - return is_valid; - } + arrow::MemoryPool* pool) const override; Result Test(const Literal& value, const std::vector& literals) const override { if (literals.size() < LITERAL_LIMIT) { diff --git a/src/paimon/common/predicate/null_false_leaf_binary_function_test.cpp b/src/paimon/common/predicate/null_false_leaf_binary_function_test.cpp new file mode 100644 index 000000000..ff23eafd7 --- /dev/null +++ b/src/paimon/common/predicate/null_false_leaf_binary_function_test.cpp @@ -0,0 +1,460 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/common/predicate/null_false_leaf_binary_function.h" + +#include +#include +#include +#include +#include +#include + +#include "arrow/api.h" +#include "arrow/array/array_dict.h" +#include "arrow/ipc/json_simple.h" +#include "gtest/gtest.h" +#include "paimon/common/predicate/contains.h" +#include "paimon/common/predicate/equal.h" +#include "paimon/common/predicate/greater_or_equal.h" +#include "paimon/common/predicate/greater_than.h" +#include "paimon/common/predicate/less_or_equal.h" +#include "paimon/common/predicate/less_than.h" +#include "paimon/common/predicate/literal_converter.h" +#include "paimon/common/predicate/not_equal.h" +#include "paimon/common/predicate/starts_with.h" +#include "paimon/common/utils/decimal_utils.h" +#include "paimon/data/decimal.h" +#include "paimon/data/timestamp.h" +#include "paimon/defs.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { +class NullFalseLeafBinaryFunctionTest : public ::testing::Test { + public: + static Literal StringLiteral(const std::string& value) { + return Literal(FieldType::STRING, value.data(), value.size()); + } + + static Literal BinaryLiteral(const std::string& value) { + return Literal(FieldType::BINARY, value.data(), value.size()); + } + + // A decimal literal of `unscaled` as written by the digits of `precision` and `scale`. + static Literal DecimalLiteral(int32_t precision, int32_t scale, const std::string& unscaled) { + EXPECT_OK_AND_ASSIGN(Decimal::int128_t value, DecimalUtils::StrToInt128(unscaled)); + return Literal(Decimal(precision, scale, value)); + } + + // A timestamp literal of `millis` since the epoch and `nanos` within the millisecond. + static Literal TimestampLiteral(int64_t millis, int32_t nanos = 0) { + return Literal(Timestamp::FromEpochMillis(millis, nanos)); + } + + // Evaluates the whole batch and returns the per row result, asserting the call succeeded. Both + // go through `LeafFunction`, because the `Test` overloads a subclass declares hide the batch + // one. + static std::vector Eval(const LeafFunction& function, const Literal& literal, + const std::shared_ptr& array) { + EXPECT_OK_AND_ASSIGN(std::vector is_valid, + function.Test(*array, {literal}, arrow::default_memory_pool())); + return is_valid; + } + + static void AssertEvalFails(const LeafFunction& function, const std::vector& literals, + const std::shared_ptr& array) { + ASSERT_NOK(function.Test(*array, literals, arrow::default_memory_pool())); + } +}; + +TEST_F(NullFalseLeafBinaryFunctionTest, TestInt) { + auto array = arrow::ipc::internal::json::ArrayFromJSON(arrow::int32(), R"([0, 1, 2, 3, null])") + .ValueOrDie(); + ASSERT_EQ(Eval(Equal::Instance(), Literal(2), array), std::vector({0, 0, 1, 0, 0})); + ASSERT_EQ(Eval(NotEqual::Instance(), Literal(2), array), std::vector({1, 1, 0, 1, 0})); + ASSERT_EQ(Eval(LessThan::Instance(), Literal(2), array), std::vector({1, 1, 0, 0, 0})); + ASSERT_EQ(Eval(LessOrEqual::Instance(), Literal(2), array), std::vector({1, 1, 1, 0, 0})); + ASSERT_EQ(Eval(GreaterThan::Instance(), Literal(2), array), std::vector({0, 0, 0, 1, 0})); + ASSERT_EQ(Eval(GreaterOrEqual::Instance(), Literal(2), array), + std::vector({0, 0, 1, 1, 0})); +} + +TEST_F(NullFalseLeafBinaryFunctionTest, TestTinyIntSmallIntAndBigInt) { + auto tinyint_array = + arrow::ipc::internal::json::ArrayFromJSON(arrow::int8(), R"([-128, 0, 127, null])") + .ValueOrDie(); + ASSERT_EQ(Eval(LessThan::Instance(), Literal(int8_t{0}), tinyint_array), + std::vector({1, 0, 0, 0})); + ASSERT_EQ(Eval(GreaterThan::Instance(), Literal(int8_t{0}), tinyint_array), + std::vector({0, 0, 1, 0})); + + auto smallint_array = + arrow::ipc::internal::json::ArrayFromJSON(arrow::int16(), R"([-30000, 0, 30000, null])") + .ValueOrDie(); + ASSERT_EQ(Eval(Equal::Instance(), Literal(int16_t{-30000}), smallint_array), + std::vector({1, 0, 0, 0})); + + // The int64 boundaries have to survive the trip through the scalar the kernel compares against. + auto bigint_array = + arrow::ipc::internal::json::ArrayFromJSON( + arrow::int64(), R"([-9223372036854775808, 0, 9223372036854775807, null])") + .ValueOrDie(); + ASSERT_EQ( + Eval(LessOrEqual::Instance(), Literal(std::numeric_limits::min()), bigint_array), + std::vector({1, 0, 0, 0})); + ASSERT_EQ(Eval(GreaterOrEqual::Instance(), Literal(std::numeric_limits::max()), + bigint_array), + std::vector({0, 0, 1, 0})); +} + +TEST_F(NullFalseLeafBinaryFunctionTest, TestBoolean) { + auto array = + arrow::ipc::internal::json::ArrayFromJSON(arrow::boolean(), R"([true, false, null])") + .ValueOrDie(); + ASSERT_EQ(Eval(Equal::Instance(), Literal(true), array), std::vector({1, 0, 0})); + ASSERT_EQ(Eval(LessThan::Instance(), Literal(true), array), std::vector({0, 1, 0})); + ASSERT_EQ(Eval(GreaterThan::Instance(), Literal(false), array), std::vector({1, 0, 0})); +} + +TEST_F(NullFalseLeafBinaryFunctionTest, TestDate) { + auto array = arrow::ipc::internal::json::ArrayFromJSON(arrow::date32(), R"([1, 2, 3, null])") + .ValueOrDie(); + const Literal second{FieldType::DATE, 2}; + ASSERT_EQ(Eval(Equal::Instance(), second, array), std::vector({0, 1, 0, 0})); + ASSERT_EQ(Eval(LessThan::Instance(), second, array), std::vector({1, 0, 0, 0})); + ASSERT_EQ(Eval(GreaterOrEqual::Instance(), second, array), std::vector({0, 1, 1, 0})); +} + +TEST_F(NullFalseLeafBinaryFunctionTest, TestString) { + // `Literal::CompareTo` orders two strings by `std::string_view::compare`, which is the byte by + // byte order a kernel compares them in, a prefix included. + auto array = arrow::ipc::internal::json::ArrayFromJSON( + arrow::utf8(), R"(["apple", "", "app", "banana", null])") + .ValueOrDie(); + const Literal apple = StringLiteral("apple"); + ASSERT_EQ(Eval(Equal::Instance(), apple, array), std::vector({1, 0, 0, 0, 0})); + ASSERT_EQ(Eval(NotEqual::Instance(), apple, array), std::vector({0, 1, 1, 1, 0})); + ASSERT_EQ(Eval(LessThan::Instance(), apple, array), std::vector({0, 1, 1, 0, 0})); + ASSERT_EQ(Eval(LessOrEqual::Instance(), apple, array), std::vector({1, 1, 1, 0, 0})); + ASSERT_EQ(Eval(GreaterThan::Instance(), apple, array), std::vector({0, 0, 0, 1, 0})); + ASSERT_EQ(Eval(GreaterOrEqual::Instance(), apple, array), std::vector({1, 0, 0, 1, 0})); +} + +TEST_F(NullFalseLeafBinaryFunctionTest, TestBinary) { + // Binary values keep their embedded zero bytes, which a kernel compares as bytes and + // `std::string_view::compare` does too. The JSON reader cannot spell a value like that, so the + // column is built with a builder. + arrow::BinaryBuilder builder; + ASSERT_TRUE(builder.Append(std::string("\x00\x01", 2)).ok()); + ASSERT_TRUE(builder.Append(std::string("\x00", 1)).ok()); + ASSERT_TRUE(builder.Append("xyz").ok()); + ASSERT_TRUE(builder.AppendNull().ok()); + std::shared_ptr array; + ASSERT_TRUE(builder.Finish(&array).ok()); + + ASSERT_EQ(Eval(Equal::Instance(), BinaryLiteral(std::string("\x00\x01", 2)), array), + std::vector({1, 0, 0, 0})); + ASSERT_EQ(Eval(LessThan::Instance(), BinaryLiteral("xyz"), array), + std::vector({1, 1, 0, 0})); +} + +TEST_F(NullFalseLeafBinaryFunctionTest, TestDecimal) { + // A column carrying the scale of the literal is compared by the kernel, which compares two + // decimals of one scale by their unscaled value, exactly what `Decimal::CompareTo` finds. + auto array = arrow::ipc::internal::json::ArrayFromJSON(arrow::decimal128(10, 2), + R"(["1.00", "2.00", "3.00", null])") + .ValueOrDie(); + const Literal two = DecimalLiteral(10, 2, "200"); + ASSERT_EQ(Eval(Equal::Instance(), two, array), std::vector({0, 1, 0, 0})); + ASSERT_EQ(Eval(LessThan::Instance(), two, array), std::vector({1, 0, 0, 0})); + ASSERT_EQ(Eval(GreaterOrEqual::Instance(), two, array), std::vector({0, 1, 1, 0})); + + // Another precision carrying the same scale is widened without losing a digit, so the kernel + // still compares what `Decimal::CompareTo` compares. + ASSERT_EQ(Eval(Equal::Instance(), DecimalLiteral(20, 2, "200"), array), + std::vector({0, 1, 0, 0})); +} + +TEST_F(NullFalseLeafBinaryFunctionTest, TestDecimalOffTheKernelPath) { + // `Decimal::CompareTo` rescales two decimals and compares them by value, so a literal of + // another scale still orders against the same number. A kernel would cast one side to the + // other, which fails on a value that does not fit the other scale, so a column of another + // scale keeps the row by row path and the result stays the same either way. + auto array = arrow::ipc::internal::json::ArrayFromJSON(arrow::decimal128(10, 2), + R"(["1.00", "2.00", null])") + .ValueOrDie(); + const Literal one_of_scale_four = DecimalLiteral(20, 4, "10000"); + ASSERT_EQ(Eval(Equal::Instance(), one_of_scale_four, array), std::vector({1, 0, 0})); + ASSERT_EQ(Eval(GreaterThan::Instance(), one_of_scale_four, array), + std::vector({0, 1, 0})); + + // Casting between those scales is what the row by row path spares: the literal does not fit the + // scale of the column, which would make a kernel report an error where comparing by value + // merely orders the two. + auto lossy_array = + arrow::ipc::internal::json::ArrayFromJSON(arrow::decimal128(38, 2), R"(["1.01", "2.00"])") + .ValueOrDie(); + const Literal huge = DecimalLiteral(38, 0, "99999999999999999999999999999999999999"); + ASSERT_EQ(Eval(LessThan::Instance(), huge, lossy_array), std::vector({1, 1})); + ASSERT_EQ(Eval(GreaterThan::Instance(), huge, lossy_array), std::vector({0, 0})); +} + +TEST_F(NullFalseLeafBinaryFunctionTest, TestTimestamp) { + // A column of the unit the literal needs is compared by the kernel, both sides naming the very + // same instant without a cast in between. + auto array = arrow::ipc::internal::json::ArrayFromJSON(arrow::timestamp(arrow::TimeUnit::MILLI), + R"([1000, 2000, 3000, null])") + .ValueOrDie(); + const Literal two_seconds = TimestampLiteral(2000); + ASSERT_EQ(Eval(Equal::Instance(), two_seconds, array), std::vector({0, 1, 0, 0})); + ASSERT_EQ(Eval(LessThan::Instance(), two_seconds, array), std::vector({1, 0, 0, 0})); + ASSERT_EQ(Eval(GreaterOrEqual::Instance(), two_seconds, array), + std::vector({0, 1, 1, 0})); + + // A literal that needs microseconds is compared against a column of microseconds. + auto micro_array = arrow::ipc::internal::json::ArrayFromJSON( + arrow::timestamp(arrow::TimeUnit::MICRO), R"([1500000, 2000000, null])") + .ValueOrDie(); + ASSERT_EQ(Eval(LessThan::Instance(), TimestampLiteral(2000), micro_array), + std::vector({1, 0, 0})); +} + +TEST_F(NullFalseLeafBinaryFunctionTest, TestTimestampOffTheKernelPath) { + // A column of a coarser unit than the literal needs keeps the row by row path, because a kernel + // would cast the literal to the column unit and fail on a value the unit does not keep. Every + // value of the coarser column is earlier than one needing the finer unit, so comparing by the + // instant merely orders the two. + auto milli_array = arrow::ipc::internal::json::ArrayFromJSON( + arrow::timestamp(arrow::TimeUnit::MILLI), R"([1000, 2000, null])") + .ValueOrDie(); + const Literal one_and_a_half = TimestampLiteral(1, 500000); + ASSERT_EQ(Eval(GreaterThan::Instance(), one_and_a_half, milli_array), + std::vector({1, 1, 0})); + ASSERT_EQ(Eval(LessThan::Instance(), one_and_a_half, milli_array), + std::vector({0, 0, 0})); + + // A column of a finer unit as well, casting the column to the unit of the literal can overflow + // int64 where comparing by the instant still orders the same two instants. + auto nano_array = arrow::ipc::internal::json::ArrayFromJSON( + arrow::timestamp(arrow::TimeUnit::NANO), R"([1000000000, null])") + .ValueOrDie(); + ASSERT_EQ(Eval(Equal::Instance(), TimestampLiteral(1000), nano_array), + std::vector({1, 0})); + + // A column with a time zone too: a kernel refuses to compare a zoned timestamp against an + // unzoned scalar, so the row by row path compares the instants the values name instead. + auto zoned_array = arrow::ipc::internal::json::ArrayFromJSON( + arrow::timestamp(arrow::TimeUnit::MILLI, "UTC"), R"([1000, 2000, null])") + .ValueOrDie(); + ASSERT_EQ(Eval(Equal::Instance(), TimestampLiteral(1000), zoned_array), + std::vector({1, 0, 0})); + ASSERT_EQ(Eval(GreaterThan::Instance(), TimestampLiteral(1000), zoned_array), + std::vector({0, 1, 0})); +} + +TEST_F(NullFalseLeafBinaryFunctionTest, TestFloatAndDoubleStayOffTheKernelPath) { + // `FieldsComparator::CompareFloatingPoint` orders `-0.0 < +0.0` and makes every NaN equal to + // every NaN and greater than every other value, where a kernel follows IEEE 754 and says + // `-0.0 == +0.0` and that no NaN compares to anything. That divergence is a property of the + // column rather than of a literal one could look for, so a float column keeps the row by row + // path. The JSON reader cannot spell a sign flipped NaN, so the column is built with a builder. + const double canonical_nan = std::numeric_limits::quiet_NaN(); + arrow::DoubleBuilder builder; + ASSERT_TRUE(builder.Append(-0.0).ok()); + ASSERT_TRUE(builder.Append(0.0).ok()); + ASSERT_TRUE(builder.Append(canonical_nan).ok()); + ASSERT_TRUE(builder.Append(-canonical_nan).ok()); + ASSERT_TRUE(builder.Append(1.0).ok()); + ASSERT_TRUE(builder.AppendNull().ok()); + std::shared_ptr array; + ASSERT_TRUE(builder.Finish(&array).ok()); + + ASSERT_EQ(Eval(Equal::Instance(), Literal(0.0), array), std::vector({0, 1, 0, 0, 0, 0})); + ASSERT_EQ(Eval(LessThan::Instance(), Literal(0.0), array), + std::vector({1, 0, 0, 0, 0, 0})); + ASSERT_EQ(Eval(GreaterThan::Instance(), Literal(0.0), array), + std::vector({0, 0, 1, 1, 1, 0})); + + // Every NaN equals every NaN and is greater than every other value. + ASSERT_EQ(Eval(Equal::Instance(), Literal(canonical_nan), array), + std::vector({0, 0, 1, 1, 0, 0})); + ASSERT_EQ(Eval(GreaterThan::Instance(), Literal(canonical_nan), array), + std::vector({0, 0, 0, 0, 0, 0})); + ASSERT_EQ(Eval(LessThan::Instance(), Literal(canonical_nan), array), + std::vector({1, 1, 0, 0, 1, 0})); + + auto float_array = + arrow::ipc::internal::json::ArrayFromJSON(arrow::float32(), R"([1.5, 2.5, -0.0, null])") + .ValueOrDie(); + ASSERT_EQ(Eval(LessThan::Instance(), Literal(2.5f), float_array), + std::vector({1, 0, 1, 0})); + ASSERT_EQ(Eval(Equal::Instance(), Literal(0.0f), float_array), std::vector({0, 0, 0, 0})); +} + +TEST_F(NullFalseLeafBinaryFunctionTest, TestDictionaryString) { + auto dictionary = + arrow::ipc::internal::json::ArrayFromJSON(arrow::utf8(), R"(["a", "b", "c"])").ValueOrDie(); + auto indices = + arrow::ipc::internal::json::ArrayFromJSON(arrow::int32(), R"([0, 1, 2, 2, null, 0])") + .ValueOrDie(); + std::shared_ptr array = + arrow::DictionaryArray::FromArrays(arrow::dictionary(arrow::int32(), arrow::utf8()), + indices, dictionary) + .ValueOrDie(); + + const Literal b = StringLiteral("b"); + ASSERT_EQ(Eval(Equal::Instance(), b, array), std::vector({0, 1, 0, 0, 0, 0})); + ASSERT_EQ(Eval(LessThan::Instance(), b, array), std::vector({1, 0, 0, 0, 0, 1})); + ASSERT_EQ(Eval(GreaterOrEqual::Instance(), b, array), std::vector({0, 1, 1, 1, 0, 0})); +} + +TEST_F(NullFalseLeafBinaryFunctionTest, TestLargeStringDictionary) { + auto dictionary = + arrow::ipc::internal::json::ArrayFromJSON(arrow::large_utf8(), R"(["a", "b", "c"])") + .ValueOrDie(); + auto indices = arrow::ipc::internal::json::ArrayFromJSON(arrow::int64(), R"([0, 2, null, 1])") + .ValueOrDie(); + std::shared_ptr array = + arrow::DictionaryArray::FromArrays(arrow::dictionary(arrow::int64(), arrow::large_utf8()), + indices, dictionary) + .ValueOrDie(); + + ASSERT_EQ(Eval(Equal::Instance(), StringLiteral("c"), array), std::vector({0, 1, 0, 0})); + ASSERT_EQ(Eval(GreaterThan::Instance(), StringLiteral("b"), array), + std::vector({0, 1, 0, 0})); +} + +TEST_F(NullFalseLeafBinaryFunctionTest, TestDictionaryWithNullValue) { + // A kernel decodes the dictionary, so a row pointing at a null dictionary value becomes a null + // row and is false for every comparison. The row by row path reads the slot the index names + // instead, which holds no value, and an empty literal used to match it. An empty literal must + // not. + auto dictionary = + arrow::ipc::internal::json::ArrayFromJSON(arrow::utf8(), R"([null, "a"])").ValueOrDie(); + auto indices = arrow::ipc::internal::json::ArrayFromJSON(arrow::int32(), R"([0, 1, null, 0])") + .ValueOrDie(); + std::shared_ptr array = + arrow::DictionaryArray::FromArrays(arrow::dictionary(arrow::int32(), arrow::utf8()), + indices, dictionary) + .ValueOrDie(); + + ASSERT_EQ(Eval(Equal::Instance(), StringLiteral(""), array), std::vector({0, 0, 0, 0})); + ASSERT_EQ(Eval(Equal::Instance(), StringLiteral("a"), array), std::vector({0, 1, 0, 0})); + ASSERT_EQ(Eval(NotEqual::Instance(), StringLiteral("a"), array), + std::vector({0, 0, 0, 0})); +} + +TEST_F(NullFalseLeafBinaryFunctionTest, TestNullLiteralIsFalseForEveryRow) { + auto array = + arrow::ipc::internal::json::ArrayFromJSON(arrow::int32(), R"([0, 1, null])").ValueOrDie(); + const Literal null_int{FieldType::INT}; + ASSERT_EQ(Eval(Equal::Instance(), null_int, array), std::vector({0, 0, 0})); + ASSERT_EQ(Eval(NotEqual::Instance(), null_int, array), std::vector({0, 0, 0})); + ASSERT_EQ(Eval(LessThan::Instance(), null_int, array), std::vector({0, 0, 0})); + + const Literal null_string{FieldType::STRING}; + auto string_array = + arrow::ipc::internal::json::ArrayFromJSON(arrow::utf8(), R"(["a", null])").ValueOrDie(); + ASSERT_EQ(Eval(GreaterThan::Instance(), null_string, string_array), std::vector({0, 0})); +} + +TEST_F(NullFalseLeafBinaryFunctionTest, TestEmptyLiteralsReportTheError) { + auto array = + arrow::ipc::internal::json::ArrayFromJSON(arrow::int32(), R"([0, 1])").ValueOrDie(); + AssertEvalFails(Equal::Instance(), {}, array); + AssertEvalFails(LessThan::Instance(), {}, array); +} + +TEST_F(NullFalseLeafBinaryFunctionTest, TestMismatchedLiteralTypeReportsTheError) { + // A literal typed differently from the column makes `Literal::CompareTo` fail, so it has to + // keep reporting the error instead of reaching a kernel that would cast one side to the other + // and compare by chance. + auto int_array = + arrow::ipc::internal::json::ArrayFromJSON(arrow::int32(), R"([0])").ValueOrDie(); + AssertEvalFails(Equal::Instance(), {Literal(int64_t{0})}, int_array); + AssertEvalFails(LessThan::Instance(), {Literal(int64_t{0})}, int_array); + + auto string_array = + arrow::ipc::internal::json::ArrayFromJSON(arrow::utf8(), R"(["1"])").ValueOrDie(); + AssertEvalFails(Equal::Instance(), {Literal(1)}, string_array); + + // A string literal against a binary column is a mismatch too, the two are different field types + // even though both are bytes. + auto binary_array = + arrow::ipc::internal::json::ArrayFromJSON(arrow::binary(), R"(["1"])").ValueOrDie(); + AssertEvalFails(Equal::Instance(), {StringLiteral("1")}, binary_array); + ASSERT_EQ(Eval(Equal::Instance(), BinaryLiteral("1"), binary_array), std::vector({1})); +} + +TEST_F(NullFalseLeafBinaryFunctionTest, TestLayoutsLiteralConversionRejects) { + // A layout `LiteralConverter::ConvertLiteralsFromArray` rejects keeps reporting through the row + // by row path, because no `FieldType` says what a kernel would have to compare it as. + auto large_string_array = + arrow::ipc::internal::json::ArrayFromJSON(arrow::large_utf8(), R"(["a"])").ValueOrDie(); + ASSERT_NOK(LiteralConverter::ConvertLiteralsFromArray(*large_string_array, /*own_data=*/false)); + AssertEvalFails(Equal::Instance(), {StringLiteral("a")}, large_string_array); + + // A dictionary of anything but the two string layouts is rejected as well, even though a kernel + // would decode it, so that the row by row path stays the one that says no. + auto int64_dictionary = + arrow::ipc::internal::json::ArrayFromJSON(arrow::int64(), R"([10, 20])").ValueOrDie(); + auto int32_indices = + arrow::ipc::internal::json::ArrayFromJSON(arrow::int32(), R"([0, 1])").ValueOrDie(); + std::shared_ptr int64_dict_array = + arrow::DictionaryArray::FromArrays(arrow::dictionary(arrow::int32(), arrow::int64()), + int32_indices, int64_dictionary) + .ValueOrDie(); + ASSERT_NOK(LiteralConverter::ConvertLiteralsFromArray(*int64_dict_array, /*own_data=*/false)); + AssertEvalFails(Equal::Instance(), {Literal(int64_t{10})}, int64_dict_array); +} + +TEST_F(NullFalseLeafBinaryFunctionTest, TestStringPatternFunctionsKeepTheRowByRowPath) { + // `STARTS_WITH`, `ENDS_WITH`, `CONTAINS` and `LIKE` are `NullFalseLeafBinaryFunction`s too, but + // they match a pattern instead of ordering two values, so no kernel stands in for them. + auto array = arrow::ipc::internal::json::ArrayFromJSON(arrow::utf8(), + R"(["apple", "pineapple", "app", null])") + .ValueOrDie(); + ASSERT_EQ(Eval(StartsWith::Instance(), StringLiteral("app"), array), + std::vector({1, 0, 1, 0})); + ASSERT_EQ(Eval(Contains::Instance(), StringLiteral("apple"), array), + std::vector({1, 1, 0, 0})); +} + +TEST_F(NullFalseLeafBinaryFunctionTest, TestSlicedArray) { + auto array = + arrow::ipc::internal::json::ArrayFromJSON(arrow::int32(), R"([1, 2, 3, 4, 5, null])") + .ValueOrDie(); + auto sliced = array->Slice(2, 4); + ASSERT_EQ(Eval(LessThan::Instance(), Literal(5), sliced), std::vector({1, 1, 0, 0})); + ASSERT_EQ(Eval(Equal::Instance(), Literal(4), sliced), std::vector({0, 1, 0, 0})); + + auto string_array = + arrow::ipc::internal::json::ArrayFromJSON(arrow::utf8(), R"(["a", "b", "c", "d"])") + .ValueOrDie(); + auto sliced_string = string_array->Slice(1, 3); + ASSERT_EQ(Eval(GreaterThan::Instance(), StringLiteral("b"), sliced_string), + std::vector({0, 1, 1})); + ASSERT_EQ(Eval(LessThan::Instance(), StringLiteral("d"), sliced_string), + std::vector({1, 1, 0})); +} + +TEST_F(NullFalseLeafBinaryFunctionTest, TestEmptyArray) { + auto array = arrow::ipc::internal::json::ArrayFromJSON(arrow::utf8(), R"([])").ValueOrDie(); + ASSERT_EQ(Eval(Equal::Instance(), StringLiteral("a"), array), std::vector({})); +} +} // namespace paimon::test