diff --git a/be/src/exprs/function/function_timezone_hour_minute.cpp b/be/src/exprs/function/function_timezone_hour_minute.cpp new file mode 100644 index 00000000000000..049365f14f4259 --- /dev/null +++ b/be/src/exprs/function/function_timezone_hour_minute.cpp @@ -0,0 +1,162 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include +#include +#include +#include + +#include "common/status.h" +#include "core/assert_cast.h" +#include "core/block/block.h" +#include "core/block/column_numbers.h" +#include "core/column/column.h" +#include "core/column/column_const.h" +#include "core/column/column_nullable.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_timestamptz.h" +#include "core/data_type/primitive_type.h" +#include "core/value/timestamptz_value.h" +#include "exprs/function_context.h" +#include "exprs/function/function.h" +#include "exprs/function/simple_function_factory.h" +#include "runtime/runtime_state.h" + +namespace doris { + +namespace { +constexpr int64_t SECONDS_PER_HOUR = 3600; +constexpr int64_t SECONDS_PER_MINUTE = 60; + +// TIMESTAMPTZ values are stored as UTC instants without the input zone, so the +// offset extracted here is the offset of the session time zone at the instant. +// See TimestampTzValue for the storage design. +Status execute_timezone_offset_part(FunctionContext* context, Block& block, + const ColumnNumbers& arguments, uint32_t result, + size_t input_rows_count, bool extract_hour) { + ColumnPtr col = block.get_by_position(arguments[0]).column; + // Fast path: the framework constant path is disabled + // (use_default_implementation_for_constants() == false) so a constant + // argument must not be expanded to input_rows_count rows and re-evaluated + // per row. Compute the single value once and keep the block-local const + // shape. The result type must be non-nullable here: the framework's + // default null handling wraps the result into a ColumnNullable, which a + // ColumnConst cannot be nested in. + if (is_column_const(*col) && !block.get_by_position(result).type->is_nullable()) { + const auto& const_col = assert_cast(*col); + const auto& tz_column = + assert_cast(*const_col.get_data_column_ptr()); + int64_t offset = tz_column.get_data()[0].utc_offset(context->state()->timezone_obj()); + int64_t value = extract_hour ? offset / SECONDS_PER_HOUR + : (offset % SECONDS_PER_HOUR) / SECONDS_PER_MINUTE; + auto nested = ColumnInt64::create(); + nested->insert_value(value); + block.get_by_position(result).column = + ColumnConst::create(std::move(nested), input_rows_count); + return Status::OK(); + } + // Unwrap nullable and const wrappers in any nesting order so that + // ColumnNullable(ColumnConst(...)) and ColumnConst(ColumnNullable(...)) + // inputs both reach the plain ColumnTimeStampTz data below. + col = remove_nullable(col); + if (is_column_const(*col)) { + col = assert_cast(*col).convert_to_full_column(); + col = remove_nullable(col); + } + const auto* tz_column = assert_cast(col.get()); + const auto& tz_data = tz_column->get_data(); + + auto result_column = ColumnInt64::create(); + auto& result_data = result_column->get_data(); + result_data.resize(input_rows_count); + + const cctz::time_zone& timezone = context->state()->timezone_obj(); + for (size_t i = 0; i < input_rows_count; ++i) { + int64_t offset = tz_data[i].utc_offset(timezone); + result_data[i] = extract_hour ? offset / SECONDS_PER_HOUR + : (offset % SECONDS_PER_HOUR) / SECONDS_PER_MINUTE; + } + + block.get_by_position(result).column = std::move(result_column); + return Status::OK(); +} +} // namespace + +class FunctionTimezoneHour : public IFunction { +public: + static constexpr auto name = "timezone_hour"; + + static FunctionPtr create() { return std::make_shared(); } + + String get_name() const override { return name; } + + size_t get_number_of_arguments() const override { return 1; } + + DataTypePtr get_return_type_impl(const DataTypes& arguments) const override { + return std::make_shared(); + } + + // The result depends on the session time_zone, so a constant result must + // never be cached: the point-query short-circuit executor opens output + // expressions with the default timezone and later reuses cached constant + // columns without re-evaluating them (VectorizedFnCall::is_constant + // consults this flag). Disable it like other nondeterministic functions + // (e.g. random, uuid). + bool use_default_implementation_for_constants() const override { return false; } + + Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments, + uint32_t result, size_t input_rows_count) const override { + return execute_timezone_offset_part(context, block, arguments, result, input_rows_count, + true); + } +}; + +class FunctionTimezoneMinute : public IFunction { +public: + static constexpr auto name = "timezone_minute"; + + static FunctionPtr create() { return std::make_shared(); } + + String get_name() const override { return name; } + + size_t get_number_of_arguments() const override { return 1; } + + DataTypePtr get_return_type_impl(const DataTypes& arguments) const override { + return std::make_shared(); + } + + // See FunctionTimezoneHour::use_default_implementation_for_constants. + bool use_default_implementation_for_constants() const override { return false; } + + Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments, + uint32_t result, size_t input_rows_count) const override { + return execute_timezone_offset_part(context, block, arguments, result, input_rows_count, + false); + } +}; + +void register_function_timezone_hour_minute(SimpleFunctionFactory& factory) { + factory.register_function(); + factory.register_function(); +} + +} // namespace doris diff --git a/be/src/exprs/function/simple_function_factory.h b/be/src/exprs/function/simple_function_factory.h index 5dd09e0847dba5..52b5a27b9ac98e 100644 --- a/be/src/exprs/function/simple_function_factory.h +++ b/be/src/exprs/function/simple_function_factory.h @@ -87,6 +87,7 @@ void register_function_uuid_transforms(SimpleFunctionFactory& factory); void register_function_grouping(SimpleFunctionFactory& factory); void register_function_datetime_floor_ceil(SimpleFunctionFactory& factory); void register_function_convert_tz(SimpleFunctionFactory& factory); +void register_function_timezone_hour_minute(SimpleFunctionFactory& factory); void register_function_least_greast(SimpleFunctionFactory& factory); void register_function_fake(SimpleFunctionFactory& factory); void register_function_array(SimpleFunctionFactory& factory); @@ -332,6 +333,7 @@ class SimpleFunctionFactory { register_function_grouping(instance); register_function_datetime_floor_ceil(instance); register_function_convert_tz(instance); + register_function_timezone_hour_minute(instance); register_function_least_greast(instance); register_function_fake(instance); register_function_encryption(instance); diff --git a/be/src/exprs/vectorized_fn_call.cpp b/be/src/exprs/vectorized_fn_call.cpp index ecbeeedaddbe07..58b5234ba57ea7 100644 --- a/be/src/exprs/vectorized_fn_call.cpp +++ b/be/src/exprs/vectorized_fn_call.cpp @@ -666,7 +666,9 @@ bool VectorizedFnCall::can_push_down_to_index() const { bool VectorizedFnCall::is_deterministic() const { static const std::set NON_DETERMINISTIC_FUNCTIONS = { - "random", "rand", "random_bytes", "uuid", "uuid_numeric"}; + "random", "rand", "random_bytes", "uuid", "uuid_numeric", + // timezone_hour/timezone_minute depend on the session time_zone. + "timezone_hour", "timezone_minute"}; return !NON_DETERMINISTIC_FUNCTIONS.contains(_function_name) && VExpr::is_deterministic(); } diff --git a/be/src/service/point_query_executor.cpp b/be/src/service/point_query_executor.cpp index cee41c611e0ef5..51672948e7710b 100644 --- a/be/src/service/point_query_executor.cpp +++ b/be/src/service/point_query_executor.cpp @@ -130,9 +130,16 @@ static void extract_slot_ref(const VExprSPtr& expr, TupleDescriptor* tuple_desc, Status Reusable::init(const TDescriptorTable& t_desc_tbl, const std::vector& output_exprs, const TQueryOptions& query_options, const TabletSchema& schema, - size_t block_size) { + size_t block_size, const std::string& time_zone) { _runtime_state = RuntimeState::create_unique(); _runtime_state->set_query_options(query_options); + // Install the request's session timezone before the expressions are + // opened: VExpr::open() evaluates constant children (e.g. a const + // VCastExpr) with the runtime state's timezone, which would otherwise be + // the default +08:00 even though the request may use a different one. + if (!time_zone.empty()) { + _runtime_state->set_timezone(time_zone); + } RETURN_IF_ERROR(DescriptorTbl::create(_runtime_state->obj_pool(), t_desc_tbl, &_desc_tbl)); _runtime_state->set_desc_tbl(_desc_tbl); for (const auto* slot : tuple_desc()->slots()) { @@ -306,6 +313,12 @@ Status PointQueryExecutor::init(const PTabletKeyLookupRequest* request, auto cache_handle = LookupConnectionCache::instance()->get(uuid); _binary_row_format = request->is_binary_row(); _tablet = DORIS_TRY(ExecEnv::get_tablet(request->tablet_id())); + // Timezone of the session that sent the request. It must be installed on + // the reusable's runtime state before the output expressions are opened + // so that constant children are evaluated with it (see Reusable::init). + std::string request_time_zone = + (request->has_time_zone() && !request->time_zone().empty()) ? request->time_zone() + : std::string(); if (cache_handle != nullptr) { _reusable = cache_handle; _profile_metrics.hit_lookup_cache = true; @@ -352,18 +365,18 @@ Status PointQueryExecutor::init(const PTabletKeyLookupRequest* request, if (uuid != 0) { // could be reused by requests after, pre allocte more blocks RETURN_IF_ERROR(reusable_ptr->init(t_desc_tbl, t_output_exprs.exprs, t_query_options, - *_tablet->tablet_schema(), - s_preallocted_blocks_num)); + *_tablet->tablet_schema(), s_preallocted_blocks_num, + request_time_zone)); LookupConnectionCache::instance()->add(uuid, reusable_ptr); } else { RETURN_IF_ERROR(reusable_ptr->init(t_desc_tbl, t_output_exprs.exprs, t_query_options, - *_tablet->tablet_schema(), 1)); + *_tablet->tablet_schema(), 1, request_time_zone)); } } _init_remote_scan_cache_write_limiter(); // Set timezone from request for functions like from_unixtime() - if (request->has_time_zone() && !request->time_zone().empty()) { - _reusable->runtime_state()->set_timezone(request->time_zone()); + if (!request_time_zone.empty()) { + _reusable->runtime_state()->set_timezone(request_time_zone); } if (request->has_version() && request->version() >= 0) { _version = request->version(); diff --git a/be/src/service/point_query_executor.h b/be/src/service/point_query_executor.h index 91a8632b00ac9f..965ee15596b8c1 100644 --- a/be/src/service/point_query_executor.h +++ b/be/src/service/point_query_executor.h @@ -73,7 +73,7 @@ class Reusable { Status init(const TDescriptorTable& t_desc_tbl, const std::vector& output_exprs, const TQueryOptions& query_options, const TabletSchema& schema, - size_t block_size = 1); + size_t block_size = 1, const std::string& time_zone = ""); std::unique_ptr get_block(); diff --git a/be/test/exprs/function/function_timezone_hour_minute_test.cpp b/be/test/exprs/function/function_timezone_hour_minute_test.cpp new file mode 100644 index 00000000000000..80fc61b227e425 --- /dev/null +++ b/be/test/exprs/function/function_timezone_hour_minute_test.cpp @@ -0,0 +1,272 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include + +#include +#include +#include + +#include "core/assert_cast.h" +#include "core/block/block.h" +#include "core/block/column_numbers.h" +#include "core/column/column.h" +#include "core/column/column_nullable.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_timestamptz.h" +#include "core/data_type/primitive_type.h" +#include "exprs/function/function.h" +#include "exprs/function/simple_function_factory.h" +#include "exprs/vexpr.h" +#include "exprs/vexpr_context.h" +#include "gen_cpp/Exprs_types.h" +#include "gen_cpp/Types_types.h" +#include "runtime/descriptors.h" +#include "testutil/column_helper.h" +#include "testutil/datetime_ut_util.h" +#include "testutil/mock/mock_runtime_state.h" +#include "util/timezone_utils.h" + +namespace doris { + +class FunctionTimezoneHourMinuteTest : public testing::Test { +public: + void SetUp() override { + TimezoneUtils::load_offsets_to_cache(); + TimezoneUtils::load_timezones_to_cache(); + context._state = &_state; + arguments = {0}; + result = 1; + } + + void set_session_timezone(const cctz::time_zone& tz) { _state._timezone_obj = tz; } + + void check_result(const std::string& func_name, const Block& block, + const std::vector& expected) { + auto return_type = std::make_shared(); + FunctionBasePtr func = SimpleFunctionFactory::instance().get_function( + func_name, block.get_columns_with_type_and_name(), return_type); + ASSERT_NE(func, nullptr); + Block input_block = block; + input_block.insert({nullptr, return_type, "result"}); + auto st = func->execute(&context, input_block, arguments, result, input_block.rows()); + ASSERT_TRUE(st.ok()) << st.to_string(); + // Constant input may produce a const result column; materialize it + // before inspecting elements. + auto result_col = input_block.get_by_position(result).column->convert_to_full_column_if_const(); + const auto& col = assert_cast(*result_col); + ASSERT_EQ(col.size(), expected.size()); + for (size_t i = 0; i < expected.size(); ++i) { + EXPECT_EQ(col.get_element(i), expected[i]) << "at row " << i; + } + } + + MockRuntimeState _state; + FunctionContext context; + ColumnNumbers arguments; + uint32_t result; +}; + +TEST_F(FunctionTimezoneHourMinuteTest, fixed_offset_shanghai) { + // Asia/Shanghai has a fixed UTC+08:00 offset without DST, so the offset + // part of the session timezone is the same for every instant. + set_session_timezone(cctz::fixed_time_zone(std::chrono::hours(8))); + + auto block = ColumnHelper::create_block( + {make_timestamptz(2024, 1, 15, 12, 0, 0, 0), + make_timestamptz(2024, 7, 15, 12, 0, 0, 0)}); + + check_result("timezone_hour", block, {8, 8}); + check_result("timezone_minute", block, {0, 0}); +} + +TEST_F(FunctionTimezoneHourMinuteTest, dst_new_york) { + // America/New_York switches between EST (UTC-05:00) in winter and + // EDT (UTC-04:00) in summer, which is reflected in the returned offset. + cctz::time_zone tz; + ASSERT_TRUE(TimezoneUtils::find_cctz_time_zone("America/New_York", tz)); + set_session_timezone(tz); + + auto winter_block = ColumnHelper::create_block( + {make_timestamptz(2024, 1, 15, 12, 0, 0, 0)}); + auto summer_block = ColumnHelper::create_block( + {make_timestamptz(2024, 7, 15, 12, 0, 0, 0)}); + + check_result("timezone_hour", winter_block, {-5}); + check_result("timezone_minute", winter_block, {0}); + check_result("timezone_hour", summer_block, {-4}); + check_result("timezone_minute", summer_block, {0}); +} + +TEST_F(FunctionTimezoneHourMinuteTest, fractional_offsets) { + // Trino returns truncated integer values for fractional offsets: + // timezone_hour(UTC-04:30) = -4 and timezone_minute(UTC-04:30) = -30. + set_session_timezone(cctz::fixed_time_zone(std::chrono::seconds(-4 * 3600 - 30 * 60))); + auto block = ColumnHelper::create_block( + {make_timestamptz(2024, 6, 20, 12, 0, 0, 0)}); + check_result("timezone_hour", block, {-4}); + check_result("timezone_minute", block, {-30}); + + // Nepal Standard Time (UTC+05:45). + set_session_timezone(cctz::fixed_time_zone(std::chrono::seconds(5 * 3600 + 45 * 60))); + check_result("timezone_hour", block, {5}); + check_result("timezone_minute", block, {45}); +} + +TEST_F(FunctionTimezoneHourMinuteTest, const_input) { + // TIMESTAMPTZ stores a UTC instant without the input zone; even when the + // value was produced by CAST with an explicit zone (here '2024-01-15 + // 12:00:00-04:30'), the extracted offset is the session zone's offset. + set_session_timezone(cctz::fixed_time_zone(std::chrono::hours(8))); + + auto inner = ColumnTimeStampTz::create(); + inner->insert_value(make_timestamptz(2024, 1, 15, 16, 30, 0, 0)); + auto const_col = ColumnConst::create(std::move(inner), 3); + Block block; + block.insert({std::move(const_col), std::make_shared(), "arg"}); + + check_result("timezone_hour", block, {8, 8, 8}); + check_result("timezone_minute", block, {0, 0, 0}); +} + +TEST_F(FunctionTimezoneHourMinuteTest, const_input_returns_const_column) { + // With the framework constant path disabled, a constant argument is + // evaluated once per execution and the result keeps the block-local const + // shape instead of being expanded to input_rows_count rows. + set_session_timezone(cctz::fixed_time_zone(std::chrono::hours(8))); + + auto inner = ColumnTimeStampTz::create(); + inner->insert_value(make_timestamptz(2024, 1, 15, 12, 0, 0, 0)); + auto const_col = ColumnConst::create(std::move(inner), 3); + Block block; + block.insert({std::move(const_col), std::make_shared(), "arg"}); + + auto return_type = std::make_shared(); + FunctionBasePtr func = SimpleFunctionFactory::instance().get_function( + "timezone_hour", block.get_columns_with_type_and_name(), return_type); + ASSERT_NE(func, nullptr); + Block input_block = block; + input_block.insert({nullptr, return_type, "result"}); + auto st = func->execute(&context, input_block, arguments, result, input_block.rows()); + ASSERT_TRUE(st.ok()) << st.to_string(); + + const auto& result_col = input_block.get_by_position(result).column; + ASSERT_TRUE(is_column_const(*result_col)) << "result must keep the const shape"; + ASSERT_EQ(result_col->size(), 3); + const auto& const_result = assert_cast(*result_col); + const auto& data = assert_cast(*const_result.get_data_column_ptr()); + ASSERT_EQ(data.size(), 1) << "the const value must be computed once"; + EXPECT_EQ(data.get_element(0), 8); +} + +TEST_F(FunctionTimezoneHourMinuteTest, session_zone_wins_over_input_zone) { + // The input instant is noon in UTC-04:30, i.e. 16:30 UTC. Trino would + // return -4/-30 from the input zone; Doris stores only the UTC instant + // and therefore returns the session zone offset (America/New_York in + // winter: -5/0). + cctz::time_zone tz; + ASSERT_TRUE(TimezoneUtils::find_cctz_time_zone("America/New_York", tz)); + set_session_timezone(tz); + + auto block = ColumnHelper::create_block( + {make_timestamptz(2024, 1, 15, 16, 30, 0, 0)}); + + check_result("timezone_hour", block, {-5}); + check_result("timezone_minute", block, {0}); +} + +TEST_F(FunctionTimezoneHourMinuteTest, nullable_input) { + set_session_timezone(cctz::fixed_time_zone(std::chrono::hours(8))); + + auto nested = ColumnTimeStampTz::create(); + nested->insert_value(make_timestamptz(2024, 1, 15, 12, 0, 0, 0)); + nested->insert_value(make_timestamptz(2024, 1, 15, 12, 0, 0, 0)); + auto null_map = ColumnUInt8::create(); + null_map->insert_value(0); + null_map->insert_value(1); + auto nullable_col = ColumnNullable::create(std::move(nested), std::move(null_map)); + Block block; + block.insert({std::move(nullable_col), make_nullable(std::make_shared()), + "arg"}); + + auto return_type = make_nullable(std::make_shared()); + FunctionBasePtr func = SimpleFunctionFactory::instance().get_function( + "timezone_hour", block.get_columns_with_type_and_name(), return_type); + ASSERT_NE(func, nullptr); + block.insert({nullptr, return_type, "result"}); + auto st = func->execute(&context, block, arguments, result, block.rows()); + ASSERT_TRUE(st.ok()) << st.to_string(); + + const auto& col = assert_cast(*block.get_by_position(result).column); + const auto& data = assert_cast(col.get_nested_column()); + ASSERT_EQ(col.size(), 2); + EXPECT_EQ(data.get_element(0), 8); + EXPECT_FALSE(col.is_null_at(0)); + EXPECT_TRUE(col.is_null_at(1)); +} + +TEST_F(FunctionTimezoneHourMinuteTest, disables_default_constant_implementation) { + // The result depends on the session time_zone, so the default constant + // implementation must stay disabled: it would fold a constant input into + // a cached result column and, via VectorizedFnCall::is_constant(), let + // the point-query short-circuit executor reuse a value computed with the + // default +08:00 zone after the request's time_zone is applied. + auto block = ColumnHelper::create_block( + {make_timestamptz(2024, 1, 15, 12, 0, 0, 0)}); + for (const std::string& func_name : {"timezone_hour", "timezone_minute"}) { + FunctionBasePtr func = SimpleFunctionFactory::instance().get_function( + func_name, block.get_columns_with_type_and_name(), + std::make_shared()); + ASSERT_NE(func, nullptr) << func_name; + EXPECT_FALSE(func->is_use_default_implementation_for_constants()) << func_name; + } +} + +TEST_F(FunctionTimezoneHourMinuteTest, fn_call_with_literal_is_not_constant) { + // Regression guard for the point-query short-circuit path: even with a + // literal argument, VectorizedFnCall::is_constant() must report false so + // the executor re-evaluates the expression per request instead of + // reusing a column cached under the default timezone. + auto tz_col = ColumnTimeStampTz::create(); + tz_col->insert_value(make_timestamptz(2024, 1, 15, 12, 0, 0, 0)); + TExprNode literal_node = create_texpr_node_from((*tz_col)[0], TYPE_TIMESTAMPTZ, 0, 6); + + TExprNode fn_node; + fn_node.__set_node_type(TExprNodeType::FUNCTION_CALL); + fn_node.__set_type(create_type_desc(TYPE_BIGINT)); + TFunction fn; + fn.name.__set_function_name("timezone_hour"); + fn.__set_binary_type(TFunctionBinaryType::BUILTIN); + fn_node.__set_fn(fn); + fn_node.__set_num_children(1); + + TExpr texpr; + texpr.nodes.push_back(fn_node); + texpr.nodes.push_back(literal_node); + + VExprContextSPtr ctx; + ASSERT_TRUE(VExpr::create_expr_tree(texpr, ctx).ok()) << "create expr tree"; + ASSERT_NE(ctx, nullptr); + ASSERT_TRUE(ctx->prepare(&_state, RowDescriptor()).ok()) << "prepare expr"; + EXPECT_FALSE(ctx->root()->is_constant()); +} + +} // namespace doris diff --git a/be/test/service/point_query_exector_test.cpp b/be/test/service/point_query_exector_test.cpp index 74bd7606e3905e..583c98cfb5fba5 100644 --- a/be/test/service/point_query_exector_test.cpp +++ b/be/test/service/point_query_exector_test.cpp @@ -17,19 +17,28 @@ #include #include +#include #include #include #include #include "common/object_pool.h" +#include "core/assert_cast.h" #include "core/block/block.h" +#include "core/column/column.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type/primitive_type.h" +#include "core/field.h" #include "exprs/vexpr.h" +#include "exprs/vexpr_context.h" #include "runtime/descriptor_helper.h" #include "runtime/descriptors.h" #include "runtime/exec_env.h" #include "runtime/runtime_state.h" #include "service/point_query_executor.h" #include "storage/tablet/tablet_schema.h" +#include "util/timezone_utils.h" namespace doris { @@ -402,6 +411,81 @@ TEST_F(LookupConnectionCacheTest, PQTestDuplicateAdd) { EXPECT_EQ(entry.get(), reusable2.get()) << "Last write should win in key collision"; } +// Regression for timezone-dependent functions on the point-query +// short-circuit path: Reusable::init() must install the request's session +// timezone before opening the output expressions. A constant descendant +// (here the const CAST above least) is evaluated in VExpr::open() with the +// runtime state's timezone; opening under the default +08:00 would cache +// 2024-03-09 19:30 UTC and timezone_hour would return -5, while parsing the +// value in America/New_York yields 2024-03-10 07:30 UTC and must return -4. +TEST(ReusableTimezoneTest, ConstantDescendantUsesRequestTimezone) { + TimezoneUtils::load_timezones_to_cache(); + + auto str_literal = [](const char* value) { + return create_texpr_node_from(Field::create_field(std::string(value)), + TYPE_STRING, 0, 0); + }; + + TExprNode least_node; + least_node.__set_node_type(TExprNodeType::FUNCTION_CALL); + least_node.__set_type(create_type_desc(TYPE_STRING)); + TFunction least_fn; + least_fn.name.__set_function_name("least"); + least_fn.__set_binary_type(TFunctionBinaryType::BUILTIN); + least_node.__set_fn(least_fn); + least_node.__set_num_children(2); + + TExprNode cast_node; + cast_node.__set_node_type(TExprNodeType::CAST_EXPR); + cast_node.__set_type(create_type_desc(TYPE_TIMESTAMPTZ, 0, 6)); + cast_node.__set_num_children(1); + + TExprNode fn_node; + fn_node.__set_node_type(TExprNodeType::FUNCTION_CALL); + fn_node.__set_type(create_type_desc(TYPE_BIGINT)); + TFunction fn; + fn.name.__set_function_name("timezone_hour"); + fn.__set_binary_type(TFunctionBinaryType::BUILTIN); + fn_node.__set_fn(fn); + fn_node.__set_num_children(1); + + // timezone_hour(CAST(least('2024-03-10 03:30:00','2024-03-11 03:30:00') + // AS TIMESTAMPTZ)) + TExpr texpr; + texpr.nodes.push_back(fn_node); + texpr.nodes.push_back(cast_node); + texpr.nodes.push_back(least_node); + texpr.nodes.push_back(str_literal("2024-03-10 03:30:00")); + texpr.nodes.push_back(str_literal("2024-03-11 03:30:00")); + + auto reusable = std::make_shared(); + TQueryOptions query_options; + Status st = reusable->init(ReusableTestHelper::create_descriptor_tablet(), {texpr}, + query_options, *ReusableTestHelper::tablet_schema, 2, + "America/New_York"); + ASSERT_TRUE(st.ok()) << st.to_string(); + + // The expression tree is constant-only, so execute it over a block whose + // row count is carried by a dummy column (the pooled Reusable blocks are + // created empty). + auto dummy = ColumnUInt8::create(); + dummy->insert_many_defaults(2); + Block block; + block.insert({std::move(dummy), std::make_shared(), "dummy"}); + int result_column_id = -1; + st = reusable->output_exprs()[0]->execute(&block, &result_column_id); + ASSERT_TRUE(st.ok()) << st.to_string(); + + auto result_col = + block.get_by_position(result_column_id).column->convert_to_full_column_if_const(); + // The non-strict CAST above is nullable-typed, so the function result is + // wrapped in a ColumnNullable; unwrap it before inspecting elements. + result_col = remove_nullable(result_col); + const auto& col = assert_cast(*result_col); + ASSERT_GE(col.size(), 1); + EXPECT_EQ(col.get_element(0), -4); +} + // Test reference counting mechanism TEST_F(LookupConnectionCacheTest, PQTestEntryLifetime) { LookupConnectionCache cache(1024 * 1024); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java index 762f3afc0f6649..0354a71399cfe6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java @@ -519,6 +519,8 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.TimeFormat; import org.apache.doris.nereids.trees.expressions.functions.scalar.TimeToSec; import org.apache.doris.nereids.trees.expressions.functions.scalar.Timestamp; +import org.apache.doris.nereids.trees.expressions.functions.scalar.TimezoneHour; +import org.apache.doris.nereids.trees.expressions.functions.scalar.TimezoneMinute; import org.apache.doris.nereids.trees.expressions.functions.scalar.ToBase64; import org.apache.doris.nereids.trees.expressions.functions.scalar.ToBase64Binary; import org.apache.doris.nereids.trees.expressions.functions.scalar.ToBinary; @@ -718,6 +720,8 @@ public class BuiltinScalarFunctions implements FunctionHelper { scalar(Conv.class, "conv"), scalar(ConvertTo.class, "convert_to"), scalar(ConvertTz.class, "convert_tz"), + scalar(TimezoneHour.class, "timezone_hour"), + scalar(TimezoneMinute.class, "timezone_minute"), scalar(Cos.class, "cos"), scalar(Csc.class, "csc"), scalar(Cosh.class, "cosh"), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/TimezoneHour.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/TimezoneHour.java new file mode 100644 index 00000000000000..88c50d7e1388b6 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/TimezoneHour.java @@ -0,0 +1,86 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.functions.scalar; + +import org.apache.doris.catalog.FunctionSignature; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable; +import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; +import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; +import org.apache.doris.nereids.types.BigIntType; +import org.apache.doris.nereids.types.TimeStampTzType; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; + +import java.util.List; + +/** + * ScalarFunction 'timezone_hour'. + * + *

Returns the hour part of the UTC offset of the session time zone at the + * given instant. Note: Doris TIMESTAMPTZ values are stored as UTC instants + * without the input zone, so unlike Trino's timezone_hour, this function + * extracts the session time zone offset.

+ */ +public class TimezoneHour extends ScalarFunction + implements UnaryExpression, ExplicitlyCastableSignature, PropagateNullable { + + public static final List SIGNATURES = ImmutableList.of( + FunctionSignature.ret(BigIntType.INSTANCE).args(TimeStampTzType.WILDCARD)); + + /** + * constructor with 1 argument. + */ + public TimezoneHour(Expression arg) { + super("timezone_hour", arg); + } + + /** constructor for withChildren and reuse signature */ + private TimezoneHour(ScalarFunctionParams functionParams) { + super(functionParams); + } + + /** + * withChildren. + */ + @Override + public TimezoneHour withChildren(List children) { + Preconditions.checkArgument(children.size() == 1); + return new TimezoneHour(getFunctionParams(children)); + } + + @Override + public List getSignatures() { + return SIGNATURES; + } + + @Override + public R accept(ExpressionVisitor visitor, C context) { + return visitor.visitTimezoneHour(this, context); + } + + @Override + public boolean isDeterministic() { + // The result depends on the session time_zone, which may change between + // executions, so this function must not be folded into prepared plans + // or used in materialized views. + return false; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/TimezoneMinute.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/TimezoneMinute.java new file mode 100644 index 00000000000000..d792e7c62362af --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/TimezoneMinute.java @@ -0,0 +1,86 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.functions.scalar; + +import org.apache.doris.catalog.FunctionSignature; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable; +import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; +import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; +import org.apache.doris.nereids.types.BigIntType; +import org.apache.doris.nereids.types.TimeStampTzType; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; + +import java.util.List; + +/** + * ScalarFunction 'timezone_minute'. + * + *

Returns the minute part of the UTC offset of the session time zone at the + * given instant. Note: Doris TIMESTAMPTZ values are stored as UTC instants + * without the input zone, so unlike Trino's timezone_minute, this function + * extracts the session time zone offset.

+ */ +public class TimezoneMinute extends ScalarFunction + implements UnaryExpression, ExplicitlyCastableSignature, PropagateNullable { + + public static final List SIGNATURES = ImmutableList.of( + FunctionSignature.ret(BigIntType.INSTANCE).args(TimeStampTzType.WILDCARD)); + + /** + * constructor with 1 argument. + */ + public TimezoneMinute(Expression arg) { + super("timezone_minute", arg); + } + + /** constructor for withChildren and reuse signature */ + private TimezoneMinute(ScalarFunctionParams functionParams) { + super(functionParams); + } + + /** + * withChildren. + */ + @Override + public TimezoneMinute withChildren(List children) { + Preconditions.checkArgument(children.size() == 1); + return new TimezoneMinute(getFunctionParams(children)); + } + + @Override + public List getSignatures() { + return SIGNATURES; + } + + @Override + public R accept(ExpressionVisitor visitor, C context) { + return visitor.visitTimezoneMinute(this, context); + } + + @Override + public boolean isDeterministic() { + // The result depends on the session time_zone, which may change between + // executions, so this function must not be folded into prepared plans + // or used in materialized views. + return false; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java index 6b73a00b85440f..bc17cf884c0cf0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java @@ -537,6 +537,8 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.TimeDiff; import org.apache.doris.nereids.trees.expressions.functions.scalar.TimeFormat; import org.apache.doris.nereids.trees.expressions.functions.scalar.Timestamp; +import org.apache.doris.nereids.trees.expressions.functions.scalar.TimezoneHour; +import org.apache.doris.nereids.trees.expressions.functions.scalar.TimezoneMinute; import org.apache.doris.nereids.trees.expressions.functions.scalar.ToBase64; import org.apache.doris.nereids.trees.expressions.functions.scalar.ToBase64Binary; import org.apache.doris.nereids.trees.expressions.functions.scalar.ToBinary; @@ -1103,6 +1105,14 @@ default R visitConvertTz(ConvertTz convertTz, C context) { return visitScalarFunction(convertTz, context); } + default R visitTimezoneHour(TimezoneHour timezoneHour, C context) { + return visitScalarFunction(timezoneHour, context); + } + + default R visitTimezoneMinute(TimezoneMinute timezoneMinute, C context) { + return visitScalarFunction(timezoneMinute, context); + } + default R visitCos(Cos cos, C context) { return visitScalarFunction(cos, context); } diff --git a/pytest/qe/palo2/src/test_query_datetime_function.py b/pytest/qe/palo2/src/test_query_datetime_function.py index 67f3530b6f745c..bcf597d8e79db8 100644 --- a/pytest/qe/palo2/src/test_query_datetime_function.py +++ b/pytest/qe/palo2/src/test_query_datetime_function.py @@ -1007,6 +1007,62 @@ def test_query_time_convert_tz(): runner.check2(line1, line2) +def test_query_timezone_hour_minute(): + """ + { + "title": "test_query_datetime_function.test_query_timezone_hour_minute", + "describe": "test for timezone_hour and timezone_minute", + "tag": "function,p0" + } + """ + # The SET and the SELECT must run on the same connection, so use + # do_set_properties_sql instead of runner.init (which opens a fresh + # session for every statement). + + # UTC+08:00 has no DST, the offset of the session timezone is the same + # for every instant, so timezone_hour always returns 8 here. + ret = runner.query_palo.do_set_properties_sql( + "select timezone_hour(cast('2024-01-15 12:00:00' as TIMESTAMPTZ)), " + "timezone_minute(cast('2024-07-15 12:00:00' as TIMESTAMPTZ))", + ["set time_zone = '+08:00'"]) + assert int(ret[0][0]) == 8 and int(ret[0][1]) == 0, ret + + # America/New_York switches between EST (UTC-05:00) in winter and + # EDT (UTC-04:00) in summer. + ret = runner.query_palo.do_set_properties_sql( + "select timezone_hour(cast('2024-01-15 12:00:00' as TIMESTAMPTZ)), " + "timezone_minute(cast('2024-01-15 12:00:00' as TIMESTAMPTZ)), " + "timezone_hour(cast('2024-07-15 12:00:00' as TIMESTAMPTZ)), " + "timezone_minute(cast('2024-07-15 12:00:00' as TIMESTAMPTZ))", + ["set time_zone = 'America/New_York'"]) + assert int(ret[0][0]) == -5 and int(ret[0][1]) == 0, ret + assert int(ret[0][2]) == -4 and int(ret[0][3]) == 0, ret + + # Fractional session offsets are truncated like Trino: Asia/Kolkata is + # UTC+05:30, so timezone_hour returns 5 and timezone_minute returns 30. + ret = runner.query_palo.do_set_properties_sql( + "select timezone_hour(cast('2024-01-15 12:00:00' as TIMESTAMPTZ)), " + "timezone_minute(cast('2024-01-15 12:00:00' as TIMESTAMPTZ))", + ["set time_zone = 'Asia/Kolkata'"]) + assert int(ret[0][0]) == 5 and int(ret[0][1]) == 30, ret + + # Doris TIMESTAMPTZ stores only the UTC instant, not the input zone, so + # even when the input carries '-04:30', the extracted offset is the + # session zone's offset (Trino would return -4/-30 here). + ret = runner.query_palo.do_set_properties_sql( + "select timezone_hour(cast('2024-01-15 12:00:00-04:30' as TIMESTAMPTZ)), " + "timezone_minute(cast('2024-01-15 12:00:00-04:30' as TIMESTAMPTZ))", + ["set time_zone = '+08:00'"]) + assert int(ret[0][0]) == 8 and int(ret[0][1]) == 0, ret + + # NULL input returns NULL. + ret = runner.query_palo.do_set_properties_sql( + "select timezone_hour(cast(null as TIMESTAMPTZ)), " + "timezone_minute(cast(null as TIMESTAMPTZ))", + ["set time_zone = '+08:00'"]) + assert ret[0][0] is None and ret[0][1] is None, ret + + def test_query_timestampdiff(): """ {