Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
162 changes: 162 additions & 0 deletions be/src/exprs/function/function_timezone_hour_minute.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,162 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

#include <cctz/time_zone.h>

#include <cstdint>
#include <memory>
#include <string>
#include <utility>

#include "common/status.h"
#include "core/assert_cast.h"
#include "core/block/block.h"
#include "core/block/column_numbers.h"
#include "core/column/column.h"
#include "core/column/column_const.h"
#include "core/column/column_nullable.h"
#include "core/column/column_vector.h"
#include "core/data_type/data_type.h"
#include "core/data_type/data_type_nullable.h"
#include "core/data_type/data_type_number.h"
#include "core/data_type/data_type_timestamptz.h"
#include "core/data_type/primitive_type.h"
#include "core/value/timestamptz_value.h"
#include "exprs/function_context.h"
#include "exprs/function/function.h"
#include "exprs/function/simple_function_factory.h"
#include "runtime/runtime_state.h"

namespace doris {

namespace {
constexpr int64_t SECONDS_PER_HOUR = 3600;
constexpr int64_t SECONDS_PER_MINUTE = 60;

// TIMESTAMPTZ values are stored as UTC instants without the input zone, so the
// offset extracted here is the offset of the session time zone at the instant.
// See TimestampTzValue for the storage design.
Status execute_timezone_offset_part(FunctionContext* context, Block& block,
const ColumnNumbers& arguments, uint32_t result,
size_t input_rows_count, bool extract_hour) {
ColumnPtr col = block.get_by_position(arguments[0]).column;
// Fast path: the framework constant path is disabled
// (use_default_implementation_for_constants() == false) so a constant
// argument must not be expanded to input_rows_count rows and re-evaluated
// per row. Compute the single value once and keep the block-local const
// shape. The result type must be non-nullable here: the framework's
// default null handling wraps the result into a ColumnNullable, which a
// ColumnConst cannot be nested in.
if (is_column_const(*col) && !block.get_by_position(result).type->is_nullable()) {
const auto& const_col = assert_cast<const ColumnConst&>(*col);
const auto& tz_column =
assert_cast<const ColumnTimeStampTz&>(*const_col.get_data_column_ptr());
int64_t offset = tz_column.get_data()[0].utc_offset(context->state()->timezone_obj());
int64_t value = extract_hour ? offset / SECONDS_PER_HOUR
: (offset % SECONDS_PER_HOUR) / SECONDS_PER_MINUTE;
auto nested = ColumnInt64::create();
nested->insert_value(value);
block.get_by_position(result).column =
ColumnConst::create(std::move(nested), input_rows_count);
return Status::OK();
}
// Unwrap nullable and const wrappers in any nesting order so that
// ColumnNullable(ColumnConst(...)) and ColumnConst(ColumnNullable(...))
// inputs both reach the plain ColumnTimeStampTz data below.
col = remove_nullable(col);
if (is_column_const(*col)) {
col = assert_cast<const ColumnConst&>(*col).convert_to_full_column();

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[P2] Preserve block-local constness without enabling cross-request caching

Because the framework constant path is disabled, this expands a one-value ColumnConst to input_rows_count, allocates a full result column, and performs the identical cctz lookup for every scanned row. A projection such as timezone_hour(CAST('2024-01-15 12:00:00' AS TIMESTAMPTZ)) over a large table therefore does O(N) timezone work for one per-execution value. Keep use_default_implementation_for_constants() false so VectorizedFnCall::is_constant() cannot cache across requests, but detect the const argument here, evaluate its nested value once, and return a block-local ColumnConst; the const-input test can assert that physical shape.

col = remove_nullable(col);
}
const auto* tz_column = assert_cast<const ColumnTimeStampTz*>(col.get());
const auto& tz_data = tz_column->get_data();

auto result_column = ColumnInt64::create();
auto& result_data = result_column->get_data();
result_data.resize(input_rows_count);

const cctz::time_zone& timezone = context->state()->timezone_obj();

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[P1] Extract the input value's zone, not the session zone

Trino's timestamp with time zone retains a zone key, and timezone_hour/timezone_minute extract that value's offset. Doris converts an explicit input zone to UTC and discards it, then this line substitutes the session zone. For example, with session +08:00, CAST('2024-01-15 12:00:00-04:30' AS TIMESTAMPTZ) returns 8/0 here instead of Trino's -4/-30. That silently breaks the advertised migration compatibility. Please resolve the contract by retaining/extracting the input zone (including serialization compatibility), or explicitly scope/rename the feature as session-offset extraction, and add an end-to-end case where the input and session zones differ.

for (size_t i = 0; i < input_rows_count; ++i) {
int64_t offset = tz_data[i].utc_offset(timezone);
result_data[i] = extract_hour ? offset / SECONDS_PER_HOUR
: (offset % SECONDS_PER_HOUR) / SECONDS_PER_MINUTE;
}

block.get_by_position(result).column = std::move(result_column);
return Status::OK();
}
} // namespace

class FunctionTimezoneHour : public IFunction {
public:
static constexpr auto name = "timezone_hour";

static FunctionPtr create() { return std::make_shared<FunctionTimezoneHour>(); }

String get_name() const override { return name; }

size_t get_number_of_arguments() const override { return 1; }

DataTypePtr get_return_type_impl(const DataTypes& arguments) const override {
return std::make_shared<DataTypeInt64>();
}

// The result depends on the session time_zone, so a constant result must
// never be cached: the point-query short-circuit executor opens output
// expressions with the default timezone and later reuses cached constant
// columns without re-evaluating them (VectorizedFnCall::is_constant
// consults this flag). Disable it like other nondeterministic functions
// (e.g. random, uuid).
bool use_default_implementation_for_constants() const override { return false; }

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[P1] Apply the request timezone before constant children open

This override only makes the outer call nonconstant. VectorizedFnCall::open() still opens its children first, and a constant VCastExpr caches itself through VExpr::get_const_col(). In the point-query path, Reusable::init() opens the expression tree while its RuntimeState still has Doris's default +08:00; PointQueryExecutor::init() applies request->time_zone only afterward. For example, with session America/New_York, timezone_hour(CAST(least('2024-03-10 03:30:00','2024-03-11 03:30:00') AS TIMESTAMPTZ)) survives FE folding because least has no FE evaluator. The child cast is cached as 2024-03-09 19:30 UTC under +08:00, so the outer call returns -5; parsing the selected value in New York gives 2024-03-10 07:30 UTC and should return -4. This is distinct from the existing outer-result cache thread: the stale value is a constant descendant and is wrong on the first point-query request. Please install the request timezone before opening or caching the point-query expression tree and add this nested-constant DST regression.


Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments,
uint32_t result, size_t input_rows_count) const override {
return execute_timezone_offset_part(context, block, arguments, result, input_rows_count,
true);
}
};

class FunctionTimezoneMinute : public IFunction {
public:
static constexpr auto name = "timezone_minute";

static FunctionPtr create() { return std::make_shared<FunctionTimezoneMinute>(); }

String get_name() const override { return name; }

size_t get_number_of_arguments() const override { return 1; }

DataTypePtr get_return_type_impl(const DataTypes& arguments) const override {
return std::make_shared<DataTypeInt64>();
}

// See FunctionTimezoneHour::use_default_implementation_for_constants.
bool use_default_implementation_for_constants() const override { return false; }

Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments,
uint32_t result, size_t input_rows_count) const override {
return execute_timezone_offset_part(context, block, arguments, result, input_rows_count,
false);
}
};

void register_function_timezone_hour_minute(SimpleFunctionFactory& factory) {
factory.register_function<FunctionTimezoneHour>();
factory.register_function<FunctionTimezoneMinute>();
}

} // namespace doris
2 changes: 2 additions & 0 deletions be/src/exprs/function/simple_function_factory.h
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,7 @@ void register_function_uuid_transforms(SimpleFunctionFactory& factory);
void register_function_grouping(SimpleFunctionFactory& factory);
void register_function_datetime_floor_ceil(SimpleFunctionFactory& factory);
void register_function_convert_tz(SimpleFunctionFactory& factory);
void register_function_timezone_hour_minute(SimpleFunctionFactory& factory);
void register_function_least_greast(SimpleFunctionFactory& factory);
void register_function_fake(SimpleFunctionFactory& factory);
void register_function_array(SimpleFunctionFactory& factory);
Expand Down Expand Up @@ -332,6 +333,7 @@ class SimpleFunctionFactory {
register_function_grouping(instance);
register_function_datetime_floor_ceil(instance);
register_function_convert_tz(instance);
register_function_timezone_hour_minute(instance);
register_function_least_greast(instance);
register_function_fake(instance);
register_function_encryption(instance);
Expand Down
4 changes: 3 additions & 1 deletion be/src/exprs/vectorized_fn_call.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -666,7 +666,9 @@ bool VectorizedFnCall::can_push_down_to_index() const {

bool VectorizedFnCall::is_deterministic() const {
static const std::set<std::string> NON_DETERMINISTIC_FUNCTIONS = {
"random", "rand", "random_bytes", "uuid", "uuid_numeric"};
"random", "rand", "random_bytes", "uuid", "uuid_numeric",
// timezone_hour/timezone_minute depend on the session time_zone.
"timezone_hour", "timezone_minute"};
return !NON_DETERMINISTIC_FUNCTIONS.contains(_function_name) && VExpr::is_deterministic();
}

Expand Down
25 changes: 19 additions & 6 deletions be/src/service/point_query_executor.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -130,9 +130,16 @@ static void extract_slot_ref(const VExprSPtr& expr, TupleDescriptor* tuple_desc,

Status Reusable::init(const TDescriptorTable& t_desc_tbl, const std::vector<TExpr>& output_exprs,
const TQueryOptions& query_options, const TabletSchema& schema,
size_t block_size) {
size_t block_size, const std::string& time_zone) {
_runtime_state = RuntimeState::create_unique();
_runtime_state->set_query_options(query_options);
// Install the request's session timezone before the expressions are
// opened: VExpr::open() evaluates constant children (e.g. a const
// VCastExpr) with the runtime state's timezone, which would otherwise be
// the default +08:00 even though the request may use a different one.
if (!time_zone.empty()) {
_runtime_state->set_timezone(time_zone);
}
RETURN_IF_ERROR(DescriptorTbl::create(_runtime_state->obj_pool(), t_desc_tbl, &_desc_tbl));
_runtime_state->set_desc_tbl(_desc_tbl);
for (const auto* slot : tuple_desc()->slots()) {
Expand Down Expand Up @@ -306,6 +313,12 @@ Status PointQueryExecutor::init(const PTabletKeyLookupRequest* request,
auto cache_handle = LookupConnectionCache::instance()->get(uuid);
_binary_row_format = request->is_binary_row();
_tablet = DORIS_TRY(ExecEnv::get_tablet(request->tablet_id()));
// Timezone of the session that sent the request. It must be installed on
// the reusable's runtime state before the output expressions are opened
// so that constant children are evaluated with it (see Reusable::init).
std::string request_time_zone =
(request->has_time_zone() && !request->time_zone().empty()) ? request->time_zone()
: std::string();
if (cache_handle != nullptr) {
_reusable = cache_handle;
_profile_metrics.hit_lookup_cache = true;
Expand Down Expand Up @@ -352,18 +365,18 @@ Status PointQueryExecutor::init(const PTabletKeyLookupRequest* request,
if (uuid != 0) {
// could be reused by requests after, pre allocte more blocks
RETURN_IF_ERROR(reusable_ptr->init(t_desc_tbl, t_output_exprs.exprs, t_query_options,
*_tablet->tablet_schema(),
s_preallocted_blocks_num));
*_tablet->tablet_schema(), s_preallocted_blocks_num,
request_time_zone));
LookupConnectionCache::instance()->add(uuid, reusable_ptr);
} else {
RETURN_IF_ERROR(reusable_ptr->init(t_desc_tbl, t_output_exprs.exprs, t_query_options,
*_tablet->tablet_schema(), 1));
*_tablet->tablet_schema(), 1, request_time_zone));
}
}
_init_remote_scan_cache_write_limiter();
// Set timezone from request for functions like from_unixtime()
if (request->has_time_zone() && !request->time_zone().empty()) {
_reusable->runtime_state()->set_timezone(request->time_zone());
if (!request_time_zone.empty()) {
_reusable->runtime_state()->set_timezone(request_time_zone);
}
if (request->has_version() && request->version() >= 0) {
_version = request->version();
Expand Down
2 changes: 1 addition & 1 deletion be/src/service/point_query_executor.h
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ class Reusable {

Status init(const TDescriptorTable& t_desc_tbl, const std::vector<TExpr>& output_exprs,
const TQueryOptions& query_options, const TabletSchema& schema,
size_t block_size = 1);
size_t block_size = 1, const std::string& time_zone = "");

std::unique_ptr<Block> get_block();

Expand Down
Loading