From 41ddf2def2e3d05dbaa8d7b6a336d9c2815bef8e Mon Sep 17 00:00:00 2001 From: Cao Yu <2320485192@qq.com> Date: Tue, 18 Aug 2026 09:35:31 +0800 Subject: [PATCH 1/3] [feature](function) Add Trino-compatible timezone_hour and timezone_minute functions --- .../function_timezone_hour_minute.cpp | 124 ++++++++++++++ .../exprs/function/simple_function_factory.h | 2 + .../function_timezone_hour_minute_test.cpp | 156 ++++++++++++++++++ .../doris/catalog/BuiltinScalarFunctions.java | 4 + .../functions/scalar/TimezoneHour.java | 73 ++++++++ .../functions/scalar/TimezoneMinute.java | 73 ++++++++ .../visitor/ScalarFunctionVisitor.java | 10 ++ .../palo2/src/test_query_datetime_function.py | 31 ++++ 8 files changed, 473 insertions(+) create mode 100644 be/src/exprs/function/function_timezone_hour_minute.cpp create mode 100644 be/test/exprs/function/function_timezone_hour_minute_test.cpp create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/TimezoneHour.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/TimezoneMinute.java diff --git a/be/src/exprs/function/function_timezone_hour_minute.cpp b/be/src/exprs/function/function_timezone_hour_minute.cpp new file mode 100644 index 00000000000000..7509841d8c574b --- /dev/null +++ b/be/src/exprs/function/function_timezone_hour_minute.cpp @@ -0,0 +1,124 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include +#include +#include +#include + +#include "common/status.h" +#include "core/assert_cast.h" +#include "core/block/block.h" +#include "core/block/column_numbers.h" +#include "core/column/column.h" +#include "core/column/column_const.h" +#include "core/column/column_nullable.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_timestamptz.h" +#include "core/data_type/primitive_type.h" +#include "core/value/timestamptz_value.h" +#include "exprs/function_context.h" +#include "exprs/function/function.h" +#include "exprs/function/simple_function_factory.h" +#include "runtime/runtime_state.h" + +namespace doris { + +namespace { +constexpr int64_t SECONDS_PER_HOUR = 3600; +constexpr int64_t SECONDS_PER_MINUTE = 60; + +Status execute_timezone_offset_part(FunctionContext* context, Block& block, + const ColumnNumbers& arguments, uint32_t result, + size_t input_rows_count, bool extract_hour) { + ColumnPtr col = block.get_by_position(arguments[0]).column; + if (is_column_const(*col)) { + col = assert_cast(*col).convert_to_full_column(); + } + col = remove_nullable(col); + const auto* tz_column = assert_cast(col.get()); + const auto& tz_data = tz_column->get_data(); + + auto result_column = ColumnInt64::create(); + auto& result_data = result_column->get_data(); + result_data.resize(input_rows_count); + + const cctz::time_zone& timezone = context->state()->timezone_obj(); + for (size_t i = 0; i < input_rows_count; ++i) { + int64_t offset = tz_data[i].utc_offset(timezone); + result_data[i] = extract_hour ? offset / SECONDS_PER_HOUR + : (offset % SECONDS_PER_HOUR) / SECONDS_PER_MINUTE; + } + + block.get_by_position(result).column = std::move(result_column); + return Status::OK(); +} +} // namespace + +class FunctionTimezoneHour : public IFunction { +public: + static constexpr auto name = "timezone_hour"; + + static FunctionPtr create() { return std::make_shared(); } + + String get_name() const override { return name; } + + size_t get_number_of_arguments() const override { return 1; } + + DataTypePtr get_return_type_impl(const DataTypes& arguments) const override { + return std::make_shared(); + } + + Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments, + uint32_t result, size_t input_rows_count) const override { + return execute_timezone_offset_part(context, block, arguments, result, input_rows_count, + true); + } +}; + +class FunctionTimezoneMinute : public IFunction { +public: + static constexpr auto name = "timezone_minute"; + + static FunctionPtr create() { return std::make_shared(); } + + String get_name() const override { return name; } + + size_t get_number_of_arguments() const override { return 1; } + + DataTypePtr get_return_type_impl(const DataTypes& arguments) const override { + return std::make_shared(); + } + + Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments, + uint32_t result, size_t input_rows_count) const override { + return execute_timezone_offset_part(context, block, arguments, result, input_rows_count, + false); + } +}; + +void register_function_timezone_hour_minute(SimpleFunctionFactory& factory) { + factory.register_function(); + factory.register_function(); +} + +} // namespace doris diff --git a/be/src/exprs/function/simple_function_factory.h b/be/src/exprs/function/simple_function_factory.h index 5dd09e0847dba5..52b5a27b9ac98e 100644 --- a/be/src/exprs/function/simple_function_factory.h +++ b/be/src/exprs/function/simple_function_factory.h @@ -87,6 +87,7 @@ void register_function_uuid_transforms(SimpleFunctionFactory& factory); void register_function_grouping(SimpleFunctionFactory& factory); void register_function_datetime_floor_ceil(SimpleFunctionFactory& factory); void register_function_convert_tz(SimpleFunctionFactory& factory); +void register_function_timezone_hour_minute(SimpleFunctionFactory& factory); void register_function_least_greast(SimpleFunctionFactory& factory); void register_function_fake(SimpleFunctionFactory& factory); void register_function_array(SimpleFunctionFactory& factory); @@ -332,6 +333,7 @@ class SimpleFunctionFactory { register_function_grouping(instance); register_function_datetime_floor_ceil(instance); register_function_convert_tz(instance); + register_function_timezone_hour_minute(instance); register_function_least_greast(instance); register_function_fake(instance); register_function_encryption(instance); diff --git a/be/test/exprs/function/function_timezone_hour_minute_test.cpp b/be/test/exprs/function/function_timezone_hour_minute_test.cpp new file mode 100644 index 00000000000000..0b6643af2f90e1 --- /dev/null +++ b/be/test/exprs/function/function_timezone_hour_minute_test.cpp @@ -0,0 +1,156 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include + +#include +#include +#include + +#include "core/assert_cast.h" +#include "core/block/block.h" +#include "core/block/column_numbers.h" +#include "core/column/column.h" +#include "core/column/column_nullable.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_timestamptz.h" +#include "exprs/function/function.h" +#include "exprs/function/simple_function_factory.h" +#include "testutil/column_helper.h" +#include "testutil/datetime_ut_util.h" +#include "testutil/mock/mock_runtime_state.h" +#include "util/timezone_utils.h" + +namespace doris { + +class FunctionTimezoneHourMinuteTest : public testing::Test { +public: + void SetUp() override { + TimezoneUtils::load_offsets_to_cache(); + TimezoneUtils::load_timezones_to_cache(); + context._state = &_state; + arguments = {0}; + result = 1; + } + + void set_session_timezone(const cctz::time_zone& tz) { _state._timezone_obj = tz; } + + void check_result(const std::string& func_name, const Block& block, + const std::vector& expected) { + auto return_type = std::make_shared(); + FunctionBasePtr func = SimpleFunctionFactory::instance().get_function( + func_name, block.get_columns_with_type_and_name(), return_type); + ASSERT_NE(func, nullptr); + Block input_block = block; + input_block.insert({nullptr, return_type, "result"}); + auto st = func->execute(&context, input_block, arguments, result, input_block.rows()); + ASSERT_TRUE(st.ok()) << st.to_string(); + const auto& col = + assert_cast(*input_block.get_by_position(result).column); + ASSERT_EQ(col.size(), expected.size()); + for (size_t i = 0; i < expected.size(); ++i) { + EXPECT_EQ(col.get_element(i), expected[i]) << "at row " << i; + } + } + + MockRuntimeState _state; + FunctionContext context; + ColumnNumbers arguments; + uint32_t result; +}; + +TEST_F(FunctionTimezoneHourMinuteTest, fixed_offset_shanghai) { + // Asia/Shanghai has a fixed UTC+08:00 offset without DST, so the offset + // part of the session timezone is the same for every instant. + set_session_timezone(cctz::fixed_time_zone(std::chrono::hours(8))); + + auto block = ColumnHelper::create_block( + {make_timestamptz(2024, 1, 15, 12, 0, 0, 0), + make_timestamptz(2024, 7, 15, 12, 0, 0, 0)}); + + check_result("timezone_hour", block, {8, 8}); + check_result("timezone_minute", block, {0, 0}); +} + +TEST_F(FunctionTimezoneHourMinuteTest, dst_new_york) { + // America/New_York switches between EST (UTC-05:00) in winter and + // EDT (UTC-04:00) in summer, which is reflected in the returned offset. + cctz::time_zone tz; + ASSERT_TRUE(TimezoneUtils::find_cctz_time_zone("America/New_York", tz)); + set_session_timezone(tz); + + auto winter_block = ColumnHelper::create_block( + {make_timestamptz(2024, 1, 15, 12, 0, 0, 0)}); + auto summer_block = ColumnHelper::create_block( + {make_timestamptz(2024, 7, 15, 12, 0, 0, 0)}); + + check_result("timezone_hour", winter_block, {-5}); + check_result("timezone_minute", winter_block, {0}); + check_result("timezone_hour", summer_block, {-4}); + check_result("timezone_minute", summer_block, {0}); +} + +TEST_F(FunctionTimezoneHourMinuteTest, fractional_offsets) { + // Trino returns truncated integer values for fractional offsets: + // timezone_hour(UTC-04:30) = -4 and timezone_minute(UTC-04:30) = -30. + set_session_timezone(cctz::fixed_time_zone(std::chrono::seconds(-4 * 3600 - 30 * 60))); + auto block = ColumnHelper::create_block( + {make_timestamptz(2024, 6, 20, 12, 0, 0, 0)}); + check_result("timezone_hour", block, {-4}); + check_result("timezone_minute", block, {-30}); + + // Nepal Standard Time (UTC+05:45). + set_session_timezone(cctz::fixed_time_zone(std::chrono::seconds(5 * 3600 + 45 * 60))); + check_result("timezone_hour", block, {5}); + check_result("timezone_minute", block, {45}); +} + +TEST_F(FunctionTimezoneHourMinuteTest, nullable_input) { + set_session_timezone(cctz::fixed_time_zone(std::chrono::hours(8))); + + auto nested = ColumnTimeStampTz::create(); + nested->insert_value(make_timestamptz(2024, 1, 15, 12, 0, 0, 0)); + nested->insert_value(make_timestamptz(2024, 1, 15, 12, 0, 0, 0)); + auto null_map = ColumnUInt8::create(); + null_map->insert_value(0); + null_map->insert_value(1); + auto nullable_col = ColumnNullable::create(std::move(nested), std::move(null_map)); + Block block; + block.insert({std::move(nullable_col), make_nullable(std::make_shared()), + "arg"}); + + auto return_type = make_nullable(std::make_shared()); + FunctionBasePtr func = SimpleFunctionFactory::instance().get_function( + "timezone_hour", block.get_columns_with_type_and_name(), return_type); + ASSERT_NE(func, nullptr); + block.insert({nullptr, return_type, "result"}); + auto st = func->execute(&context, block, arguments, result, block.rows()); + ASSERT_TRUE(st.ok()) << st.to_string(); + + const auto& col = assert_cast(*block.get_by_position(result).column); + const auto& data = assert_cast(col.get_nested_column()); + ASSERT_EQ(col.size(), 2); + EXPECT_EQ(data.get_element(0), 8); + EXPECT_FALSE(col.is_null_at(0)); + EXPECT_TRUE(col.is_null_at(1)); +} + +} // namespace doris diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java index 762f3afc0f6649..0354a71399cfe6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java @@ -519,6 +519,8 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.TimeFormat; import org.apache.doris.nereids.trees.expressions.functions.scalar.TimeToSec; import org.apache.doris.nereids.trees.expressions.functions.scalar.Timestamp; +import org.apache.doris.nereids.trees.expressions.functions.scalar.TimezoneHour; +import org.apache.doris.nereids.trees.expressions.functions.scalar.TimezoneMinute; import org.apache.doris.nereids.trees.expressions.functions.scalar.ToBase64; import org.apache.doris.nereids.trees.expressions.functions.scalar.ToBase64Binary; import org.apache.doris.nereids.trees.expressions.functions.scalar.ToBinary; @@ -718,6 +720,8 @@ public class BuiltinScalarFunctions implements FunctionHelper { scalar(Conv.class, "conv"), scalar(ConvertTo.class, "convert_to"), scalar(ConvertTz.class, "convert_tz"), + scalar(TimezoneHour.class, "timezone_hour"), + scalar(TimezoneMinute.class, "timezone_minute"), scalar(Cos.class, "cos"), scalar(Csc.class, "csc"), scalar(Cosh.class, "cosh"), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/TimezoneHour.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/TimezoneHour.java new file mode 100644 index 00000000000000..6cf2d814a626fc --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/TimezoneHour.java @@ -0,0 +1,73 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.functions.scalar; + +import org.apache.doris.catalog.FunctionSignature; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable; +import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; +import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; +import org.apache.doris.nereids.types.BigIntType; +import org.apache.doris.nereids.types.TimeStampTzType; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; + +import java.util.List; + +/** + * ScalarFunction 'timezone_hour'. + */ +public class TimezoneHour extends ScalarFunction + implements UnaryExpression, ExplicitlyCastableSignature, PropagateNullable { + + public static final List SIGNATURES = ImmutableList.of( + FunctionSignature.ret(BigIntType.INSTANCE).args(TimeStampTzType.WILDCARD)); + + /** + * constructor with 1 argument. + */ + public TimezoneHour(Expression arg) { + super("timezone_hour", arg); + } + + /** constructor for withChildren and reuse signature */ + private TimezoneHour(ScalarFunctionParams functionParams) { + super(functionParams); + } + + /** + * withChildren. + */ + @Override + public TimezoneHour withChildren(List children) { + Preconditions.checkArgument(children.size() == 1); + return new TimezoneHour(getFunctionParams(children)); + } + + @Override + public List getSignatures() { + return SIGNATURES; + } + + @Override + public R accept(ExpressionVisitor visitor, C context) { + return visitor.visitTimezoneHour(this, context); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/TimezoneMinute.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/TimezoneMinute.java new file mode 100644 index 00000000000000..e2db92a7ae8432 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/TimezoneMinute.java @@ -0,0 +1,73 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.functions.scalar; + +import org.apache.doris.catalog.FunctionSignature; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable; +import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; +import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; +import org.apache.doris.nereids.types.BigIntType; +import org.apache.doris.nereids.types.TimeStampTzType; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; + +import java.util.List; + +/** + * ScalarFunction 'timezone_minute'. + */ +public class TimezoneMinute extends ScalarFunction + implements UnaryExpression, ExplicitlyCastableSignature, PropagateNullable { + + public static final List SIGNATURES = ImmutableList.of( + FunctionSignature.ret(BigIntType.INSTANCE).args(TimeStampTzType.WILDCARD)); + + /** + * constructor with 1 argument. + */ + public TimezoneMinute(Expression arg) { + super("timezone_minute", arg); + } + + /** constructor for withChildren and reuse signature */ + private TimezoneMinute(ScalarFunctionParams functionParams) { + super(functionParams); + } + + /** + * withChildren. + */ + @Override + public TimezoneMinute withChildren(List children) { + Preconditions.checkArgument(children.size() == 1); + return new TimezoneMinute(getFunctionParams(children)); + } + + @Override + public List getSignatures() { + return SIGNATURES; + } + + @Override + public R accept(ExpressionVisitor visitor, C context) { + return visitor.visitTimezoneMinute(this, context); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java index 6b73a00b85440f..bc17cf884c0cf0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java @@ -537,6 +537,8 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.TimeDiff; import org.apache.doris.nereids.trees.expressions.functions.scalar.TimeFormat; import org.apache.doris.nereids.trees.expressions.functions.scalar.Timestamp; +import org.apache.doris.nereids.trees.expressions.functions.scalar.TimezoneHour; +import org.apache.doris.nereids.trees.expressions.functions.scalar.TimezoneMinute; import org.apache.doris.nereids.trees.expressions.functions.scalar.ToBase64; import org.apache.doris.nereids.trees.expressions.functions.scalar.ToBase64Binary; import org.apache.doris.nereids.trees.expressions.functions.scalar.ToBinary; @@ -1103,6 +1105,14 @@ default R visitConvertTz(ConvertTz convertTz, C context) { return visitScalarFunction(convertTz, context); } + default R visitTimezoneHour(TimezoneHour timezoneHour, C context) { + return visitScalarFunction(timezoneHour, context); + } + + default R visitTimezoneMinute(TimezoneMinute timezoneMinute, C context) { + return visitScalarFunction(timezoneMinute, context); + } + default R visitCos(Cos cos, C context) { return visitScalarFunction(cos, context); } diff --git a/pytest/qe/palo2/src/test_query_datetime_function.py b/pytest/qe/palo2/src/test_query_datetime_function.py index 67f3530b6f745c..e3fe38a8a5c4bb 100644 --- a/pytest/qe/palo2/src/test_query_datetime_function.py +++ b/pytest/qe/palo2/src/test_query_datetime_function.py @@ -1007,6 +1007,37 @@ def test_query_time_convert_tz(): runner.check2(line1, line2) +def test_query_timezone_hour_minute(): + """ + { + "title": "test_query_datetime_function.test_query_timezone_hour_minute", + "describe": "test for timezone_hour and timezone_minute", + "tag": "function,p0" + } + """ + # UTC+08:00 has no DST, the offset of the session timezone is the same + # for every instant, so timezone_hour always returns 8 here. + runner.init("set time_zone = '+08:00'") + ret = runner.get_sql_result( + "select timezone_hour(cast('2024-01-15 12:00:00' as TIMESTAMPTZ)), " + "timezone_minute(cast('2024-07-15 12:00:00' as TIMESTAMPTZ))") + assert int(ret[0][0]) == 8 and int(ret[0][1]) == 0, ret + + # America/New_York switches between EST (UTC-05:00) in winter and + # EDT (UTC-04:00) in summer. + runner.init("set time_zone = 'America/New_York'") + ret = runner.get_sql_result( + "select timezone_hour(cast('2024-01-15 12:00:00' as TIMESTAMPTZ)), " + "timezone_minute(cast('2024-01-15 12:00:00' as TIMESTAMPTZ)), " + "timezone_hour(cast('2024-07-15 12:00:00' as TIMESTAMPTZ)), " + "timezone_minute(cast('2024-07-15 12:00:00' as TIMESTAMPTZ))") + assert int(ret[0][0]) == -5 and int(ret[0][1]) == 0, ret + assert int(ret[0][2]) == -4 and int(ret[0][3]) == 0, ret + + # Restore the default timezone of the test cluster. + runner.init("set time_zone = '+08:00'") + + def test_query_timestampdiff(): """ { From aee93c9b8e88b86434a3819dabc680bdd416a5c4 Mon Sep 17 00:00:00 2001 From: Cao Yu <2320485192@qq.com> Date: Tue, 18 Aug 2026 19:46:58 +0800 Subject: [PATCH 2/3] [fix](function) address review: session-timezone semantics, determinism and pytest connection for timezone_hour/timezone_minute --- .../function_timezone_hour_minute.cpp | 3 ++ be/src/exprs/vectorized_fn_call.cpp | 4 +- .../function_timezone_hour_minute_test.cpp | 38 ++++++++++++++++- .../functions/scalar/TimezoneHour.java | 13 ++++++ .../functions/scalar/TimezoneMinute.java | 13 ++++++ .../palo2/src/test_query_datetime_function.py | 41 +++++++++++++++---- 6 files changed, 101 insertions(+), 11 deletions(-) diff --git a/be/src/exprs/function/function_timezone_hour_minute.cpp b/be/src/exprs/function/function_timezone_hour_minute.cpp index 7509841d8c574b..58d96bf6cb9101 100644 --- a/be/src/exprs/function/function_timezone_hour_minute.cpp +++ b/be/src/exprs/function/function_timezone_hour_minute.cpp @@ -47,6 +47,9 @@ namespace { constexpr int64_t SECONDS_PER_HOUR = 3600; constexpr int64_t SECONDS_PER_MINUTE = 60; +// TIMESTAMPTZ values are stored as UTC instants without the input zone, so the +// offset extracted here is the offset of the session time zone at the instant. +// See TimestampTzValue for the storage design. Status execute_timezone_offset_part(FunctionContext* context, Block& block, const ColumnNumbers& arguments, uint32_t result, size_t input_rows_count, bool extract_hour) { diff --git a/be/src/exprs/vectorized_fn_call.cpp b/be/src/exprs/vectorized_fn_call.cpp index ecbeeedaddbe07..58b5234ba57ea7 100644 --- a/be/src/exprs/vectorized_fn_call.cpp +++ b/be/src/exprs/vectorized_fn_call.cpp @@ -666,7 +666,9 @@ bool VectorizedFnCall::can_push_down_to_index() const { bool VectorizedFnCall::is_deterministic() const { static const std::set NON_DETERMINISTIC_FUNCTIONS = { - "random", "rand", "random_bytes", "uuid", "uuid_numeric"}; + "random", "rand", "random_bytes", "uuid", "uuid_numeric", + // timezone_hour/timezone_minute depend on the session time_zone. + "timezone_hour", "timezone_minute"}; return !NON_DETERMINISTIC_FUNCTIONS.contains(_function_name) && VExpr::is_deterministic(); } diff --git a/be/test/exprs/function/function_timezone_hour_minute_test.cpp b/be/test/exprs/function/function_timezone_hour_minute_test.cpp index 0b6643af2f90e1..49b0fff140f6d9 100644 --- a/be/test/exprs/function/function_timezone_hour_minute_test.cpp +++ b/be/test/exprs/function/function_timezone_hour_minute_test.cpp @@ -63,8 +63,10 @@ class FunctionTimezoneHourMinuteTest : public testing::Test { input_block.insert({nullptr, return_type, "result"}); auto st = func->execute(&context, input_block, arguments, result, input_block.rows()); ASSERT_TRUE(st.ok()) << st.to_string(); - const auto& col = - assert_cast(*input_block.get_by_position(result).column); + // Constant input may produce a const result column; materialize it + // before inspecting elements. + auto result_col = input_block.get_by_position(result).column->convert_to_full_column_if_const(); + const auto& col = assert_cast(*result_col); ASSERT_EQ(col.size(), expected.size()); for (size_t i = 0; i < expected.size(); ++i) { EXPECT_EQ(col.get_element(i), expected[i]) << "at row " << i; @@ -123,6 +125,38 @@ TEST_F(FunctionTimezoneHourMinuteTest, fractional_offsets) { check_result("timezone_minute", block, {45}); } +TEST_F(FunctionTimezoneHourMinuteTest, const_input) { + // TIMESTAMPTZ stores a UTC instant without the input zone; even when the + // value was produced by CAST with an explicit zone (here '2024-01-15 + // 12:00:00-04:30'), the extracted offset is the session zone's offset. + set_session_timezone(cctz::fixed_time_zone(std::chrono::hours(8))); + + auto inner = ColumnTimeStampTz::create(); + inner->insert_value(make_timestamptz(2024, 1, 15, 16, 30, 0, 0)); + auto const_col = ColumnConst::create(std::move(inner), 3); + Block block; + block.insert({std::move(const_col), std::make_shared(), "arg"}); + + check_result("timezone_hour", block, {8, 8, 8}); + check_result("timezone_minute", block, {0, 0, 0}); +} + +TEST_F(FunctionTimezoneHourMinuteTest, session_zone_wins_over_input_zone) { + // The input instant is noon in UTC-04:30, i.e. 16:30 UTC. Trino would + // return -4/-30 from the input zone; Doris stores only the UTC instant + // and therefore returns the session zone offset (America/New_York in + // winter: -5/0). + cctz::time_zone tz; + ASSERT_TRUE(TimezoneUtils::find_cctz_time_zone("America/New_York", tz)); + set_session_timezone(tz); + + auto block = ColumnHelper::create_block( + {make_timestamptz(2024, 1, 15, 16, 30, 0, 0)}); + + check_result("timezone_hour", block, {-5}); + check_result("timezone_minute", block, {0}); +} + TEST_F(FunctionTimezoneHourMinuteTest, nullable_input) { set_session_timezone(cctz::fixed_time_zone(std::chrono::hours(8))); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/TimezoneHour.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/TimezoneHour.java index 6cf2d814a626fc..88c50d7e1388b6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/TimezoneHour.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/TimezoneHour.java @@ -33,6 +33,11 @@ /** * ScalarFunction 'timezone_hour'. + * + *

Returns the hour part of the UTC offset of the session time zone at the + * given instant. Note: Doris TIMESTAMPTZ values are stored as UTC instants + * without the input zone, so unlike Trino's timezone_hour, this function + * extracts the session time zone offset.

*/ public class TimezoneHour extends ScalarFunction implements UnaryExpression, ExplicitlyCastableSignature, PropagateNullable { @@ -70,4 +75,12 @@ public List getSignatures() { public R accept(ExpressionVisitor visitor, C context) { return visitor.visitTimezoneHour(this, context); } + + @Override + public boolean isDeterministic() { + // The result depends on the session time_zone, which may change between + // executions, so this function must not be folded into prepared plans + // or used in materialized views. + return false; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/TimezoneMinute.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/TimezoneMinute.java index e2db92a7ae8432..d792e7c62362af 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/TimezoneMinute.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/TimezoneMinute.java @@ -33,6 +33,11 @@ /** * ScalarFunction 'timezone_minute'. + * + *

Returns the minute part of the UTC offset of the session time zone at the + * given instant. Note: Doris TIMESTAMPTZ values are stored as UTC instants + * without the input zone, so unlike Trino's timezone_minute, this function + * extracts the session time zone offset.

*/ public class TimezoneMinute extends ScalarFunction implements UnaryExpression, ExplicitlyCastableSignature, PropagateNullable { @@ -70,4 +75,12 @@ public List getSignatures() { public R accept(ExpressionVisitor visitor, C context) { return visitor.visitTimezoneMinute(this, context); } + + @Override + public boolean isDeterministic() { + // The result depends on the session time_zone, which may change between + // executions, so this function must not be folded into prepared plans + // or used in materialized views. + return false; + } } diff --git a/pytest/qe/palo2/src/test_query_datetime_function.py b/pytest/qe/palo2/src/test_query_datetime_function.py index e3fe38a8a5c4bb..bcf597d8e79db8 100644 --- a/pytest/qe/palo2/src/test_query_datetime_function.py +++ b/pytest/qe/palo2/src/test_query_datetime_function.py @@ -1015,27 +1015,52 @@ def test_query_timezone_hour_minute(): "tag": "function,p0" } """ + # The SET and the SELECT must run on the same connection, so use + # do_set_properties_sql instead of runner.init (which opens a fresh + # session for every statement). + # UTC+08:00 has no DST, the offset of the session timezone is the same # for every instant, so timezone_hour always returns 8 here. - runner.init("set time_zone = '+08:00'") - ret = runner.get_sql_result( + ret = runner.query_palo.do_set_properties_sql( "select timezone_hour(cast('2024-01-15 12:00:00' as TIMESTAMPTZ)), " - "timezone_minute(cast('2024-07-15 12:00:00' as TIMESTAMPTZ))") + "timezone_minute(cast('2024-07-15 12:00:00' as TIMESTAMPTZ))", + ["set time_zone = '+08:00'"]) assert int(ret[0][0]) == 8 and int(ret[0][1]) == 0, ret # America/New_York switches between EST (UTC-05:00) in winter and # EDT (UTC-04:00) in summer. - runner.init("set time_zone = 'America/New_York'") - ret = runner.get_sql_result( + ret = runner.query_palo.do_set_properties_sql( "select timezone_hour(cast('2024-01-15 12:00:00' as TIMESTAMPTZ)), " "timezone_minute(cast('2024-01-15 12:00:00' as TIMESTAMPTZ)), " "timezone_hour(cast('2024-07-15 12:00:00' as TIMESTAMPTZ)), " - "timezone_minute(cast('2024-07-15 12:00:00' as TIMESTAMPTZ))") + "timezone_minute(cast('2024-07-15 12:00:00' as TIMESTAMPTZ))", + ["set time_zone = 'America/New_York'"]) assert int(ret[0][0]) == -5 and int(ret[0][1]) == 0, ret assert int(ret[0][2]) == -4 and int(ret[0][3]) == 0, ret - # Restore the default timezone of the test cluster. - runner.init("set time_zone = '+08:00'") + # Fractional session offsets are truncated like Trino: Asia/Kolkata is + # UTC+05:30, so timezone_hour returns 5 and timezone_minute returns 30. + ret = runner.query_palo.do_set_properties_sql( + "select timezone_hour(cast('2024-01-15 12:00:00' as TIMESTAMPTZ)), " + "timezone_minute(cast('2024-01-15 12:00:00' as TIMESTAMPTZ))", + ["set time_zone = 'Asia/Kolkata'"]) + assert int(ret[0][0]) == 5 and int(ret[0][1]) == 30, ret + + # Doris TIMESTAMPTZ stores only the UTC instant, not the input zone, so + # even when the input carries '-04:30', the extracted offset is the + # session zone's offset (Trino would return -4/-30 here). + ret = runner.query_palo.do_set_properties_sql( + "select timezone_hour(cast('2024-01-15 12:00:00-04:30' as TIMESTAMPTZ)), " + "timezone_minute(cast('2024-01-15 12:00:00-04:30' as TIMESTAMPTZ))", + ["set time_zone = '+08:00'"]) + assert int(ret[0][0]) == 8 and int(ret[0][1]) == 0, ret + + # NULL input returns NULL. + ret = runner.query_palo.do_set_properties_sql( + "select timezone_hour(cast(null as TIMESTAMPTZ)), " + "timezone_minute(cast(null as TIMESTAMPTZ))", + ["set time_zone = '+08:00'"]) + assert ret[0][0] is None and ret[0][1] is None, ret def test_query_timestampdiff(): From 77bbf781872840146465d21e7cab925dcfa6c651 Mon Sep 17 00:00:00 2001 From: Cao Yu <2320485192@qq.com> Date: Tue, 18 Aug 2026 20:22:24 +0800 Subject: [PATCH 3/3] [fix](function) unwrap nullable before const in timezone_hour/timezone_minute for robustness --- be/src/exprs/function/function_timezone_hour_minute.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/be/src/exprs/function/function_timezone_hour_minute.cpp b/be/src/exprs/function/function_timezone_hour_minute.cpp index 58d96bf6cb9101..73fac5ec0d129a 100644 --- a/be/src/exprs/function/function_timezone_hour_minute.cpp +++ b/be/src/exprs/function/function_timezone_hour_minute.cpp @@ -54,10 +54,14 @@ Status execute_timezone_offset_part(FunctionContext* context, Block& block, const ColumnNumbers& arguments, uint32_t result, size_t input_rows_count, bool extract_hour) { ColumnPtr col = block.get_by_position(arguments[0]).column; + // Unwrap nullable and const wrappers in any nesting order so that + // ColumnNullable(ColumnConst(...)) and ColumnConst(ColumnNullable(...)) + // inputs both reach the plain ColumnTimeStampTz data below. + col = remove_nullable(col); if (is_column_const(*col)) { col = assert_cast(*col).convert_to_full_column(); + col = remove_nullable(col); } - col = remove_nullable(col); const auto* tz_column = assert_cast(col.get()); const auto& tz_data = tz_column->get_data();