From a710cd39a1808f18c7f0eadc40401aa9c18fea4a Mon Sep 17 00:00:00 2001 From: Bolin Lin Date: Tue, 19 May 2026 21:31:08 -0400 Subject: [PATCH 1/5] feat: add CometConvertTimezone --- .../apache/comet/serde/QueryPlanSerde.scala | 1 + .../org/apache/comet/serde/datetime.scala | 23 ++++++++++++++++++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala index d85a2c30cb..296b521104 100644 --- a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala +++ b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala @@ -216,6 +216,7 @@ object QueryPlanSerde extends Logging with CometExprShim with CometTypeShim { private[comet] val temporalExpressions: Map[Class[_ <: Expression], CometExpressionSerde[_]] = Map( + classOf[ConvertTimezone] -> CometConvertTimezone, classOf[DateAdd] -> CometDateAdd, classOf[DateDiff] -> CometDateDiff, classOf[DateFormatClass] -> CometDateFormat, diff --git a/spark/src/main/scala/org/apache/comet/serde/datetime.scala b/spark/src/main/scala/org/apache/comet/serde/datetime.scala index 3192697b26..65ba1beccb 100644 --- a/spark/src/main/scala/org/apache/comet/serde/datetime.scala +++ b/spark/src/main/scala/org/apache/comet/serde/datetime.scala @@ -21,7 +21,7 @@ package org.apache.comet.serde import java.util.Locale -import org.apache.spark.sql.catalyst.expressions.{Attribute, DateAdd, DateDiff, DateFormatClass, DateFromUnixDate, DateSub, DayOfMonth, DayOfWeek, DayOfYear, Days, FromUTCTimestamp, GetDateField, Hour, Hours, LastDay, Literal, MakeDate, Minute, Month, NextDay, Quarter, Second, SecondsToTimestamp, ToUTCTimestamp, TruncDate, TruncTimestamp, UnixDate, UnixTimestamp, WeekDay, WeekOfYear, Year} +import org.apache.spark.sql.catalyst.expressions.{Attribute, ConvertTimezone, DateAdd, DateDiff, DateFormatClass, DateFromUnixDate, DateSub, DayOfMonth, DayOfWeek, DayOfYear, Days, FromUTCTimestamp, GetDateField, Hour, Hours, LastDay, Literal, MakeDate, Minute, Month, NextDay, Quarter, Second, SecondsToTimestamp, ToUTCTimestamp, TruncDate, TruncTimestamp, UnixDate, UnixTimestamp, WeekDay, WeekOfYear, Year} import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{DateType, DoubleType, FloatType, IntegerType, LongType, StringType, TimestampNTZType, TimestampType} import org.apache.spark.unsafe.types.UTF8String @@ -409,6 +409,27 @@ object CometToUTCTimestamp extends CometExpressionSerde[ToUTCTimestamp] { } } +object CometConvertTimezone extends CometExpressionSerde[ConvertTimezone] { + + override def getSupportLevel(expr: ConvertTimezone): SupportLevel = + Incompatible(Some(UTCTimestampSerde.tzParseIncompatReason)) + + override def getIncompatibleReasons(): Seq[String] = + Seq(UTCTimestampSerde.tzParseIncompatReason) + + override def convert( + expr: ConvertTimezone, + inputs: Seq[Attribute], + binding: Boolean): Option[ExprOuterClass.Expr] = { + val srcTz = exprToProtoInternal(expr.sourceTz, inputs, binding) + val tgtTz = exprToProtoInternal(expr.targetTz, inputs, binding) + val ts = exprToProtoInternal(expr.sourceTs, inputs, binding) + val toUtc = scalarFunctionExprToProto("to_utc_timestamp", ts, srcTz) + val fromUtc = scalarFunctionExprToProto("from_utc_timestamp", toUtc, tgtTz) + optExprWitahInfo(fromUtc, expr, expr.children: _*) + } +} + object CometNextDay extends CometScalarFunction[NextDay]("next_day") object CometMakeDate extends CometScalarFunction[MakeDate]("make_date") From fe1d46088598c3218806bcea04ac45c2f463649f Mon Sep 17 00:00:00 2001 From: Bolin Lin Date: Tue, 19 May 2026 21:31:50 -0400 Subject: [PATCH 2/5] doc: mark convert_timezone --- docs/source/contributor-guide/spark_expressions_support.md | 2 +- docs/source/user-guide/latest/expressions.md | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/source/contributor-guide/spark_expressions_support.md b/docs/source/contributor-guide/spark_expressions_support.md index 0efc581d55..362b833a21 100644 --- a/docs/source/contributor-guide/spark_expressions_support.md +++ b/docs/source/contributor-guide/spark_expressions_support.md @@ -215,7 +215,7 @@ ### datetime_funcs - [ ] add_months -- [ ] convert_timezone +- [x] convert_timezone - [ ] curdate - [ ] current_date - [ ] current_time diff --git a/docs/source/user-guide/latest/expressions.md b/docs/source/user-guide/latest/expressions.md index 9bfec5b722..f5d83ecc6c 100644 --- a/docs/source/user-guide/latest/expressions.md +++ b/docs/source/user-guide/latest/expressions.md @@ -101,6 +101,7 @@ of expressions that be disabled. | Expression | SQL | | ---------------- | ---------------------------- | +| ConvertTimezone | `convert_timezone` | | CurrentTimeZone | `current_timezone` | | DateAdd | `date_add` | | DateDiff | `datediff` | From 793f41be075770c266536c95f85616c6f8a2441b Mon Sep 17 00:00:00 2001 From: Bolin Lin Date: Tue, 19 May 2026 21:42:39 -0400 Subject: [PATCH 3/5] test: convert timezone --- .../expressions/datetime/convert_timezone.sql | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 spark/src/test/resources/sql-tests/expressions/datetime/convert_timezone.sql diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/convert_timezone.sql b/spark/src/test/resources/sql-tests/expressions/datetime/convert_timezone.sql new file mode 100644 index 0000000000..44463ebcaf --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/datetime/convert_timezone.sql @@ -0,0 +1,41 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- Config: spark.comet.expression.ConvertTimezone.allowIncompatible=true +-- ConfigMatrix: spark.sql.session.timeZone=UTC,America/Los_Angeles + +statement +CREATE TABLE test_convert_timezone(ts timestamp_ntz, src string, tgt string) USING parquet + +statement +INSERT INTO test_convert_timezone VALUES + (timestamp_ntz'2021-12-06 08:00:00', 'UTC', 'America/Los_Angeles'), + (timestamp_ntz'2021-07-01 12:00:00', 'America/New_York', 'Asia/Tokyo'), + (timestamp_ntz'2023-01-15 09:30:00', 'America/Los_Angeles', 'UTC'), + (NULL, 'UTC', 'Asia/Tokyo') + +query +SELECT convert_timezone('UTC', 'America/Los_Angeles', timestamp_ntz'2021-12-06 08:00:00') + +query +SELECT convert_timezone('Asia/Tokyo', 'Europe/Berlin', timestamp_ntz'2021-12-06 12:00:00') + +query +SELECT convert_timezone('America/Los_Angeles', 'Asia/Tokyo', timestamp_ntz'2023-01-15 20:00:00') + +query +SELECT convert_timezone(src, tgt, ts) FROM test_convert_timezone From 31b649f2d8220a37466b7b2a3dc064c25beead52 Mon Sep 17 00:00:00 2001 From: Bolin Lin Date: Tue, 19 May 2026 23:33:15 -0400 Subject: [PATCH 4/5] fix: typo --- spark/src/main/scala/org/apache/comet/serde/datetime.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spark/src/main/scala/org/apache/comet/serde/datetime.scala b/spark/src/main/scala/org/apache/comet/serde/datetime.scala index 65ba1beccb..b57b1e4e56 100644 --- a/spark/src/main/scala/org/apache/comet/serde/datetime.scala +++ b/spark/src/main/scala/org/apache/comet/serde/datetime.scala @@ -426,7 +426,7 @@ object CometConvertTimezone extends CometExpressionSerde[ConvertTimezone] { val ts = exprToProtoInternal(expr.sourceTs, inputs, binding) val toUtc = scalarFunctionExprToProto("to_utc_timestamp", ts, srcTz) val fromUtc = scalarFunctionExprToProto("from_utc_timestamp", toUtc, tgtTz) - optExprWitahInfo(fromUtc, expr, expr.children: _*) + optExprWithInfo(fromUtc, expr, expr.children: _*) } } From df972bdffef456f71139d3f37881c40c39dcc589 Mon Sep 17 00:00:00 2001 From: Bolin Lin Date: Wed, 20 May 2026 11:37:43 -0400 Subject: [PATCH 5/5] test: add test case for source and target timezone --- .../expressions/datetime/convert_timezone.sql | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/convert_timezone.sql b/spark/src/test/resources/sql-tests/expressions/datetime/convert_timezone.sql index 44463ebcaf..6c9a8947b5 100644 --- a/spark/src/test/resources/sql-tests/expressions/datetime/convert_timezone.sql +++ b/spark/src/test/resources/sql-tests/expressions/datetime/convert_timezone.sql @@ -26,7 +26,9 @@ INSERT INTO test_convert_timezone VALUES (timestamp_ntz'2021-12-06 08:00:00', 'UTC', 'America/Los_Angeles'), (timestamp_ntz'2021-07-01 12:00:00', 'America/New_York', 'Asia/Tokyo'), (timestamp_ntz'2023-01-15 09:30:00', 'America/Los_Angeles', 'UTC'), - (NULL, 'UTC', 'Asia/Tokyo') + (NULL, 'UTC', 'Asia/Tokyo'), + (timestamp_ntz'2021-12-06 08:00:00', NULL, 'Asia/Tokyo'), + (timestamp_ntz'2021-12-06 08:00:00', 'UTC', NULL) query SELECT convert_timezone('UTC', 'America/Los_Angeles', timestamp_ntz'2021-12-06 08:00:00') @@ -37,5 +39,11 @@ SELECT convert_timezone('Asia/Tokyo', 'Europe/Berlin', timestamp_ntz'2021-12-06 query SELECT convert_timezone('America/Los_Angeles', 'Asia/Tokyo', timestamp_ntz'2023-01-15 20:00:00') +query +SELECT convert_timezone(CAST(NULL AS STRING), 'Asia/Tokyo', timestamp_ntz'2021-12-06 08:00:00') + +query +SELECT convert_timezone('UTC', CAST(NULL AS STRING), timestamp_ntz'2021-12-06 08:00:00') + query SELECT convert_timezone(src, tgt, ts) FROM test_convert_timezone