From a45bb87851796e94b8bc23ffc8ece6315dff6f0c Mon Sep 17 00:00:00 2001 From: Dustin Smith Date: Fri, 11 Sep 2026 22:46:06 +0700 Subject: [PATCH] feat: route date and timestamp interval arithmetic through codegen dispatch Spark's date and timestamp interval arithmetic had no serde, so any projection using it fell back to Spark. Register the six Catalyst classes as codegen-dispatch serdes, with the version-specific TimeAdd and TimestampAddInterval registered through the shims, and cover them with SQL-file fixtures over parquet tables, including both legacy interval modes, DST rows, month-end clamping and native shuffle. Closes #3094 Closes #3112 Closes #3086 Closes #3115 Closes #3114 --- docs/source/user-guide/latest/expressions.md | 4 +- .../apache/comet/serde/QueryPlanSerde.scala | 5 + .../org/apache/comet/serde/datetime.scala | 33 ++++- .../apache/comet/shims/CometExprShim.scala | 4 +- .../apache/comet/shims/CometExprShim.scala | 6 +- .../apache/comet/shims/CometExprShim.scala | 9 +- .../serde/CometTimestampAddInterval.scala | 28 +++++ .../apache/comet/shims/CometExprShim.scala | 7 ++ .../datetime/date_add_interval.sql | 84 +++++++++++++ .../datetime/date_add_interval_ansi.sql | 47 ++++++++ .../date_add_interval_ansi_spark35.sql | 47 ++++++++ .../datetime/date_add_ym_interval.sql | 71 +++++++++++ .../expressions/datetime/subtract_dates.sql | 65 ++++++++++ .../datetime/subtract_timestamps.sql | 98 +++++++++++++++ .../datetime/subtract_timestamps_legacy.sql | 87 ++++++++++++++ .../subtract_timestamps_long_span.sql | 46 +++++++ .../subtract_timestamps_long_span_legacy.sql | 47 ++++++++ .../datetime/timestamp_add_interval.sql | 113 ++++++++++++++++++ .../datetime/timestamp_add_ym_interval.sql | 84 +++++++++++++ 19 files changed, 877 insertions(+), 8 deletions(-) create mode 100644 spark/src/main/spark-4.1+/org/apache/comet/serde/CometTimestampAddInterval.scala create mode 100644 spark/src/test/resources/sql-tests/expressions/datetime/date_add_interval.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/datetime/date_add_interval_ansi.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/datetime/date_add_interval_ansi_spark35.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/datetime/date_add_ym_interval.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/datetime/subtract_dates.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/datetime/subtract_timestamps.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/datetime/subtract_timestamps_legacy.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/datetime/subtract_timestamps_long_span.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/datetime/subtract_timestamps_long_span_legacy.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/datetime/timestamp_add_interval.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/datetime/timestamp_add_ym_interval.sql diff --git a/docs/source/user-guide/latest/expressions.md b/docs/source/user-guide/latest/expressions.md index 208a5f3f124..e96fc2eb727 100644 --- a/docs/source/user-guide/latest/expressions.md +++ b/docs/source/user-guide/latest/expressions.md @@ -415,8 +415,8 @@ The type-name conversion functions (`bigint`, `binary`, `boolean`, `date`, `deci | --- | --- | --- | --- | | `%` | ✅ | Native | | | `*` | ✅ | Native | DayTime interval multiplication routes through the JVM codegen dispatcher; YearMonth and Calendar interval multiplication fall back | -| `+` | ✅ | Native | | -| `-` | ✅ | Native | | +| `+` | ✅ | Native | Adding a calendar, year-month or day-time interval to a date or timestamp routes through the JVM codegen dispatcher | +| `-` | ✅ | Native | `date - date`, `timestamp - timestamp` and subtracting an interval from a date or timestamp route through the JVM codegen dispatcher; `timestamp - timestamp` falls back to Spark in legacy interval mode (`spark.sql.legacy.interval.enabled=true`) because its calendar-interval result can exceed what the dispatcher output can carry | | `/` | ✅ | Native | | | `abs` | ✅ | Hybrid | Interval types route through the JVM codegen dispatcher; numeric types run natively | | `acos` | ✅ | Native | | diff --git a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala index be4bc9c3412..25350245c29 100644 --- a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala +++ b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala @@ -307,6 +307,11 @@ object QueryPlanSerde extends Logging with CometExprShim with CometTypeShim { classOf[MultiplyDTInterval] -> CometMultiplyDTInterval, classOf[TimestampAdd] -> CometTimestampAdd, classOf[TimestampDiff] -> CometTimestampDiff, + classOf[DateAddInterval] -> CometDateAddInterval, + classOf[DateAddYMInterval] -> CometDateAddYMInterval, + classOf[TimestampAddYMInterval] -> CometTimestampAddYMInterval, + classOf[SubtractDates] -> CometSubtractDates, + classOf[SubtractTimestamps] -> CometSubtractTimestamps, classOf[MicrosToTimestamp] -> CometMicrosToTimestamp, classOf[MillisToTimestamp] -> CometMillisToTimestamp, classOf[MonthsBetween] -> CometMonthsBetween, diff --git a/spark/src/main/scala/org/apache/comet/serde/datetime.scala b/spark/src/main/scala/org/apache/comet/serde/datetime.scala index 8f2b9ab6719..629d999868e 100644 --- a/spark/src/main/scala/org/apache/comet/serde/datetime.scala +++ b/spark/src/main/scala/org/apache/comet/serde/datetime.scala @@ -21,7 +21,7 @@ package org.apache.comet.serde import java.util.Locale -import org.apache.spark.sql.catalyst.expressions.{AddMonths, Attribute, Cast, ConvertTimezone, DateAdd, DateDiff, DateFormatClass, DateFromUnixDate, DateSub, DayOfMonth, DayOfWeek, DayOfYear, Days, Expression, FromUTCTimestamp, GetDateField, GetTimestamp, Hour, Hours, LastDay, Literal, MakeDate, MakeDTInterval, MakeInterval, MakeTimestamp, MakeYMInterval, MicrosToTimestamp, MillisToTimestamp, Minute, Month, MonthsBetween, MultiplyDTInterval, NextDay, PreciseTimestampConversion, Quarter, Second, SecondsToTimestamp, TimestampAdd, TimestampDiff, ToUnixTimestamp, ToUTCTimestamp, TruncDate, TruncTimestamp, UnixDate, UnixMicros, UnixMillis, UnixSeconds, UnixTimestamp, WeekDay, WeekOfYear, Year} +import org.apache.spark.sql.catalyst.expressions.{AddMonths, Attribute, Cast, ConvertTimezone, DateAdd, DateAddInterval, DateAddYMInterval, DateDiff, DateFormatClass, DateFromUnixDate, DateSub, DayOfMonth, DayOfWeek, DayOfYear, Days, Expression, FromUTCTimestamp, GetDateField, GetTimestamp, Hour, Hours, LastDay, Literal, MakeDate, MakeDTInterval, MakeInterval, MakeTimestamp, MakeYMInterval, MicrosToTimestamp, MillisToTimestamp, Minute, Month, MonthsBetween, MultiplyDTInterval, NextDay, PreciseTimestampConversion, Quarter, Second, SecondsToTimestamp, SubtractDates, SubtractTimestamps, TimestampAdd, TimestampAddYMInterval, TimestampDiff, ToUnixTimestamp, ToUTCTimestamp, TruncDate, TruncTimestamp, UnixDate, UnixMicros, UnixMillis, UnixSeconds, UnixTimestamp, WeekDay, WeekOfYear, Year} import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{CalendarIntervalType, DataType, DateType, DoubleType, FloatType, IntegerType, LongType, StringType, TimestampNTZType, TimestampType} import org.apache.spark.unsafe.types.UTF8String @@ -997,6 +997,37 @@ object CometTimestampAdd extends CometCodegenDispatch[TimestampAdd] object CometTimestampDiff extends CometCodegenDispatch[TimestampDiff] +// Date and timestamp interval arithmetic. `timestamp + day-time or calendar interval` resolves +// to `TimeAdd` on Spark 3.4 through 4.0 and to `TimestampAddInterval` on 4.1+, so that serde +// lives in the version shims. +object CometDateAddInterval extends CometCodegenDispatch[DateAddInterval] + +object CometDateAddYMInterval extends CometCodegenDispatch[DateAddYMInterval] + +object CometTimestampAddYMInterval extends CometCodegenDispatch[TimestampAddYMInterval] + +object CometSubtractDates extends CometCodegenDispatch[SubtractDates] + +object CometSubtractTimestamps extends CometCodegenDispatch[SubtractTimestamps] { + private val legacyIntervalReason = + "In legacy interval mode (`spark.sql.legacy.interval.enabled=true`) the result is a" + + " `CalendarIntervalType`, and the JVM codegen dispatcher's calendar-interval output" + + " cannot carry a span past about 292 years (see" + + " https://github.com/apache/datafusion-comet/issues/5279), so the expression falls" + + " back to Spark" + + override def getUnsupportedReasons(): Seq[String] = Seq(legacyIntervalReason) + + // Same `Math.multiplyExact(microseconds, 1000L)` limit `CometMakeInterval` documents as a + // compatible note. That one only overflows on extreme arguments; `ts - ts` produces an + // arbitrary span from ordinary data, and legacy mode is off by default, so decline it. + // Remove this branch once #5279 carries CalendarInterval across the boundary losslessly. + override def getSupportLevel(expr: SubtractTimestamps): SupportLevel = expr.dataType match { + case CalendarIntervalType => Unsupported(Some(legacyIntervalReason)) + case _ => Compatible() + } +} + /** * Spark's internal `PreciseTimestampConversion` reinterprets a value between the timestamp types * (`TimestampType` / `TimestampNTZType`) and `LongType` without losing microsecond precision. It diff --git a/spark/src/main/spark-3.4/org/apache/comet/shims/CometExprShim.scala b/spark/src/main/spark-3.4/org/apache/comet/shims/CometExprShim.scala index 594c85e02fd..c101e3d3008 100644 --- a/spark/src/main/spark-3.4/org/apache/comet/shims/CometExprShim.scala +++ b/spark/src/main/spark-3.4/org/apache/comet/shims/CometExprShim.scala @@ -23,7 +23,7 @@ import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.expressions.aggregate.Sum import org.apache.comet.expressions.CometEvalMode -import org.apache.comet.serde.{CometEncode, CometExpressionSerde, CometStringDecode} +import org.apache.comet.serde.{CometCodegenDispatch, CometEncode, CometExpressionSerde, CometStringDecode} import org.apache.comet.serde.ExprOuterClass.{BinaryOutputStyle, Expr} /** @@ -41,7 +41,7 @@ trait CometExprShim { def sparkVersionSpecificMathExpressions: Map[Class[_ <: Expression], CometExpressionSerde[_]] = Map.empty def sparkVersionSpecificMiscExpressions: Map[Class[_ <: Expression], CometExpressionSerde[_]] = - Map.empty + Map(classOf[TimeAdd] -> new CometCodegenDispatch[TimeAdd]) def sparkVersionSpecificMapExpressions: Map[Class[_ <: Expression], CometExpressionSerde[_]] = Map.empty diff --git a/spark/src/main/spark-3.5/org/apache/comet/shims/CometExprShim.scala b/spark/src/main/spark-3.5/org/apache/comet/shims/CometExprShim.scala index 0bfea5cd6ec..4a6cb67bfc6 100644 --- a/spark/src/main/spark-3.5/org/apache/comet/shims/CometExprShim.scala +++ b/spark/src/main/spark-3.5/org/apache/comet/shims/CometExprShim.scala @@ -23,7 +23,7 @@ import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.expressions.aggregate.Sum import org.apache.comet.expressions.CometEvalMode -import org.apache.comet.serde.{CometEncode, CometExpressionSerde, CometStringDecode, CometToPrettyString, CometWidthBucket} +import org.apache.comet.serde.{CometCodegenDispatch, CometEncode, CometExpressionSerde, CometStringDecode, CometToPrettyString, CometWidthBucket} import org.apache.comet.serde.ExprOuterClass.{BinaryOutputStyle, Expr} /** @@ -41,7 +41,9 @@ trait CometExprShim { def sparkVersionSpecificMathExpressions: Map[Class[_ <: Expression], CometExpressionSerde[_]] = Map(classOf[WidthBucket] -> CometWidthBucket) def sparkVersionSpecificMiscExpressions: Map[Class[_ <: Expression], CometExpressionSerde[_]] = - Map(classOf[ToPrettyString] -> CometToPrettyString) + Map( + classOf[ToPrettyString] -> CometToPrettyString, + classOf[TimeAdd] -> new CometCodegenDispatch[TimeAdd]) def sparkVersionSpecificMapExpressions: Map[Class[_ <: Expression], CometExpressionSerde[_]] = Map.empty diff --git a/spark/src/main/spark-4.0/org/apache/comet/shims/CometExprShim.scala b/spark/src/main/spark-4.0/org/apache/comet/shims/CometExprShim.scala index ca3c79930f3..4903f0b8c0a 100644 --- a/spark/src/main/spark-4.0/org/apache/comet/shims/CometExprShim.scala +++ b/spark/src/main/spark-4.0/org/apache/comet/shims/CometExprShim.scala @@ -19,11 +19,12 @@ package org.apache.comet.shims -import org.apache.spark.sql.catalyst.expressions.EvalMode +import org.apache.spark.sql.catalyst.expressions.{EvalMode, Expression, TimeAdd} import org.apache.spark.sql.catalyst.expressions.aggregate.Sum import org.apache.spark.sql.internal.SQLConf import org.apache.comet.expressions.CometEvalMode +import org.apache.comet.serde.{CometCodegenDispatch, CometExpressionSerde} import org.apache.comet.serde.ExprOuterClass.BinaryOutputStyle /** @@ -42,6 +43,12 @@ trait CometExprShim extends Spark4xCometExprShim { case _ => BinaryOutputStyle.HEX_DISCRETE } } + + // Spark 4.0 still spells `timestamp + interval` as `TimeAdd`; 4.1 renames it. + override def sparkVersionSpecificMiscExpressions + : Map[Class[_ <: Expression], CometExpressionSerde[_]] = + super.sparkVersionSpecificMiscExpressions + (classOf[TimeAdd] -> new CometCodegenDispatch[ + TimeAdd]) } object CometEvalModeUtil { diff --git a/spark/src/main/spark-4.1+/org/apache/comet/serde/CometTimestampAddInterval.scala b/spark/src/main/spark-4.1+/org/apache/comet/serde/CometTimestampAddInterval.scala new file mode 100644 index 00000000000..1f93b5ac3ec --- /dev/null +++ b/spark/src/main/spark-4.1+/org/apache/comet/serde/CometTimestampAddInterval.scala @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet.serde + +import org.apache.spark.sql.catalyst.expressions.TimestampAddInterval + +/** + * `timestamp + day-time or calendar interval` resolves to `TimestampAddInterval` on Spark 4.1+ + * (`TimeAdd` on earlier versions) and runs through the codegen dispatcher. + */ +object CometTimestampAddInterval extends CometCodegenDispatch[TimestampAddInterval] diff --git a/spark/src/main/spark-4.1+/org/apache/comet/shims/CometExprShim.scala b/spark/src/main/spark-4.1+/org/apache/comet/shims/CometExprShim.scala index a2380e4008a..bcebcbae03a 100644 --- a/spark/src/main/spark-4.1+/org/apache/comet/shims/CometExprShim.scala +++ b/spark/src/main/spark-4.1+/org/apache/comet/shims/CometExprShim.scala @@ -27,6 +27,7 @@ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.TimeType import org.apache.comet.expressions.CometEvalMode +import org.apache.comet.serde.{CometExpressionSerde, CometTimestampAddInterval} import org.apache.comet.serde.ExprOuterClass.{BinaryOutputStyle, Expr} import org.apache.comet.serde.QueryPlanSerde.{exprToProtoInternal, scalarFunctionExprToProtoWithReturnType} @@ -46,6 +47,12 @@ trait CometExprShim extends Spark4xCometExprShim { } } + // Spark 4.1 renames `TimeAdd` to `TimestampAddInterval`. + override def sparkVersionSpecificMiscExpressions + : Map[Class[_ <: Expression], CometExpressionSerde[_]] = + super.sparkVersionSpecificMiscExpressions + + (classOf[TimestampAddInterval] -> CometTimestampAddInterval) + // Spark 4.1 introduced TimeType and the make_time / to_time / try_to_time functions. // Their planner forms differ from the shared 4.x patterns (DateTimeUtils.makeTime // StaticInvoke and ToTimeParser Invoke / TryEval(Invoke)), so they live here rather diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/date_add_interval.sql b/spark/src/test/resources/sql-tests/expressions/datetime/date_add_interval.sql new file mode 100644 index 00000000000..32e46fcac43 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/datetime/date_add_interval.sql @@ -0,0 +1,84 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- date + calendar interval resolves to DateAddInterval and runs through the codegen dispatcher +-- so results match Spark exactly. With ANSI off, an interval carrying a time part is applied +-- on the timestamp and the result truncated back to a date; the ANSI error case lives in +-- date_add_interval_ansi.sql. America/Los_Angeles is pinned so the 25-hour row crosses DST. +-- Config: spark.sql.session.timeZone=America/Los_Angeles +-- Config: spark.comet.exec.scalaUDF.codegen.enabled=true +-- Config: spark.comet.shuffle.mode=native + +statement +CREATE TABLE test_date_add_interval(d date, y int, m int, dd int, h int, k int) USING parquet + +statement +INSERT INTO test_date_add_interval VALUES + (date'2024-01-31', 0, 1, 0, 0, 1), + (date'2024-01-31', 1, 1, 1, 0, 1), + (date'2024-02-29', 1, 0, 0, 0, 2), + (date'2024-03-31', 0, -1, 0, 0, 2), + (date'2024-12-31', 0, 0, 1, 0, 3), + (date'2024-03-09', 0, 0, 1, 0, 3), + (date'2024-03-09', 0, 0, 0, 25, 4), + (date'1970-01-01', -1, -1, -1, -1, 4), + (date'2024-06-15', NULL, 1, 1, 0, 5), + (date'2024-06-15', 0, NULL, 1, 0, 5), + (date'2024-06-15', 0, 0, NULL, 0, 6), + (date'2024-06-15', 0, 0, 0, NULL, 6), + (NULL, 1, 1, 1, 0, 7) + +-- column date plus a calendar interval built from columns. Month arithmetic clamps to the end +-- of the shorter month before the day part is added. +query +SELECT d, y, m, dd, d + make_interval(y, m, 0, dd) FROM test_date_add_interval + +-- interval on the left +query +SELECT make_interval(y, m, 0, dd) + d FROM test_date_add_interval + +-- with ANSI off the hour part is applied on the timestamp and truncated away again +query +SELECT d, h, d + make_interval(y, m, 0, dd, h) FROM test_date_add_interval + +-- The parser rejects interval literals that mix year-month and day-time units unless +-- spark.sql.legacy.interval.enabled is set, so literal calendar intervals come from +-- make_interval. Subtraction rewrites to an addition of the negated interval. +query +SELECT + d + make_interval(1, 0, 0, 1), + d + make_interval(-1, 0, 0, -1), + d + make_interval(0, 1, 0, 1), + d + make_interval(0, 1, 0, 1, 12), + d - make_interval(1, 0, 0, 1), + d - make_interval(0, 1, 0, 1) +FROM test_date_add_interval + +-- all-literal operands (constant folding is disabled by the test suite) +query +SELECT + date'2024-01-31' + make_interval(0, 1, 0, 1), + date'2024-02-29' + make_interval(1, 0, 0, 1), + date'2024-01-31' - make_interval(0, 1, 0, 1), + CAST(NULL AS DATE) + make_interval(0, 1, 0, 1), + date'2024-01-31' + CAST(NULL AS INTERVAL) + +-- date output through native shuffle +query +SELECT k, d + make_interval(y, m, 0, dd) AS r +FROM test_date_add_interval +DISTRIBUTE BY k diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/date_add_interval_ansi.sql b/spark/src/test/resources/sql-tests/expressions/datetime/date_add_interval_ansi.sql new file mode 100644 index 00000000000..bb4913d9eb5 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/datetime/date_add_interval_ansi.sql @@ -0,0 +1,47 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- With ANSI on, DateAddInterval keeps day-granular intervals on the date path and rejects an +-- interval that carries hours, minutes, seconds or fractions of a second. The parser rejects +-- interval literals that mix year-month and day-time units, so make_interval builds them. +-- Config: spark.sql.ansi.enabled=true +-- Config: spark.comet.exec.scalaUDF.codegen.enabled=true +-- MinSparkVersion: 4.0 + +statement +CREATE TABLE test_date_add_interval_ansi(d date, m int, dd int) USING parquet + +statement +INSERT INTO test_date_add_interval_ansi VALUES + (date'2024-01-31', 1, 1), + (date'2024-02-29', 12, 0), + (date'2024-03-31', -1, -1), + (date'2024-06-15', NULL, 1), + (NULL, 1, 1) + +-- sentinel: a day-granular interval succeeds and asserts native execution +query +SELECT d, m, dd, d + make_interval(0, m, 0, dd), d - make_interval(0, 1, 0, 1) +FROM test_date_add_interval_ansi + +-- a NULL interval yields NULL rather than an error +query +SELECT d + CAST(NULL AS INTERVAL) FROM test_date_add_interval_ansi + +-- an interval with a time part is rejected +query expect_error(INVALID_INTERVAL_WITH_MICROSECONDS_ADDITION) +SELECT d + make_interval(0, 1, 0, 1, 12) FROM test_date_add_interval_ansi diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/date_add_interval_ansi_spark35.sql b/spark/src/test/resources/sql-tests/expressions/datetime/date_add_interval_ansi_spark35.sql new file mode 100644 index 00000000000..100dd707725 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/datetime/date_add_interval_ansi_spark35.sql @@ -0,0 +1,47 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- With ANSI on, DateAddInterval keeps day-granular intervals on the date path and rejects an +-- interval that carries hours, minutes, seconds or fractions of a second. The parser rejects +-- interval literals that mix year-month and day-time units, so make_interval builds them. +-- Config: spark.sql.ansi.enabled=true +-- Config: spark.comet.exec.scalaUDF.codegen.enabled=true +-- MaxSparkVersion: 3.5 + +statement +CREATE TABLE test_date_add_interval_ansi(d date, m int, dd int) USING parquet + +statement +INSERT INTO test_date_add_interval_ansi VALUES + (date'2024-01-31', 1, 1), + (date'2024-02-29', 12, 0), + (date'2024-03-31', -1, -1), + (date'2024-06-15', NULL, 1), + (NULL, 1, 1) + +-- sentinel: a day-granular interval succeeds and asserts native execution +query +SELECT d, m, dd, d + make_interval(0, m, 0, dd), d - make_interval(0, 1, 0, 1) +FROM test_date_add_interval_ansi + +-- a NULL interval yields NULL rather than an error +query +SELECT d + CAST(NULL AS INTERVAL) FROM test_date_add_interval_ansi + +-- an interval with a time part is rejected +query expect_error(Cannot add hours, minutes or seconds, milliseconds, microseconds to a date) +SELECT d + make_interval(0, 1, 0, 1, 12) FROM test_date_add_interval_ansi diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/date_add_ym_interval.sql b/spark/src/test/resources/sql-tests/expressions/datetime/date_add_ym_interval.sql new file mode 100644 index 00000000000..094aeeeff61 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/datetime/date_add_ym_interval.sql @@ -0,0 +1,71 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- date + year-month interval resolves to DateAddYMInterval and runs through the codegen +-- dispatcher so results match Spark exactly. Month arithmetic clamps to the last day of the +-- target month. +-- Config: spark.comet.exec.scalaUDF.codegen.enabled=true +-- Config: spark.comet.shuffle.mode=native + +statement +CREATE TABLE test_date_add_ym_interval(d date, y int, m int, k int) USING parquet + +statement +INSERT INTO test_date_add_ym_interval VALUES + (date'2024-01-31', 0, 1, 1), + (date'2024-01-31', 0, 13, 1), + (date'2024-02-29', 1, 0, 2), + (date'2024-02-29', 4, 0, 2), + (date'2024-03-31', 0, -1, 3), + (date'2024-12-31', 0, 2, 3), + (date'1970-01-01', -1, -1, 4), + (date'2024-06-15', NULL, 1, 5), + (date'2024-06-15', 1, NULL, 5), + (NULL, 1, 1, 6) + +-- column date plus an interval built from columns, both directions +query +SELECT d, y, m, d + make_ym_interval(y, m), make_ym_interval(y, m) + d +FROM test_date_add_ym_interval + +-- interval literals in the unit, unit-to-unit and multi-unit spellings. Subtraction rewrites +-- to an addition of the negated interval. +query +SELECT + d + INTERVAL '1' YEAR, + d + INTERVAL '1' MONTH, + d + INTERVAL '1-2' YEAR TO MONTH, + d + INTERVAL '1 year 1 month', + d + INTERVAL '-1' MONTH, + d - INTERVAL '1' MONTH, + d - INTERVAL '1' YEAR +FROM test_date_add_ym_interval + +-- all-literal operands (constant folding is disabled by the test suite) +query +SELECT + date'2024-01-31' + INTERVAL '1' MONTH, + date'2024-02-29' + INTERVAL '1' YEAR, + date'2024-03-31' - INTERVAL '1' MONTH, + CAST(NULL AS DATE) + INTERVAL '1' MONTH, + date'2024-01-31' + CAST(NULL AS INTERVAL YEAR TO MONTH) + +-- date output through native shuffle +query +SELECT k, d + make_ym_interval(y, m) AS r +FROM test_date_add_ym_interval +DISTRIBUTE BY k diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/subtract_dates.sql b/spark/src/test/resources/sql-tests/expressions/datetime/subtract_dates.sql new file mode 100644 index 00000000000..26ea41a011b --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/datetime/subtract_dates.sql @@ -0,0 +1,65 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- date - date resolves to SubtractDates and runs through the codegen dispatcher so results +-- match Spark exactly. The output type follows spark.sql.legacy.interval.enabled: a +-- DayTimeIntervalType(DAY) by default, a CalendarIntervalType in legacy mode. +-- Config: spark.comet.exec.scalaUDF.codegen.enabled=true +-- Config: spark.comet.shuffle.mode=native +-- ConfigMatrix: spark.sql.legacy.interval.enabled=false,true + +statement +CREATE TABLE test_subtract_dates(d1 date, d2 date, k int) USING parquet + +-- the 2300 rows span about 330 years, past the 292-year limit of a nanosecond long, but the +-- day count of a date difference never touches that field +statement +INSERT INTO test_subtract_dates VALUES + (date'2024-03-15', date'2024-01-01', 1), + (date'2024-01-01', date'2024-03-15', 1), + (date'2024-02-29', date'2023-02-28', 2), + (date'1969-12-31', date'1970-01-02', 2), + (date'2024-06-01', date'2024-06-01', 3), + (date'1900-01-01', date'2100-12-31', 3), + (date'2300-01-01', date'1970-01-01', 6), + (date'1970-01-01', date'2300-01-01', 6), + (NULL, date'2024-01-01', 4), + (date'2024-01-01', NULL, 4), + (NULL, NULL, 5) + +-- column - column in both directions, covering negative and zero spans +query +SELECT d1, d2, d1 - d2, d2 - d1 FROM test_subtract_dates + +-- literal on either side +query +SELECT d1 - date'2024-01-01', date'2024-01-01' - d2 FROM test_subtract_dates + +-- all-literal operands (constant folding is disabled by the test suite). A NULL literal operand +-- is left out: NullPropagation folds it to a null interval literal, and the native literal +-- path rejects CalendarIntervalType (#5058). NULL operands are covered by the column rows above. +query +SELECT + date'2024-03-15' - date'2024-01-01', + date'2024-01-01' - date'2024-03-15', + date'2024-01-01' - date'2024-01-01' + +-- interval output through native shuffle, at top level and nested in a struct +query +SELECT k, d1 - d2 AS i, named_struct('i', d2 - d1) AS s +FROM test_subtract_dates +DISTRIBUTE BY k diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/subtract_timestamps.sql b/spark/src/test/resources/sql-tests/expressions/datetime/subtract_timestamps.sql new file mode 100644 index 00000000000..bebb7e709bb --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/datetime/subtract_timestamps.sql @@ -0,0 +1,98 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- timestamp - timestamp resolves to SubtractTimestamps and runs through the codegen dispatcher +-- so results match Spark exactly. The output type follows spark.sql.legacy.interval.enabled: +-- a DayTimeIntervalType measured on local wall-clock time by default, a CalendarIntervalType +-- holding the elapsed microseconds in legacy mode. Legacy mode falls back to Spark and is covered +-- by subtract_timestamps_legacy.sql. America/Los_Angeles is pinned so the DST rows below +-- straddle real transitions. +-- Config: spark.sql.session.timeZone=America/Los_Angeles +-- Config: spark.comet.exec.scalaUDF.codegen.enabled=true +-- Config: spark.comet.shuffle.mode=native +-- Config: spark.sql.legacy.interval.enabled=false + +statement +CREATE TABLE test_subtract_timestamps( + ts1 timestamp, + ts2 timestamp, + ntz1 timestamp_ntz, + ntz2 timestamp_ntz, + d date, + k int) USING parquet + +-- rows 3 and 4 span the spring-forward and fall-back transitions +statement +INSERT INTO test_subtract_timestamps VALUES + (timestamp'2024-03-15 10:30:45.123456', timestamp'2024-01-01 00:00:00', + timestamp_ntz'2024-03-15 10:30:45.123456', timestamp_ntz'2024-01-01 00:00:00', + date'2024-01-01', 1), + (timestamp'2024-01-01 00:00:00', timestamp'2024-03-15 10:30:45.123456', + timestamp_ntz'2024-01-01 00:00:00', timestamp_ntz'2024-03-15 10:30:45.123456', + date'2024-06-30', 1), + (timestamp'2024-03-10 12:00:00', timestamp'2024-03-09 12:00:00', + timestamp_ntz'2024-03-10 12:00:00', timestamp_ntz'2024-03-09 12:00:00', + date'2024-03-09', 2), + (timestamp'2024-11-03 12:00:00', timestamp'2024-11-02 12:00:00', + timestamp_ntz'2024-11-03 12:00:00', timestamp_ntz'2024-11-02 12:00:00', + date'2024-11-02', 2), + (timestamp'1969-12-31 23:59:59.999999', timestamp'1970-01-01 00:00:00', + timestamp_ntz'1969-12-31 23:59:59.999999', timestamp_ntz'1970-01-01 00:00:00', + date'1970-01-01', 3), + (timestamp'2024-06-01 08:00:00', timestamp'2024-06-01 08:00:00', + timestamp_ntz'2024-06-01 08:00:00', timestamp_ntz'2024-06-01 08:00:00', + date'2024-06-01', 3), + (NULL, timestamp'2024-01-01 00:00:00', NULL, timestamp_ntz'2024-01-01 00:00:00', + date'2024-01-01', 4), + (timestamp'2024-01-01 00:00:00', NULL, timestamp_ntz'2024-01-01 00:00:00', NULL, NULL, 4), + (NULL, NULL, NULL, NULL, NULL, 5) + +-- TIMESTAMP columns in both directions. Across a DST transition the default mode reports one +-- calendar day rather than the elapsed 23 or 25 hours. +query +SELECT ts1, ts2, ts1 - ts2, ts2 - ts1 FROM test_subtract_timestamps + +-- TIMESTAMP_NTZ columns compile a separate kernel and never see the session time zone +query +SELECT ntz1 - ntz2, ntz2 - ntz1 FROM test_subtract_timestamps + +-- a DATE operand is implicitly cast to TIMESTAMP inside the kernel +query +SELECT ts1 - d, d - ts1 FROM test_subtract_timestamps + +-- literal on either side +query +SELECT + ts1 - timestamp'2024-01-01 00:00:00', + timestamp'2024-01-01 00:00:00' - ts2, + ntz1 - timestamp_ntz'2024-01-01 00:00:00' +FROM test_subtract_timestamps + +-- all-literal operands (constant folding is disabled by the test suite). A NULL literal operand +-- is left out: NullPropagation folds it to a null interval literal, and the native literal +-- path rejects CalendarIntervalType (#5058). NULL operands are covered by the column rows above. +query +SELECT + timestamp'2024-03-15 10:30:45.123456' - timestamp'2024-01-01 00:00:00', + timestamp'2024-01-01 00:00:00' - timestamp'2024-03-15 10:30:45.123456', + timestamp_ntz'2024-03-10 12:00:00' - timestamp_ntz'2024-03-09 12:00:00' + +-- interval output through native shuffle, at top level and nested in a struct +query +SELECT k, ts1 - ts2 AS i, named_struct('i', ntz1 - ntz2) AS s +FROM test_subtract_timestamps +DISTRIBUTE BY k diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/subtract_timestamps_legacy.sql b/spark/src/test/resources/sql-tests/expressions/datetime/subtract_timestamps_legacy.sql new file mode 100644 index 00000000000..1ac7c20cb86 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/datetime/subtract_timestamps_legacy.sql @@ -0,0 +1,87 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- timestamp - timestamp in legacy interval mode resolves to SubtractTimestamps with a +-- CalendarIntervalType result holding the elapsed microseconds. The codegen dispatcher's +-- calendar-interval output cannot carry that span past about 292 years, so the expression falls +-- back to Spark. America/Los_Angeles is pinned so the DST rows below straddle real transitions. +-- Config: spark.sql.session.timeZone=America/Los_Angeles +-- Config: spark.comet.exec.scalaUDF.codegen.enabled=true +-- Config: spark.sql.legacy.interval.enabled=true + +statement +CREATE TABLE test_subtract_timestamps_legacy( + ts1 timestamp, + ts2 timestamp, + ntz1 timestamp_ntz, + ntz2 timestamp_ntz, + d date, + k int) USING parquet + +-- rows 3 and 4 span the spring-forward and fall-back transitions +statement +INSERT INTO test_subtract_timestamps_legacy VALUES + (timestamp'2024-03-15 10:30:45.123456', timestamp'2024-01-01 00:00:00', + timestamp_ntz'2024-03-15 10:30:45.123456', timestamp_ntz'2024-01-01 00:00:00', + date'2024-01-01', 1), + (timestamp'2024-01-01 00:00:00', timestamp'2024-03-15 10:30:45.123456', + timestamp_ntz'2024-01-01 00:00:00', timestamp_ntz'2024-03-15 10:30:45.123456', + date'2024-06-30', 1), + (timestamp'2024-03-10 12:00:00', timestamp'2024-03-09 12:00:00', + timestamp_ntz'2024-03-10 12:00:00', timestamp_ntz'2024-03-09 12:00:00', + date'2024-03-09', 2), + (timestamp'2024-11-03 12:00:00', timestamp'2024-11-02 12:00:00', + timestamp_ntz'2024-11-03 12:00:00', timestamp_ntz'2024-11-02 12:00:00', + date'2024-11-02', 2), + (timestamp'1969-12-31 23:59:59.999999', timestamp'1970-01-01 00:00:00', + timestamp_ntz'1969-12-31 23:59:59.999999', timestamp_ntz'1970-01-01 00:00:00', + date'1970-01-01', 3), + (timestamp'2024-06-01 08:00:00', timestamp'2024-06-01 08:00:00', + timestamp_ntz'2024-06-01 08:00:00', timestamp_ntz'2024-06-01 08:00:00', + date'2024-06-01', 3), + (NULL, timestamp'2024-01-01 00:00:00', NULL, timestamp_ntz'2024-01-01 00:00:00', + date'2024-01-01', 4), + (timestamp'2024-01-01 00:00:00', NULL, timestamp_ntz'2024-01-01 00:00:00', NULL, NULL, 4), + (NULL, NULL, NULL, NULL, NULL, 5) + +-- TIMESTAMP columns in both directions. Across a DST transition legacy mode reports the elapsed +-- 23 or 25 hours rather than one calendar day. +query expect_fallback(cannot carry a span past about 292 years) +SELECT ts1, ts2, ts1 - ts2, ts2 - ts1 FROM test_subtract_timestamps_legacy + +-- TIMESTAMP_NTZ columns never see the session time zone +query expect_fallback(cannot carry a span past about 292 years) +SELECT ntz1 - ntz2, ntz2 - ntz1 FROM test_subtract_timestamps_legacy + +-- a DATE operand is implicitly cast to TIMESTAMP +query expect_fallback(cannot carry a span past about 292 years) +SELECT ts1 - d, d - ts1 FROM test_subtract_timestamps_legacy + +-- literal on either side +query expect_fallback(cannot carry a span past about 292 years) +SELECT + ts1 - timestamp'2024-01-01 00:00:00', + timestamp'2024-01-01 00:00:00' - ts2, + ntz1 - timestamp_ntz'2024-01-01 00:00:00' +FROM test_subtract_timestamps_legacy + +-- all-literal operands (constant folding is disabled by the test suite) +query expect_fallback(cannot carry a span past about 292 years) +SELECT + timestamp'2024-03-15 10:30:45.123456' - timestamp'2024-01-01 00:00:00', + timestamp'2024-01-01 00:00:00' - timestamp'2024-03-15 10:30:45.123456', + timestamp_ntz'2024-03-10 12:00:00' - timestamp_ntz'2024-03-09 12:00:00' diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/subtract_timestamps_long_span.sql b/spark/src/test/resources/sql-tests/expressions/datetime/subtract_timestamps_long_span.sql new file mode 100644 index 00000000000..fa213c2bf59 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/datetime/subtract_timestamps_long_span.sql @@ -0,0 +1,46 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- timestamp - timestamp over a span past about 292 years. In default mode the result is a +-- DayTimeIntervalType that holds the microseconds in a long, so the span fits and the expression +-- stays on the codegen dispatcher. UTC is pinned so the elapsed microseconds are exact. +-- Config: spark.sql.session.timeZone=UTC +-- Config: spark.comet.exec.scalaUDF.codegen.enabled=true +-- Config: spark.sql.legacy.interval.enabled=false + +statement +CREATE TABLE test_subtract_timestamps_long_span( + ts1 timestamp, + ts2 timestamp, + ntz1 timestamp_ntz, + ntz2 timestamp_ntz) USING parquet + +-- 2300-01-01 - 1970-01-01 is 10413792000000000 microseconds, about 330 years +statement +INSERT INTO test_subtract_timestamps_long_span VALUES + (timestamp'2300-01-01 00:00:00', timestamp'1970-01-01 00:00:00', + timestamp_ntz'2300-01-01 00:00:00', timestamp_ntz'1970-01-01 00:00:00'), + (timestamp'1970-01-01 00:00:00', timestamp'2300-01-01 00:00:00', + timestamp_ntz'1970-01-01 00:00:00', timestamp_ntz'2300-01-01 00:00:00') + +-- TIMESTAMP columns in both operand orders +query +SELECT ts1, ts2, ts1 - ts2, ts2 - ts1 FROM test_subtract_timestamps_long_span + +-- TIMESTAMP_NTZ columns in both operand orders +query +SELECT ntz1 - ntz2, ntz2 - ntz1 FROM test_subtract_timestamps_long_span diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/subtract_timestamps_long_span_legacy.sql b/spark/src/test/resources/sql-tests/expressions/datetime/subtract_timestamps_long_span_legacy.sql new file mode 100644 index 00000000000..58c5d8d63fa --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/datetime/subtract_timestamps_long_span_legacy.sql @@ -0,0 +1,47 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- timestamp - timestamp over a span past about 292 years in legacy interval mode. The result is +-- a CalendarIntervalType whose microseconds the codegen dispatcher writes as nanoseconds, which +-- overflows a long for this span, so the expression must fall back to Spark. UTC is pinned so the +-- elapsed microseconds are exact. +-- Config: spark.sql.session.timeZone=UTC +-- Config: spark.comet.exec.scalaUDF.codegen.enabled=true +-- Config: spark.sql.legacy.interval.enabled=true + +statement +CREATE TABLE test_subtract_timestamps_long_span_legacy( + ts1 timestamp, + ts2 timestamp, + ntz1 timestamp_ntz, + ntz2 timestamp_ntz) USING parquet + +-- 2300-01-01 - 1970-01-01 is 10413792000000000 microseconds, past the dispatcher's limit +statement +INSERT INTO test_subtract_timestamps_long_span_legacy VALUES + (timestamp'2300-01-01 00:00:00', timestamp'1970-01-01 00:00:00', + timestamp_ntz'2300-01-01 00:00:00', timestamp_ntz'1970-01-01 00:00:00'), + (timestamp'1970-01-01 00:00:00', timestamp'2300-01-01 00:00:00', + timestamp_ntz'1970-01-01 00:00:00', timestamp_ntz'2300-01-01 00:00:00') + +-- TIMESTAMP columns in both operand orders +query expect_fallback(cannot carry a span past about 292 years) +SELECT ts1, ts2, ts1 - ts2, ts2 - ts1 FROM test_subtract_timestamps_long_span_legacy + +-- TIMESTAMP_NTZ columns in both operand orders +query expect_fallback(cannot carry a span past about 292 years) +SELECT ntz1 - ntz2, ntz2 - ntz1 FROM test_subtract_timestamps_long_span_legacy diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/timestamp_add_interval.sql b/spark/src/test/resources/sql-tests/expressions/datetime/timestamp_add_interval.sql new file mode 100644 index 00000000000..6aa0354f68e --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/datetime/timestamp_add_interval.sql @@ -0,0 +1,113 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- timestamp + day-time or calendar interval resolves to TimeAdd (TimestampAddInterval on +-- Spark 4.1+) and runs through the codegen dispatcher so results match Spark exactly. Days and +-- months are applied on local time in the session zone, so America/Los_Angeles is pinned and +-- the DST rows straddle the spring-forward and fall-back transitions. +-- Config: spark.sql.session.timeZone=America/Los_Angeles +-- Config: spark.comet.exec.scalaUDF.codegen.enabled=true +-- Config: spark.comet.shuffle.mode=native + +statement +CREATE TABLE test_timestamp_add_interval( + ts timestamp, + ts_ntz timestamp_ntz, + d date, + dd int, + h int, + mi int, + s decimal(18, 6), + k int) USING parquet + +statement +INSERT INTO test_timestamp_add_interval VALUES + (timestamp'2024-01-15 10:30:45.123456', timestamp_ntz'2024-01-15 10:30:45.123456', + date'2024-01-15', 1, 2, 3, 4.5, 1), + (timestamp'2024-01-31 23:00:00', timestamp_ntz'2024-01-31 23:00:00', + date'2024-01-31', 1, 1, 0, 0, 1), + (timestamp'2024-03-09 12:00:00', timestamp_ntz'2024-03-09 12:00:00', + date'2024-03-09', 1, 0, 0, 0, 2), + (timestamp'2024-03-09 12:00:00', timestamp_ntz'2024-03-09 12:00:00', + date'2024-03-09', 0, 24, 0, 0, 2), + (timestamp'2024-11-02 12:00:00', timestamp_ntz'2024-11-02 12:00:00', + date'2024-11-02', 1, 0, 0, 0, 3), + (timestamp'2024-11-02 12:00:00', timestamp_ntz'2024-11-02 12:00:00', + date'2024-11-02', 0, 24, 0, 0, 3), + (timestamp'2024-12-31 23:59:59.999999', timestamp_ntz'2024-12-31 23:59:59.999999', + date'2024-12-31', 0, 0, 0, 0.000001, 4), + (timestamp'1970-01-01 00:00:00', timestamp_ntz'1970-01-01 00:00:00', + date'1970-01-01', -1, -1, -1, -1.5, 4), + (timestamp'2024-06-15 08:00:00', timestamp_ntz'2024-06-15 08:00:00', + date'2024-06-15', NULL, 1, 1, 1, 5), + (timestamp'2024-06-15 08:00:00', timestamp_ntz'2024-06-15 08:00:00', + date'2024-06-15', 1, 1, 1, NULL, 5), + (NULL, NULL, NULL, 1, 1, 1, 1, 6) + +-- TIMESTAMP column plus a day-time interval built from columns, both directions +query +SELECT ts, dd, h, mi, s, ts + make_dt_interval(dd, h, mi, s), make_dt_interval(dd, h, mi, s) + ts +FROM test_timestamp_add_interval + +-- TIMESTAMP column plus a calendar interval built from columns, both directions +query +SELECT ts + make_interval(0, 1, 0, dd, h, mi, s), make_interval(0, 1, 0, dd, h, mi, s) + ts +FROM test_timestamp_add_interval + +-- TIMESTAMP_NTZ compiles a separate kernel and never sees the session time zone +query +SELECT ts_ntz + make_dt_interval(dd, h, mi, s), ts_ntz + make_interval(0, 1, 0, dd, h, mi, s) +FROM test_timestamp_add_interval + +-- a DATE operand is cast to TIMESTAMP before a day-time interval finer than a day is added +query +SELECT d + make_dt_interval(dd, h, mi, s), d + INTERVAL '1 12:00:00' DAY TO SECOND +FROM test_timestamp_add_interval + +-- day-time interval literals in the unit, unit-to-unit and multi-unit spellings. The parser +-- rejects literals that mix year-month and day-time units unless spark.sql.legacy.interval.enabled +-- is set, so literal calendar intervals come from make_interval. Subtraction rewrites to an +-- addition of the negated interval. +query +SELECT + ts + INTERVAL '1' DAY, + ts + INTERVAL '36' HOUR, + ts + INTERVAL '1 02:30:00.5' DAY TO SECOND, + ts + INTERVAL '1 day 2 hours', + ts + make_interval(0, 1, 0, 1, 2), + ts - INTERVAL '1' DAY, + ts - make_interval(0, 1, 0, 1, 2), + ts_ntz + INTERVAL '1' DAY, + ts_ntz - make_interval(0, 1, 0, 1) +FROM test_timestamp_add_interval + +-- all-literal operands (constant folding is disabled by the test suite) +query +SELECT + timestamp'2024-03-09 12:00:00' + INTERVAL '1' DAY, + timestamp'2024-03-09 12:00:00' + INTERVAL '24' HOUR, + timestamp'2024-01-31 23:00:00' + make_interval(0, 1, 0, 1), + timestamp_ntz'2024-03-09 12:00:00' + INTERVAL '1' DAY, + CAST(NULL AS TIMESTAMP) + INTERVAL '1' DAY, + timestamp'2024-01-31 00:00:00' + CAST(NULL AS INTERVAL DAY TO SECOND), + timestamp'2024-01-31 00:00:00' + CAST(NULL AS INTERVAL) + +-- timestamp output through native shuffle +query +SELECT k, ts + make_dt_interval(dd, h, mi, s) AS r, ts_ntz + make_interval(0, 1, 0, dd) AS r_ntz +FROM test_timestamp_add_interval +DISTRIBUTE BY k diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/timestamp_add_ym_interval.sql b/spark/src/test/resources/sql-tests/expressions/datetime/timestamp_add_ym_interval.sql new file mode 100644 index 00000000000..daa4d6ff13f --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/datetime/timestamp_add_ym_interval.sql @@ -0,0 +1,84 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- timestamp + year-month interval resolves to TimestampAddYMInterval and runs through the +-- codegen dispatcher so results match Spark exactly. TIMESTAMP values are shifted on local +-- time in the session zone, so America/Los_Angeles is pinned and two rows cross a DST change. +-- Config: spark.sql.session.timeZone=America/Los_Angeles +-- Config: spark.comet.exec.scalaUDF.codegen.enabled=true +-- Config: spark.comet.shuffle.mode=native + +statement +CREATE TABLE test_timestamp_add_ym_interval( + ts timestamp, + ts_ntz timestamp_ntz, + y int, + m int, + k int) USING parquet + +-- rows 5 and 6 cross the spring-forward and fall-back transitions +statement +INSERT INTO test_timestamp_add_ym_interval VALUES + (timestamp'2024-01-31 10:30:45.123456', timestamp_ntz'2024-01-31 10:30:45.123456', 0, 1, 1), + (timestamp'2024-01-31 10:30:45.123456', timestamp_ntz'2024-01-31 10:30:45.123456', 0, 13, 1), + (timestamp'2024-02-29 23:59:59', timestamp_ntz'2024-02-29 23:59:59', 1, 0, 2), + (timestamp'2024-03-31 00:00:00', timestamp_ntz'2024-03-31 00:00:00', 0, -1, 2), + (timestamp'2024-02-10 12:00:00', timestamp_ntz'2024-02-10 12:00:00', 0, 1, 3), + (timestamp'2024-10-15 12:00:00', timestamp_ntz'2024-10-15 12:00:00', 0, 1, 3), + (timestamp'1970-01-01 00:00:00', timestamp_ntz'1970-01-01 00:00:00', -1, -1, 4), + (timestamp'2024-06-15 08:00:00', timestamp_ntz'2024-06-15 08:00:00', NULL, 1, 5), + (timestamp'2024-06-15 08:00:00', timestamp_ntz'2024-06-15 08:00:00', 1, NULL, 5), + (NULL, NULL, 1, 1, 6) + +-- TIMESTAMP column plus an interval built from columns, both directions +query +SELECT ts, y, m, ts + make_ym_interval(y, m), make_ym_interval(y, m) + ts +FROM test_timestamp_add_ym_interval + +-- TIMESTAMP_NTZ compiles a separate kernel and never sees the session time zone +query +SELECT ts_ntz, ts_ntz + make_ym_interval(y, m), make_ym_interval(y, m) + ts_ntz +FROM test_timestamp_add_ym_interval + +-- interval literals in the unit, unit-to-unit and multi-unit spellings. Subtraction rewrites +-- to an addition of the negated interval. +query +SELECT + ts + INTERVAL '1' YEAR, + ts + INTERVAL '1' MONTH, + ts + INTERVAL '1-2' YEAR TO MONTH, + ts + INTERVAL '1 year 1 month', + ts - INTERVAL '1' MONTH, + ts_ntz + INTERVAL '1' YEAR, + ts_ntz - INTERVAL '1' MONTH +FROM test_timestamp_add_ym_interval + +-- all-literal operands (constant folding is disabled by the test suite) +query +SELECT + timestamp'2024-01-31 10:30:45.123456' + INTERVAL '1' MONTH, + timestamp'2024-02-29 23:59:59' + INTERVAL '1' YEAR, + timestamp_ntz'2024-01-31 10:30:45.123456' + INTERVAL '1' MONTH, + timestamp'2024-03-31 00:00:00' - INTERVAL '1' MONTH, + CAST(NULL AS TIMESTAMP) + INTERVAL '1' MONTH, + timestamp'2024-01-31 00:00:00' + CAST(NULL AS INTERVAL YEAR TO MONTH) + +-- timestamp output through native shuffle +query +SELECT k, ts + make_ym_interval(y, m) AS r, ts_ntz + make_ym_interval(y, m) AS r_ntz +FROM test_timestamp_add_ym_interval +DISTRIBUTE BY k