diff --git a/docs/source/user-guide/latest/expressions.md b/docs/source/user-guide/latest/expressions.md index 208a5f3f124..e96fc2eb727 100644 --- a/docs/source/user-guide/latest/expressions.md +++ b/docs/source/user-guide/latest/expressions.md @@ -415,8 +415,8 @@ The type-name conversion functions (`bigint`, `binary`, `boolean`, `date`, `deci | --- | --- | --- | --- | | `%` | ✅ | Native | | | `*` | ✅ | Native | DayTime interval multiplication routes through the JVM codegen dispatcher; YearMonth and Calendar interval multiplication fall back | -| `+` | ✅ | Native | | -| `-` | ✅ | Native | | +| `+` | ✅ | Native | Adding a calendar, year-month or day-time interval to a date or timestamp routes through the JVM codegen dispatcher | +| `-` | ✅ | Native | `date - date`, `timestamp - timestamp` and subtracting an interval from a date or timestamp route through the JVM codegen dispatcher; `timestamp - timestamp` falls back to Spark in legacy interval mode (`spark.sql.legacy.interval.enabled=true`) because its calendar-interval result can exceed what the dispatcher output can carry | | `/` | ✅ | Native | | | `abs` | ✅ | Hybrid | Interval types route through the JVM codegen dispatcher; numeric types run natively | | `acos` | ✅ | Native | | diff --git a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala index be4bc9c3412..25350245c29 100644 --- a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala +++ b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala @@ -307,6 +307,11 @@ object QueryPlanSerde extends Logging with CometExprShim with CometTypeShim { classOf[MultiplyDTInterval] -> CometMultiplyDTInterval, classOf[TimestampAdd] -> CometTimestampAdd, classOf[TimestampDiff] -> CometTimestampDiff, + classOf[DateAddInterval] -> CometDateAddInterval, + classOf[DateAddYMInterval] -> CometDateAddYMInterval, + classOf[TimestampAddYMInterval] -> CometTimestampAddYMInterval, + classOf[SubtractDates] -> CometSubtractDates, + classOf[SubtractTimestamps] -> CometSubtractTimestamps, classOf[MicrosToTimestamp] -> CometMicrosToTimestamp, classOf[MillisToTimestamp] -> CometMillisToTimestamp, classOf[MonthsBetween] -> CometMonthsBetween, diff --git a/spark/src/main/scala/org/apache/comet/serde/datetime.scala b/spark/src/main/scala/org/apache/comet/serde/datetime.scala index 8f2b9ab6719..629d999868e 100644 --- a/spark/src/main/scala/org/apache/comet/serde/datetime.scala +++ b/spark/src/main/scala/org/apache/comet/serde/datetime.scala @@ -21,7 +21,7 @@ package org.apache.comet.serde import java.util.Locale -import org.apache.spark.sql.catalyst.expressions.{AddMonths, Attribute, Cast, ConvertTimezone, DateAdd, DateDiff, DateFormatClass, DateFromUnixDate, DateSub, DayOfMonth, DayOfWeek, DayOfYear, Days, Expression, FromUTCTimestamp, GetDateField, GetTimestamp, Hour, Hours, LastDay, Literal, MakeDate, MakeDTInterval, MakeInterval, MakeTimestamp, MakeYMInterval, MicrosToTimestamp, MillisToTimestamp, Minute, Month, MonthsBetween, MultiplyDTInterval, NextDay, PreciseTimestampConversion, Quarter, Second, SecondsToTimestamp, TimestampAdd, TimestampDiff, ToUnixTimestamp, ToUTCTimestamp, TruncDate, TruncTimestamp, UnixDate, UnixMicros, UnixMillis, UnixSeconds, UnixTimestamp, WeekDay, WeekOfYear, Year} +import org.apache.spark.sql.catalyst.expressions.{AddMonths, Attribute, Cast, ConvertTimezone, DateAdd, DateAddInterval, DateAddYMInterval, DateDiff, DateFormatClass, DateFromUnixDate, DateSub, DayOfMonth, DayOfWeek, DayOfYear, Days, Expression, FromUTCTimestamp, GetDateField, GetTimestamp, Hour, Hours, LastDay, Literal, MakeDate, MakeDTInterval, MakeInterval, MakeTimestamp, MakeYMInterval, MicrosToTimestamp, MillisToTimestamp, Minute, Month, MonthsBetween, MultiplyDTInterval, NextDay, PreciseTimestampConversion, Quarter, Second, SecondsToTimestamp, SubtractDates, SubtractTimestamps, TimestampAdd, TimestampAddYMInterval, TimestampDiff, ToUnixTimestamp, ToUTCTimestamp, TruncDate, TruncTimestamp, UnixDate, UnixMicros, UnixMillis, UnixSeconds, UnixTimestamp, WeekDay, WeekOfYear, Year} import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{CalendarIntervalType, DataType, DateType, DoubleType, FloatType, IntegerType, LongType, StringType, TimestampNTZType, TimestampType} import org.apache.spark.unsafe.types.UTF8String @@ -997,6 +997,37 @@ object CometTimestampAdd extends CometCodegenDispatch[TimestampAdd] object CometTimestampDiff extends CometCodegenDispatch[TimestampDiff] +// Date and timestamp interval arithmetic. `timestamp + day-time or calendar interval` resolves +// to `TimeAdd` on Spark 3.4 through 4.0 and to `TimestampAddInterval` on 4.1+, so that serde +// lives in the version shims. +object CometDateAddInterval extends CometCodegenDispatch[DateAddInterval] + +object CometDateAddYMInterval extends CometCodegenDispatch[DateAddYMInterval] + +object CometTimestampAddYMInterval extends CometCodegenDispatch[TimestampAddYMInterval] + +object CometSubtractDates extends CometCodegenDispatch[SubtractDates] + +object CometSubtractTimestamps extends CometCodegenDispatch[SubtractTimestamps] { + private val legacyIntervalReason = + "In legacy interval mode (`spark.sql.legacy.interval.enabled=true`) the result is a" + + " `CalendarIntervalType`, and the JVM codegen dispatcher's calendar-interval output" + + " cannot carry a span past about 292 years (see" + + " https://github.com/apache/datafusion-comet/issues/5279), so the expression falls" + + " back to Spark" + + override def getUnsupportedReasons(): Seq[String] = Seq(legacyIntervalReason) + + // Same `Math.multiplyExact(microseconds, 1000L)` limit `CometMakeInterval` documents as a + // compatible note. That one only overflows on extreme arguments; `ts - ts` produces an + // arbitrary span from ordinary data, and legacy mode is off by default, so decline it. + // Remove this branch once #5279 carries CalendarInterval across the boundary losslessly. + override def getSupportLevel(expr: SubtractTimestamps): SupportLevel = expr.dataType match { + case CalendarIntervalType => Unsupported(Some(legacyIntervalReason)) + case _ => Compatible() + } +} + /** * Spark's internal `PreciseTimestampConversion` reinterprets a value between the timestamp types * (`TimestampType` / `TimestampNTZType`) and `LongType` without losing microsecond precision. It diff --git a/spark/src/main/spark-3.4/org/apache/comet/shims/CometExprShim.scala b/spark/src/main/spark-3.4/org/apache/comet/shims/CometExprShim.scala index 594c85e02fd..c101e3d3008 100644 --- a/spark/src/main/spark-3.4/org/apache/comet/shims/CometExprShim.scala +++ b/spark/src/main/spark-3.4/org/apache/comet/shims/CometExprShim.scala @@ -23,7 +23,7 @@ import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.expressions.aggregate.Sum import org.apache.comet.expressions.CometEvalMode -import org.apache.comet.serde.{CometEncode, CometExpressionSerde, CometStringDecode} +import org.apache.comet.serde.{CometCodegenDispatch, CometEncode, CometExpressionSerde, CometStringDecode} import org.apache.comet.serde.ExprOuterClass.{BinaryOutputStyle, Expr} /** @@ -41,7 +41,7 @@ trait CometExprShim { def sparkVersionSpecificMathExpressions: Map[Class[_ <: Expression], CometExpressionSerde[_]] = Map.empty def sparkVersionSpecificMiscExpressions: Map[Class[_ <: Expression], CometExpressionSerde[_]] = - Map.empty + Map(classOf[TimeAdd] -> new CometCodegenDispatch[TimeAdd]) def sparkVersionSpecificMapExpressions: Map[Class[_ <: Expression], CometExpressionSerde[_]] = Map.empty diff --git a/spark/src/main/spark-3.5/org/apache/comet/shims/CometExprShim.scala b/spark/src/main/spark-3.5/org/apache/comet/shims/CometExprShim.scala index 0bfea5cd6ec..4a6cb67bfc6 100644 --- a/spark/src/main/spark-3.5/org/apache/comet/shims/CometExprShim.scala +++ b/spark/src/main/spark-3.5/org/apache/comet/shims/CometExprShim.scala @@ -23,7 +23,7 @@ import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.expressions.aggregate.Sum import org.apache.comet.expressions.CometEvalMode -import org.apache.comet.serde.{CometEncode, CometExpressionSerde, CometStringDecode, CometToPrettyString, CometWidthBucket} +import org.apache.comet.serde.{CometCodegenDispatch, CometEncode, CometExpressionSerde, CometStringDecode, CometToPrettyString, CometWidthBucket} import org.apache.comet.serde.ExprOuterClass.{BinaryOutputStyle, Expr} /** @@ -41,7 +41,9 @@ trait CometExprShim { def sparkVersionSpecificMathExpressions: Map[Class[_ <: Expression], CometExpressionSerde[_]] = Map(classOf[WidthBucket] -> CometWidthBucket) def sparkVersionSpecificMiscExpressions: Map[Class[_ <: Expression], CometExpressionSerde[_]] = - Map(classOf[ToPrettyString] -> CometToPrettyString) + Map( + classOf[ToPrettyString] -> CometToPrettyString, + classOf[TimeAdd] -> new CometCodegenDispatch[TimeAdd]) def sparkVersionSpecificMapExpressions: Map[Class[_ <: Expression], CometExpressionSerde[_]] = Map.empty diff --git a/spark/src/main/spark-4.0/org/apache/comet/shims/CometExprShim.scala b/spark/src/main/spark-4.0/org/apache/comet/shims/CometExprShim.scala index ca3c79930f3..4903f0b8c0a 100644 --- a/spark/src/main/spark-4.0/org/apache/comet/shims/CometExprShim.scala +++ b/spark/src/main/spark-4.0/org/apache/comet/shims/CometExprShim.scala @@ -19,11 +19,12 @@ package org.apache.comet.shims -import org.apache.spark.sql.catalyst.expressions.EvalMode +import org.apache.spark.sql.catalyst.expressions.{EvalMode, Expression, TimeAdd} import org.apache.spark.sql.catalyst.expressions.aggregate.Sum import org.apache.spark.sql.internal.SQLConf import org.apache.comet.expressions.CometEvalMode +import org.apache.comet.serde.{CometCodegenDispatch, CometExpressionSerde} import org.apache.comet.serde.ExprOuterClass.BinaryOutputStyle /** @@ -42,6 +43,12 @@ trait CometExprShim extends Spark4xCometExprShim { case _ => BinaryOutputStyle.HEX_DISCRETE } } + + // Spark 4.0 still spells `timestamp + interval` as `TimeAdd`; 4.1 renames it. + override def sparkVersionSpecificMiscExpressions + : Map[Class[_ <: Expression], CometExpressionSerde[_]] = + super.sparkVersionSpecificMiscExpressions + (classOf[TimeAdd] -> new CometCodegenDispatch[ + TimeAdd]) } object CometEvalModeUtil { diff --git a/spark/src/main/spark-4.1+/org/apache/comet/serde/CometTimestampAddInterval.scala b/spark/src/main/spark-4.1+/org/apache/comet/serde/CometTimestampAddInterval.scala new file mode 100644 index 00000000000..1f93b5ac3ec --- /dev/null +++ b/spark/src/main/spark-4.1+/org/apache/comet/serde/CometTimestampAddInterval.scala @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet.serde + +import org.apache.spark.sql.catalyst.expressions.TimestampAddInterval + +/** + * `timestamp + day-time or calendar interval` resolves to `TimestampAddInterval` on Spark 4.1+ + * (`TimeAdd` on earlier versions) and runs through the codegen dispatcher. + */ +object CometTimestampAddInterval extends CometCodegenDispatch[TimestampAddInterval] diff --git a/spark/src/main/spark-4.1+/org/apache/comet/shims/CometExprShim.scala b/spark/src/main/spark-4.1+/org/apache/comet/shims/CometExprShim.scala index a2380e4008a..bcebcbae03a 100644 --- a/spark/src/main/spark-4.1+/org/apache/comet/shims/CometExprShim.scala +++ b/spark/src/main/spark-4.1+/org/apache/comet/shims/CometExprShim.scala @@ -27,6 +27,7 @@ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.TimeType import org.apache.comet.expressions.CometEvalMode +import org.apache.comet.serde.{CometExpressionSerde, CometTimestampAddInterval} import org.apache.comet.serde.ExprOuterClass.{BinaryOutputStyle, Expr} import org.apache.comet.serde.QueryPlanSerde.{exprToProtoInternal, scalarFunctionExprToProtoWithReturnType} @@ -46,6 +47,12 @@ trait CometExprShim extends Spark4xCometExprShim { } } + // Spark 4.1 renames `TimeAdd` to `TimestampAddInterval`. + override def sparkVersionSpecificMiscExpressions + : Map[Class[_ <: Expression], CometExpressionSerde[_]] = + super.sparkVersionSpecificMiscExpressions + + (classOf[TimestampAddInterval] -> CometTimestampAddInterval) + // Spark 4.1 introduced TimeType and the make_time / to_time / try_to_time functions. // Their planner forms differ from the shared 4.x patterns (DateTimeUtils.makeTime // StaticInvoke and ToTimeParser Invoke / TryEval(Invoke)), so they live here rather diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/date_add_interval.sql b/spark/src/test/resources/sql-tests/expressions/datetime/date_add_interval.sql new file mode 100644 index 00000000000..32e46fcac43 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/datetime/date_add_interval.sql @@ -0,0 +1,84 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- date + calendar interval resolves to DateAddInterval and runs through the codegen dispatcher +-- so results match Spark exactly. With ANSI off, an interval carrying a time part is applied +-- on the timestamp and the result truncated back to a date; the ANSI error case lives in +-- date_add_interval_ansi.sql. America/Los_Angeles is pinned so the 25-hour row crosses DST. +-- Config: spark.sql.session.timeZone=America/Los_Angeles +-- Config: spark.comet.exec.scalaUDF.codegen.enabled=true +-- Config: spark.comet.shuffle.mode=native + +statement +CREATE TABLE test_date_add_interval(d date, y int, m int, dd int, h int, k int) USING parquet + +statement +INSERT INTO test_date_add_interval VALUES + (date'2024-01-31', 0, 1, 0, 0, 1), + (date'2024-01-31', 1, 1, 1, 0, 1), + (date'2024-02-29', 1, 0, 0, 0, 2), + (date'2024-03-31', 0, -1, 0, 0, 2), + (date'2024-12-31', 0, 0, 1, 0, 3), + (date'2024-03-09', 0, 0, 1, 0, 3), + (date'2024-03-09', 0, 0, 0, 25, 4), + (date'1970-01-01', -1, -1, -1, -1, 4), + (date'2024-06-15', NULL, 1, 1, 0, 5), + (date'2024-06-15', 0, NULL, 1, 0, 5), + (date'2024-06-15', 0, 0, NULL, 0, 6), + (date'2024-06-15', 0, 0, 0, NULL, 6), + (NULL, 1, 1, 1, 0, 7) + +-- column date plus a calendar interval built from columns. Month arithmetic clamps to the end +-- of the shorter month before the day part is added. +query +SELECT d, y, m, dd, d + make_interval(y, m, 0, dd) FROM test_date_add_interval + +-- interval on the left +query +SELECT make_interval(y, m, 0, dd) + d FROM test_date_add_interval + +-- with ANSI off the hour part is applied on the timestamp and truncated away again +query +SELECT d, h, d + make_interval(y, m, 0, dd, h) FROM test_date_add_interval + +-- The parser rejects interval literals that mix year-month and day-time units unless +-- spark.sql.legacy.interval.enabled is set, so literal calendar intervals come from +-- make_interval. Subtraction rewrites to an addition of the negated interval. +query +SELECT + d + make_interval(1, 0, 0, 1), + d + make_interval(-1, 0, 0, -1), + d + make_interval(0, 1, 0, 1), + d + make_interval(0, 1, 0, 1, 12), + d - make_interval(1, 0, 0, 1), + d - make_interval(0, 1, 0, 1) +FROM test_date_add_interval + +-- all-literal operands (constant folding is disabled by the test suite) +query +SELECT + date'2024-01-31' + make_interval(0, 1, 0, 1), + date'2024-02-29' + make_interval(1, 0, 0, 1), + date'2024-01-31' - make_interval(0, 1, 0, 1), + CAST(NULL AS DATE) + make_interval(0, 1, 0, 1), + date'2024-01-31' + CAST(NULL AS INTERVAL) + +-- date output through native shuffle +query +SELECT k, d + make_interval(y, m, 0, dd) AS r +FROM test_date_add_interval +DISTRIBUTE BY k diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/date_add_interval_ansi.sql b/spark/src/test/resources/sql-tests/expressions/datetime/date_add_interval_ansi.sql new file mode 100644 index 00000000000..bb4913d9eb5 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/datetime/date_add_interval_ansi.sql @@ -0,0 +1,47 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- With ANSI on, DateAddInterval keeps day-granular intervals on the date path and rejects an +-- interval that carries hours, minutes, seconds or fractions of a second. The parser rejects +-- interval literals that mix year-month and day-time units, so make_interval builds them. +-- Config: spark.sql.ansi.enabled=true +-- Config: spark.comet.exec.scalaUDF.codegen.enabled=true +-- MinSparkVersion: 4.0 + +statement +CREATE TABLE test_date_add_interval_ansi(d date, m int, dd int) USING parquet + +statement +INSERT INTO test_date_add_interval_ansi VALUES + (date'2024-01-31', 1, 1), + (date'2024-02-29', 12, 0), + (date'2024-03-31', -1, -1), + (date'2024-06-15', NULL, 1), + (NULL, 1, 1) + +-- sentinel: a day-granular interval succeeds and asserts native execution +query +SELECT d, m, dd, d + make_interval(0, m, 0, dd), d - make_interval(0, 1, 0, 1) +FROM test_date_add_interval_ansi + +-- a NULL interval yields NULL rather than an error +query +SELECT d + CAST(NULL AS INTERVAL) FROM test_date_add_interval_ansi + +-- an interval with a time part is rejected +query expect_error(INVALID_INTERVAL_WITH_MICROSECONDS_ADDITION) +SELECT d + make_interval(0, 1, 0, 1, 12) FROM test_date_add_interval_ansi diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/date_add_interval_ansi_spark35.sql b/spark/src/test/resources/sql-tests/expressions/datetime/date_add_interval_ansi_spark35.sql new file mode 100644 index 00000000000..100dd707725 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/datetime/date_add_interval_ansi_spark35.sql @@ -0,0 +1,47 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- With ANSI on, DateAddInterval keeps day-granular intervals on the date path and rejects an +-- interval that carries hours, minutes, seconds or fractions of a second. The parser rejects +-- interval literals that mix year-month and day-time units, so make_interval builds them. +-- Config: spark.sql.ansi.enabled=true +-- Config: spark.comet.exec.scalaUDF.codegen.enabled=true +-- MaxSparkVersion: 3.5 + +statement +CREATE TABLE test_date_add_interval_ansi(d date, m int, dd int) USING parquet + +statement +INSERT INTO test_date_add_interval_ansi VALUES + (date'2024-01-31', 1, 1), + (date'2024-02-29', 12, 0), + (date'2024-03-31', -1, -1), + (date'2024-06-15', NULL, 1), + (NULL, 1, 1) + +-- sentinel: a day-granular interval succeeds and asserts native execution +query +SELECT d, m, dd, d + make_interval(0, m, 0, dd), d - make_interval(0, 1, 0, 1) +FROM test_date_add_interval_ansi + +-- a NULL interval yields NULL rather than an error +query +SELECT d + CAST(NULL AS INTERVAL) FROM test_date_add_interval_ansi + +-- an interval with a time part is rejected +query expect_error(Cannot add hours, minutes or seconds, milliseconds, microseconds to a date) +SELECT d + make_interval(0, 1, 0, 1, 12) FROM test_date_add_interval_ansi diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/date_add_ym_interval.sql b/spark/src/test/resources/sql-tests/expressions/datetime/date_add_ym_interval.sql new file mode 100644 index 00000000000..094aeeeff61 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/datetime/date_add_ym_interval.sql @@ -0,0 +1,71 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- date + year-month interval resolves to DateAddYMInterval and runs through the codegen +-- dispatcher so results match Spark exactly. Month arithmetic clamps to the last day of the +-- target month. +-- Config: spark.comet.exec.scalaUDF.codegen.enabled=true +-- Config: spark.comet.shuffle.mode=native + +statement +CREATE TABLE test_date_add_ym_interval(d date, y int, m int, k int) USING parquet + +statement +INSERT INTO test_date_add_ym_interval VALUES + (date'2024-01-31', 0, 1, 1), + (date'2024-01-31', 0, 13, 1), + (date'2024-02-29', 1, 0, 2), + (date'2024-02-29', 4, 0, 2), + (date'2024-03-31', 0, -1, 3), + (date'2024-12-31', 0, 2, 3), + (date'1970-01-01', -1, -1, 4), + (date'2024-06-15', NULL, 1, 5), + (date'2024-06-15', 1, NULL, 5), + (NULL, 1, 1, 6) + +-- column date plus an interval built from columns, both directions +query +SELECT d, y, m, d + make_ym_interval(y, m), make_ym_interval(y, m) + d +FROM test_date_add_ym_interval + +-- interval literals in the unit, unit-to-unit and multi-unit spellings. Subtraction rewrites +-- to an addition of the negated interval. +query +SELECT + d + INTERVAL '1' YEAR, + d + INTERVAL '1' MONTH, + d + INTERVAL '1-2' YEAR TO MONTH, + d + INTERVAL '1 year 1 month', + d + INTERVAL '-1' MONTH, + d - INTERVAL '1' MONTH, + d - INTERVAL '1' YEAR +FROM test_date_add_ym_interval + +-- all-literal operands (constant folding is disabled by the test suite) +query +SELECT + date'2024-01-31' + INTERVAL '1' MONTH, + date'2024-02-29' + INTERVAL '1' YEAR, + date'2024-03-31' - INTERVAL '1' MONTH, + CAST(NULL AS DATE) + INTERVAL '1' MONTH, + date'2024-01-31' + CAST(NULL AS INTERVAL YEAR TO MONTH) + +-- date output through native shuffle +query +SELECT k, d + make_ym_interval(y, m) AS r +FROM test_date_add_ym_interval +DISTRIBUTE BY k diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/subtract_dates.sql b/spark/src/test/resources/sql-tests/expressions/datetime/subtract_dates.sql new file mode 100644 index 00000000000..26ea41a011b --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/datetime/subtract_dates.sql @@ -0,0 +1,65 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- date - date resolves to SubtractDates and runs through the codegen dispatcher so results +-- match Spark exactly. The output type follows spark.sql.legacy.interval.enabled: a +-- DayTimeIntervalType(DAY) by default, a CalendarIntervalType in legacy mode. +-- Config: spark.comet.exec.scalaUDF.codegen.enabled=true +-- Config: spark.comet.shuffle.mode=native +-- ConfigMatrix: spark.sql.legacy.interval.enabled=false,true + +statement +CREATE TABLE test_subtract_dates(d1 date, d2 date, k int) USING parquet + +-- the 2300 rows span about 330 years, past the 292-year limit of a nanosecond long, but the +-- day count of a date difference never touches that field +statement +INSERT INTO test_subtract_dates VALUES + (date'2024-03-15', date'2024-01-01', 1), + (date'2024-01-01', date'2024-03-15', 1), + (date'2024-02-29', date'2023-02-28', 2), + (date'1969-12-31', date'1970-01-02', 2), + (date'2024-06-01', date'2024-06-01', 3), + (date'1900-01-01', date'2100-12-31', 3), + (date'2300-01-01', date'1970-01-01', 6), + (date'1970-01-01', date'2300-01-01', 6), + (NULL, date'2024-01-01', 4), + (date'2024-01-01', NULL, 4), + (NULL, NULL, 5) + +-- column - column in both directions, covering negative and zero spans +query +SELECT d1, d2, d1 - d2, d2 - d1 FROM test_subtract_dates + +-- literal on either side +query +SELECT d1 - date'2024-01-01', date'2024-01-01' - d2 FROM test_subtract_dates + +-- all-literal operands (constant folding is disabled by the test suite). A NULL literal operand +-- is left out: NullPropagation folds it to a null interval literal, and the native literal +-- path rejects CalendarIntervalType (#5058). NULL operands are covered by the column rows above. +query +SELECT + date'2024-03-15' - date'2024-01-01', + date'2024-01-01' - date'2024-03-15', + date'2024-01-01' - date'2024-01-01' + +-- interval output through native shuffle, at top level and nested in a struct +query +SELECT k, d1 - d2 AS i, named_struct('i', d2 - d1) AS s +FROM test_subtract_dates +DISTRIBUTE BY k diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/subtract_timestamps.sql b/spark/src/test/resources/sql-tests/expressions/datetime/subtract_timestamps.sql new file mode 100644 index 00000000000..bebb7e709bb --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/datetime/subtract_timestamps.sql @@ -0,0 +1,98 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- timestamp - timestamp resolves to SubtractTimestamps and runs through the codegen dispatcher +-- so results match Spark exactly. The output type follows spark.sql.legacy.interval.enabled: +-- a DayTimeIntervalType measured on local wall-clock time by default, a CalendarIntervalType +-- holding the elapsed microseconds in legacy mode. Legacy mode falls back to Spark and is covered +-- by subtract_timestamps_legacy.sql. America/Los_Angeles is pinned so the DST rows below +-- straddle real transitions. +-- Config: spark.sql.session.timeZone=America/Los_Angeles +-- Config: spark.comet.exec.scalaUDF.codegen.enabled=true +-- Config: spark.comet.shuffle.mode=native +-- Config: spark.sql.legacy.interval.enabled=false + +statement +CREATE TABLE test_subtract_timestamps( + ts1 timestamp, + ts2 timestamp, + ntz1 timestamp_ntz, + ntz2 timestamp_ntz, + d date, + k int) USING parquet + +-- rows 3 and 4 span the spring-forward and fall-back transitions +statement +INSERT INTO test_subtract_timestamps VALUES + (timestamp'2024-03-15 10:30:45.123456', timestamp'2024-01-01 00:00:00', + timestamp_ntz'2024-03-15 10:30:45.123456', timestamp_ntz'2024-01-01 00:00:00', + date'2024-01-01', 1), + (timestamp'2024-01-01 00:00:00', timestamp'2024-03-15 10:30:45.123456', + timestamp_ntz'2024-01-01 00:00:00', timestamp_ntz'2024-03-15 10:30:45.123456', + date'2024-06-30', 1), + (timestamp'2024-03-10 12:00:00', timestamp'2024-03-09 12:00:00', + timestamp_ntz'2024-03-10 12:00:00', timestamp_ntz'2024-03-09 12:00:00', + date'2024-03-09', 2), + (timestamp'2024-11-03 12:00:00', timestamp'2024-11-02 12:00:00', + timestamp_ntz'2024-11-03 12:00:00', timestamp_ntz'2024-11-02 12:00:00', + date'2024-11-02', 2), + (timestamp'1969-12-31 23:59:59.999999', timestamp'1970-01-01 00:00:00', + timestamp_ntz'1969-12-31 23:59:59.999999', timestamp_ntz'1970-01-01 00:00:00', + date'1970-01-01', 3), + (timestamp'2024-06-01 08:00:00', timestamp'2024-06-01 08:00:00', + timestamp_ntz'2024-06-01 08:00:00', timestamp_ntz'2024-06-01 08:00:00', + date'2024-06-01', 3), + (NULL, timestamp'2024-01-01 00:00:00', NULL, timestamp_ntz'2024-01-01 00:00:00', + date'2024-01-01', 4), + (timestamp'2024-01-01 00:00:00', NULL, timestamp_ntz'2024-01-01 00:00:00', NULL, NULL, 4), + (NULL, NULL, NULL, NULL, NULL, 5) + +-- TIMESTAMP columns in both directions. Across a DST transition the default mode reports one +-- calendar day rather than the elapsed 23 or 25 hours. +query +SELECT ts1, ts2, ts1 - ts2, ts2 - ts1 FROM test_subtract_timestamps + +-- TIMESTAMP_NTZ columns compile a separate kernel and never see the session time zone +query +SELECT ntz1 - ntz2, ntz2 - ntz1 FROM test_subtract_timestamps + +-- a DATE operand is implicitly cast to TIMESTAMP inside the kernel +query +SELECT ts1 - d, d - ts1 FROM test_subtract_timestamps + +-- literal on either side +query +SELECT + ts1 - timestamp'2024-01-01 00:00:00', + timestamp'2024-01-01 00:00:00' - ts2, + ntz1 - timestamp_ntz'2024-01-01 00:00:00' +FROM test_subtract_timestamps + +-- all-literal operands (constant folding is disabled by the test suite). A NULL literal operand +-- is left out: NullPropagation folds it to a null interval literal, and the native literal +-- path rejects CalendarIntervalType (#5058). NULL operands are covered by the column rows above. +query +SELECT + timestamp'2024-03-15 10:30:45.123456' - timestamp'2024-01-01 00:00:00', + timestamp'2024-01-01 00:00:00' - timestamp'2024-03-15 10:30:45.123456', + timestamp_ntz'2024-03-10 12:00:00' - timestamp_ntz'2024-03-09 12:00:00' + +-- interval output through native shuffle, at top level and nested in a struct +query +SELECT k, ts1 - ts2 AS i, named_struct('i', ntz1 - ntz2) AS s +FROM test_subtract_timestamps +DISTRIBUTE BY k diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/subtract_timestamps_legacy.sql b/spark/src/test/resources/sql-tests/expressions/datetime/subtract_timestamps_legacy.sql new file mode 100644 index 00000000000..1ac7c20cb86 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/datetime/subtract_timestamps_legacy.sql @@ -0,0 +1,87 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- timestamp - timestamp in legacy interval mode resolves to SubtractTimestamps with a +-- CalendarIntervalType result holding the elapsed microseconds. The codegen dispatcher's +-- calendar-interval output cannot carry that span past about 292 years, so the expression falls +-- back to Spark. America/Los_Angeles is pinned so the DST rows below straddle real transitions. +-- Config: spark.sql.session.timeZone=America/Los_Angeles +-- Config: spark.comet.exec.scalaUDF.codegen.enabled=true +-- Config: spark.sql.legacy.interval.enabled=true + +statement +CREATE TABLE test_subtract_timestamps_legacy( + ts1 timestamp, + ts2 timestamp, + ntz1 timestamp_ntz, + ntz2 timestamp_ntz, + d date, + k int) USING parquet + +-- rows 3 and 4 span the spring-forward and fall-back transitions +statement +INSERT INTO test_subtract_timestamps_legacy VALUES + (timestamp'2024-03-15 10:30:45.123456', timestamp'2024-01-01 00:00:00', + timestamp_ntz'2024-03-15 10:30:45.123456', timestamp_ntz'2024-01-01 00:00:00', + date'2024-01-01', 1), + (timestamp'2024-01-01 00:00:00', timestamp'2024-03-15 10:30:45.123456', + timestamp_ntz'2024-01-01 00:00:00', timestamp_ntz'2024-03-15 10:30:45.123456', + date'2024-06-30', 1), + (timestamp'2024-03-10 12:00:00', timestamp'2024-03-09 12:00:00', + timestamp_ntz'2024-03-10 12:00:00', timestamp_ntz'2024-03-09 12:00:00', + date'2024-03-09', 2), + (timestamp'2024-11-03 12:00:00', timestamp'2024-11-02 12:00:00', + timestamp_ntz'2024-11-03 12:00:00', timestamp_ntz'2024-11-02 12:00:00', + date'2024-11-02', 2), + (timestamp'1969-12-31 23:59:59.999999', timestamp'1970-01-01 00:00:00', + timestamp_ntz'1969-12-31 23:59:59.999999', timestamp_ntz'1970-01-01 00:00:00', + date'1970-01-01', 3), + (timestamp'2024-06-01 08:00:00', timestamp'2024-06-01 08:00:00', + timestamp_ntz'2024-06-01 08:00:00', timestamp_ntz'2024-06-01 08:00:00', + date'2024-06-01', 3), + (NULL, timestamp'2024-01-01 00:00:00', NULL, timestamp_ntz'2024-01-01 00:00:00', + date'2024-01-01', 4), + (timestamp'2024-01-01 00:00:00', NULL, timestamp_ntz'2024-01-01 00:00:00', NULL, NULL, 4), + (NULL, NULL, NULL, NULL, NULL, 5) + +-- TIMESTAMP columns in both directions. Across a DST transition legacy mode reports the elapsed +-- 23 or 25 hours rather than one calendar day. +query expect_fallback(cannot carry a span past about 292 years) +SELECT ts1, ts2, ts1 - ts2, ts2 - ts1 FROM test_subtract_timestamps_legacy + +-- TIMESTAMP_NTZ columns never see the session time zone +query expect_fallback(cannot carry a span past about 292 years) +SELECT ntz1 - ntz2, ntz2 - ntz1 FROM test_subtract_timestamps_legacy + +-- a DATE operand is implicitly cast to TIMESTAMP +query expect_fallback(cannot carry a span past about 292 years) +SELECT ts1 - d, d - ts1 FROM test_subtract_timestamps_legacy + +-- literal on either side +query expect_fallback(cannot carry a span past about 292 years) +SELECT + ts1 - timestamp'2024-01-01 00:00:00', + timestamp'2024-01-01 00:00:00' - ts2, + ntz1 - timestamp_ntz'2024-01-01 00:00:00' +FROM test_subtract_timestamps_legacy + +-- all-literal operands (constant folding is disabled by the test suite) +query expect_fallback(cannot carry a span past about 292 years) +SELECT + timestamp'2024-03-15 10:30:45.123456' - timestamp'2024-01-01 00:00:00', + timestamp'2024-01-01 00:00:00' - timestamp'2024-03-15 10:30:45.123456', + timestamp_ntz'2024-03-10 12:00:00' - timestamp_ntz'2024-03-09 12:00:00' diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/subtract_timestamps_long_span.sql b/spark/src/test/resources/sql-tests/expressions/datetime/subtract_timestamps_long_span.sql new file mode 100644 index 00000000000..fa213c2bf59 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/datetime/subtract_timestamps_long_span.sql @@ -0,0 +1,46 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- timestamp - timestamp over a span past about 292 years. In default mode the result is a +-- DayTimeIntervalType that holds the microseconds in a long, so the span fits and the expression +-- stays on the codegen dispatcher. UTC is pinned so the elapsed microseconds are exact. +-- Config: spark.sql.session.timeZone=UTC +-- Config: spark.comet.exec.scalaUDF.codegen.enabled=true +-- Config: spark.sql.legacy.interval.enabled=false + +statement +CREATE TABLE test_subtract_timestamps_long_span( + ts1 timestamp, + ts2 timestamp, + ntz1 timestamp_ntz, + ntz2 timestamp_ntz) USING parquet + +-- 2300-01-01 - 1970-01-01 is 10413792000000000 microseconds, about 330 years +statement +INSERT INTO test_subtract_timestamps_long_span VALUES + (timestamp'2300-01-01 00:00:00', timestamp'1970-01-01 00:00:00', + timestamp_ntz'2300-01-01 00:00:00', timestamp_ntz'1970-01-01 00:00:00'), + (timestamp'1970-01-01 00:00:00', timestamp'2300-01-01 00:00:00', + timestamp_ntz'1970-01-01 00:00:00', timestamp_ntz'2300-01-01 00:00:00') + +-- TIMESTAMP columns in both operand orders +query +SELECT ts1, ts2, ts1 - ts2, ts2 - ts1 FROM test_subtract_timestamps_long_span + +-- TIMESTAMP_NTZ columns in both operand orders +query +SELECT ntz1 - ntz2, ntz2 - ntz1 FROM test_subtract_timestamps_long_span diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/subtract_timestamps_long_span_legacy.sql b/spark/src/test/resources/sql-tests/expressions/datetime/subtract_timestamps_long_span_legacy.sql new file mode 100644 index 00000000000..58c5d8d63fa --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/datetime/subtract_timestamps_long_span_legacy.sql @@ -0,0 +1,47 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- timestamp - timestamp over a span past about 292 years in legacy interval mode. The result is +-- a CalendarIntervalType whose microseconds the codegen dispatcher writes as nanoseconds, which +-- overflows a long for this span, so the expression must fall back to Spark. UTC is pinned so the +-- elapsed microseconds are exact. +-- Config: spark.sql.session.timeZone=UTC +-- Config: spark.comet.exec.scalaUDF.codegen.enabled=true +-- Config: spark.sql.legacy.interval.enabled=true + +statement +CREATE TABLE test_subtract_timestamps_long_span_legacy( + ts1 timestamp, + ts2 timestamp, + ntz1 timestamp_ntz, + ntz2 timestamp_ntz) USING parquet + +-- 2300-01-01 - 1970-01-01 is 10413792000000000 microseconds, past the dispatcher's limit +statement +INSERT INTO test_subtract_timestamps_long_span_legacy VALUES + (timestamp'2300-01-01 00:00:00', timestamp'1970-01-01 00:00:00', + timestamp_ntz'2300-01-01 00:00:00', timestamp_ntz'1970-01-01 00:00:00'), + (timestamp'1970-01-01 00:00:00', timestamp'2300-01-01 00:00:00', + timestamp_ntz'1970-01-01 00:00:00', timestamp_ntz'2300-01-01 00:00:00') + +-- TIMESTAMP columns in both operand orders +query expect_fallback(cannot carry a span past about 292 years) +SELECT ts1, ts2, ts1 - ts2, ts2 - ts1 FROM test_subtract_timestamps_long_span_legacy + +-- TIMESTAMP_NTZ columns in both operand orders +query expect_fallback(cannot carry a span past about 292 years) +SELECT ntz1 - ntz2, ntz2 - ntz1 FROM test_subtract_timestamps_long_span_legacy diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/timestamp_add_interval.sql b/spark/src/test/resources/sql-tests/expressions/datetime/timestamp_add_interval.sql new file mode 100644 index 00000000000..6aa0354f68e --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/datetime/timestamp_add_interval.sql @@ -0,0 +1,113 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- timestamp + day-time or calendar interval resolves to TimeAdd (TimestampAddInterval on +-- Spark 4.1+) and runs through the codegen dispatcher so results match Spark exactly. Days and +-- months are applied on local time in the session zone, so America/Los_Angeles is pinned and +-- the DST rows straddle the spring-forward and fall-back transitions. +-- Config: spark.sql.session.timeZone=America/Los_Angeles +-- Config: spark.comet.exec.scalaUDF.codegen.enabled=true +-- Config: spark.comet.shuffle.mode=native + +statement +CREATE TABLE test_timestamp_add_interval( + ts timestamp, + ts_ntz timestamp_ntz, + d date, + dd int, + h int, + mi int, + s decimal(18, 6), + k int) USING parquet + +statement +INSERT INTO test_timestamp_add_interval VALUES + (timestamp'2024-01-15 10:30:45.123456', timestamp_ntz'2024-01-15 10:30:45.123456', + date'2024-01-15', 1, 2, 3, 4.5, 1), + (timestamp'2024-01-31 23:00:00', timestamp_ntz'2024-01-31 23:00:00', + date'2024-01-31', 1, 1, 0, 0, 1), + (timestamp'2024-03-09 12:00:00', timestamp_ntz'2024-03-09 12:00:00', + date'2024-03-09', 1, 0, 0, 0, 2), + (timestamp'2024-03-09 12:00:00', timestamp_ntz'2024-03-09 12:00:00', + date'2024-03-09', 0, 24, 0, 0, 2), + (timestamp'2024-11-02 12:00:00', timestamp_ntz'2024-11-02 12:00:00', + date'2024-11-02', 1, 0, 0, 0, 3), + (timestamp'2024-11-02 12:00:00', timestamp_ntz'2024-11-02 12:00:00', + date'2024-11-02', 0, 24, 0, 0, 3), + (timestamp'2024-12-31 23:59:59.999999', timestamp_ntz'2024-12-31 23:59:59.999999', + date'2024-12-31', 0, 0, 0, 0.000001, 4), + (timestamp'1970-01-01 00:00:00', timestamp_ntz'1970-01-01 00:00:00', + date'1970-01-01', -1, -1, -1, -1.5, 4), + (timestamp'2024-06-15 08:00:00', timestamp_ntz'2024-06-15 08:00:00', + date'2024-06-15', NULL, 1, 1, 1, 5), + (timestamp'2024-06-15 08:00:00', timestamp_ntz'2024-06-15 08:00:00', + date'2024-06-15', 1, 1, 1, NULL, 5), + (NULL, NULL, NULL, 1, 1, 1, 1, 6) + +-- TIMESTAMP column plus a day-time interval built from columns, both directions +query +SELECT ts, dd, h, mi, s, ts + make_dt_interval(dd, h, mi, s), make_dt_interval(dd, h, mi, s) + ts +FROM test_timestamp_add_interval + +-- TIMESTAMP column plus a calendar interval built from columns, both directions +query +SELECT ts + make_interval(0, 1, 0, dd, h, mi, s), make_interval(0, 1, 0, dd, h, mi, s) + ts +FROM test_timestamp_add_interval + +-- TIMESTAMP_NTZ compiles a separate kernel and never sees the session time zone +query +SELECT ts_ntz + make_dt_interval(dd, h, mi, s), ts_ntz + make_interval(0, 1, 0, dd, h, mi, s) +FROM test_timestamp_add_interval + +-- a DATE operand is cast to TIMESTAMP before a day-time interval finer than a day is added +query +SELECT d + make_dt_interval(dd, h, mi, s), d + INTERVAL '1 12:00:00' DAY TO SECOND +FROM test_timestamp_add_interval + +-- day-time interval literals in the unit, unit-to-unit and multi-unit spellings. The parser +-- rejects literals that mix year-month and day-time units unless spark.sql.legacy.interval.enabled +-- is set, so literal calendar intervals come from make_interval. Subtraction rewrites to an +-- addition of the negated interval. +query +SELECT + ts + INTERVAL '1' DAY, + ts + INTERVAL '36' HOUR, + ts + INTERVAL '1 02:30:00.5' DAY TO SECOND, + ts + INTERVAL '1 day 2 hours', + ts + make_interval(0, 1, 0, 1, 2), + ts - INTERVAL '1' DAY, + ts - make_interval(0, 1, 0, 1, 2), + ts_ntz + INTERVAL '1' DAY, + ts_ntz - make_interval(0, 1, 0, 1) +FROM test_timestamp_add_interval + +-- all-literal operands (constant folding is disabled by the test suite) +query +SELECT + timestamp'2024-03-09 12:00:00' + INTERVAL '1' DAY, + timestamp'2024-03-09 12:00:00' + INTERVAL '24' HOUR, + timestamp'2024-01-31 23:00:00' + make_interval(0, 1, 0, 1), + timestamp_ntz'2024-03-09 12:00:00' + INTERVAL '1' DAY, + CAST(NULL AS TIMESTAMP) + INTERVAL '1' DAY, + timestamp'2024-01-31 00:00:00' + CAST(NULL AS INTERVAL DAY TO SECOND), + timestamp'2024-01-31 00:00:00' + CAST(NULL AS INTERVAL) + +-- timestamp output through native shuffle +query +SELECT k, ts + make_dt_interval(dd, h, mi, s) AS r, ts_ntz + make_interval(0, 1, 0, dd) AS r_ntz +FROM test_timestamp_add_interval +DISTRIBUTE BY k diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/timestamp_add_ym_interval.sql b/spark/src/test/resources/sql-tests/expressions/datetime/timestamp_add_ym_interval.sql new file mode 100644 index 00000000000..daa4d6ff13f --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/datetime/timestamp_add_ym_interval.sql @@ -0,0 +1,84 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- timestamp + year-month interval resolves to TimestampAddYMInterval and runs through the +-- codegen dispatcher so results match Spark exactly. TIMESTAMP values are shifted on local +-- time in the session zone, so America/Los_Angeles is pinned and two rows cross a DST change. +-- Config: spark.sql.session.timeZone=America/Los_Angeles +-- Config: spark.comet.exec.scalaUDF.codegen.enabled=true +-- Config: spark.comet.shuffle.mode=native + +statement +CREATE TABLE test_timestamp_add_ym_interval( + ts timestamp, + ts_ntz timestamp_ntz, + y int, + m int, + k int) USING parquet + +-- rows 5 and 6 cross the spring-forward and fall-back transitions +statement +INSERT INTO test_timestamp_add_ym_interval VALUES + (timestamp'2024-01-31 10:30:45.123456', timestamp_ntz'2024-01-31 10:30:45.123456', 0, 1, 1), + (timestamp'2024-01-31 10:30:45.123456', timestamp_ntz'2024-01-31 10:30:45.123456', 0, 13, 1), + (timestamp'2024-02-29 23:59:59', timestamp_ntz'2024-02-29 23:59:59', 1, 0, 2), + (timestamp'2024-03-31 00:00:00', timestamp_ntz'2024-03-31 00:00:00', 0, -1, 2), + (timestamp'2024-02-10 12:00:00', timestamp_ntz'2024-02-10 12:00:00', 0, 1, 3), + (timestamp'2024-10-15 12:00:00', timestamp_ntz'2024-10-15 12:00:00', 0, 1, 3), + (timestamp'1970-01-01 00:00:00', timestamp_ntz'1970-01-01 00:00:00', -1, -1, 4), + (timestamp'2024-06-15 08:00:00', timestamp_ntz'2024-06-15 08:00:00', NULL, 1, 5), + (timestamp'2024-06-15 08:00:00', timestamp_ntz'2024-06-15 08:00:00', 1, NULL, 5), + (NULL, NULL, 1, 1, 6) + +-- TIMESTAMP column plus an interval built from columns, both directions +query +SELECT ts, y, m, ts + make_ym_interval(y, m), make_ym_interval(y, m) + ts +FROM test_timestamp_add_ym_interval + +-- TIMESTAMP_NTZ compiles a separate kernel and never sees the session time zone +query +SELECT ts_ntz, ts_ntz + make_ym_interval(y, m), make_ym_interval(y, m) + ts_ntz +FROM test_timestamp_add_ym_interval + +-- interval literals in the unit, unit-to-unit and multi-unit spellings. Subtraction rewrites +-- to an addition of the negated interval. +query +SELECT + ts + INTERVAL '1' YEAR, + ts + INTERVAL '1' MONTH, + ts + INTERVAL '1-2' YEAR TO MONTH, + ts + INTERVAL '1 year 1 month', + ts - INTERVAL '1' MONTH, + ts_ntz + INTERVAL '1' YEAR, + ts_ntz - INTERVAL '1' MONTH +FROM test_timestamp_add_ym_interval + +-- all-literal operands (constant folding is disabled by the test suite) +query +SELECT + timestamp'2024-01-31 10:30:45.123456' + INTERVAL '1' MONTH, + timestamp'2024-02-29 23:59:59' + INTERVAL '1' YEAR, + timestamp_ntz'2024-01-31 10:30:45.123456' + INTERVAL '1' MONTH, + timestamp'2024-03-31 00:00:00' - INTERVAL '1' MONTH, + CAST(NULL AS TIMESTAMP) + INTERVAL '1' MONTH, + timestamp'2024-01-31 00:00:00' + CAST(NULL AS INTERVAL YEAR TO MONTH) + +-- timestamp output through native shuffle +query +SELECT k, ts + make_ym_interval(y, m) AS r, ts_ntz + make_ym_interval(y, m) AS r_ntz +FROM test_timestamp_add_ym_interval +DISTRIBUTE BY k