diff --git a/.github/workflows/pr_build_linux.yml b/.github/workflows/pr_build_linux.yml index 6814d75769b..4a4643380ce 100644 --- a/.github/workflows/pr_build_linux.yml +++ b/.github/workflows/pr_build_linux.yml @@ -166,6 +166,42 @@ jobs: run: | ./dev/ci/check-working-tree-clean.sh + # Compiles main and test sources with -Pstrict-warnings, which promotes scalac warnings to + # errors, so a change that reintroduces a cleared warning (an Int widened into a Long metric, + # a discarded builder result) fails here rather than accumulating. Spark 3.5 only: the profile + # passes on the Scala 2.12 profiles, and the Scala 2.13 remainder is tracked in + # https://github.com/apache/datafusion-comet/issues/5893. + strict-scala-warnings: + needs: lint + name: Strict Scala warnings (Spark 3.5, JDK 17) + runs-on: ubuntu-24.04 + container: + image: amd64/rust + steps: + - uses: actions/checkout@v7 + + - name: Setup Rust & Java toolchain + uses: ./.github/actions/setup-builder + with: + rust-version: ${{ env.RUST_VERSION }} + jdk-version: 17 + + - name: Cache Maven dependencies + uses: actions/cache@v6 + with: + path: | + ~/.m2/repository + /root/.m2/repository + key: ${{ runner.os }}-java-maven-${{ hashFiles('**/pom.xml') }}-strict-warnings + restore-keys: | + ${{ runner.os }}-java-maven- + + - name: Bootstrap Maven + uses: ./.github/actions/maven-bootstrap + + - name: Compile with strict warnings + run: ./mvnw -B test-compile -Pspark-3.5 -Pstrict-warnings -DskipTests + # Compile-only verification for Spark 4.1. Tests are intentionally skipped: the spark-4.1 # profile is currently a build target only, and several runtime/test failures are tracked # in follow-up PRs. Excluded from lint-java because semanticdb-scalac_2.13.17 is not yet diff --git a/pom.xml b/pom.xml index f4b2be220ec..50c8dbd48f3 100644 --- a/pom.xml +++ b/pom.xml @@ -859,29 +859,84 @@ under the License. + - strict-warnings - - - - net.alchim31.maven - scala-maven-plugin - - - -deprecation - -unchecked - -feature - -Xlint:_ - -Ywarn-dead-code - -Ywarn-numeric-widen - -Ywarn-value-discard - -Ywarn-unused:imports,patvars,privates,locals,params,-implicits - -Xfatal-warnings - - - - - + strict-warnings + + + + net.alchim31.maven + scala-maven-plugin + + + scala-compile-first + + + -deprecation + -unchecked + -feature + -Xlint:_ + -Ywarn-dead-code + -Ywarn-numeric-widen + -Ywarn-value-discard + -Ywarn-unused:imports,patvars,privates,locals,-implicits + -Xfatal-warnings + + + + + scala-test-compile-first + + + -deprecation + -unchecked + -feature + -Xlint:_ + -Ywarn-dead-code + -Ywarn-numeric-widen + -Ywarn-unused:imports,patvars,privates,locals,-implicits + -Xfatal-warnings + + + + + + + diff --git a/spark/src/main/scala/org/apache/comet/CometExecIterator.scala b/spark/src/main/scala/org/apache/comet/CometExecIterator.scala index e2c132904d5..22c17cb8e3d 100644 --- a/spark/src/main/scala/org/apache/comet/CometExecIterator.scala +++ b/spark/src/main/scala/org/apache/comet/CometExecIterator.scala @@ -86,7 +86,7 @@ class CometExecIterator( private val nativeLib = new Native() private val nativeUtil = new NativeUtil() private val taskAttemptId = TaskContext.get().taskAttemptId() - private val taskCPUs = TaskContext.get().cpus() + private val taskCPUs = TaskContext.get().cpus().toLong private val cometTaskMemoryManager = new CometTaskMemoryManager(id, taskAttemptId) private val plan = { diff --git a/spark/src/main/scala/org/apache/comet/iceberg/IcebergReflection.scala b/spark/src/main/scala/org/apache/comet/iceberg/IcebergReflection.scala index ba41a957288..645b657da9d 100644 --- a/spark/src/main/scala/org/apache/comet/iceberg/IcebergReflection.scala +++ b/spark/src/main/scala/org/apache/comet/iceberg/IcebergReflection.scala @@ -436,13 +436,13 @@ object IcebergReflection extends Logging { * `taskGroups()`, so for staged scans we flatten the groups instead. Both methods are protected * and require reflection. */ - def getTasks(scan: Any): Option[java.util.List[_]] = + def getTasks(scan: Any): Option[java.util.List[AnyRef]] = if (isStagedScan(scan)) tasksFromTaskGroups(scan) else tasksFromTasksAccessor(scan) - private def tasksFromTasksAccessor(scan: Any): Option[java.util.List[_]] = + private def tasksFromTasksAccessor(scan: Any): Option[java.util.List[AnyRef]] = findMethodInHierarchy(scan.getClass, "tasks") match { case Some(method) => - Some(method.invoke(scan).asInstanceOf[java.util.List[_]]) + Some(method.invoke(scan).asInstanceOf[java.util.List[AnyRef]]) case None => logError( "Iceberg reflection failure: Failed to get tasks from SparkScan: " + @@ -450,7 +450,7 @@ object IcebergReflection extends Logging { None } - private def tasksFromTaskGroups(scan: Any): Option[java.util.List[_]] = + private def tasksFromTaskGroups(scan: Any): Option[java.util.List[AnyRef]] = findMethodInHierarchy(scan.getClass, "taskGroups") match { case Some(method) => try { @@ -465,7 +465,7 @@ object IcebergReflection extends Logging { groups.forEach { group => val groupTasks = groupTasksMethod.invoke(group).asInstanceOf[java.util.Collection[_ <: AnyRef]] - flat.addAll(groupTasks) + val _ = flat.addAll(groupTasks) } Some(flat) } @@ -1688,13 +1688,19 @@ object IcebergReflection extends Logging { private def newDataManifestFile(inputFile: AnyRef, specId: Int): AnyRef = { val inputFileClass = loadClass(ClassNames.INPUT_FILE) val cls = loadClass(ClassNames.GENERIC_MANIFEST_FILE) - val (ctor, args): (java.lang.reflect.Constructor[_], Array[Object]) = + // `Constructor[AnyRef]` rather than `Constructor[_]`: the two `try`/`catch` branches + // would otherwise infer a top-level existential, which `-Xlint:existential` rejects. + val (ctor, args): (java.lang.reflect.Constructor[AnyRef], Array[Object]) = try { - val c = cls.getDeclaredConstructor(inputFileClass, classOf[Int], classOf[Long]) + val c = cls + .getDeclaredConstructor(inputFileClass, classOf[Int], classOf[Long]) + .asInstanceOf[java.lang.reflect.Constructor[AnyRef]] (c, Array[Object](inputFile, Integer.valueOf(specId), java.lang.Long.valueOf(0L))) } catch { case _: NoSuchMethodException => - val c = cls.getDeclaredConstructor(inputFileClass, classOf[Int]) + val c = cls + .getDeclaredConstructor(inputFileClass, classOf[Int]) + .asInstanceOf[java.lang.reflect.Constructor[AnyRef]] (c, Array[Object](inputFile, Integer.valueOf(specId))) } ctor.setAccessible(true) @@ -1730,10 +1736,11 @@ object IcebergReflection extends Logging { } result } finally { - try reader.getClass.getMethod("close").invoke(reader) - catch { - case e: Exception => logWarning(s"Failed to close ManifestReader: ${e.getMessage}") - } + val _ = + try reader.getClass.getMethod("close").invoke(reader) + catch { + case e: Exception => logWarning(s"Failed to close ManifestReader: ${e.getMessage}") + } } } diff --git a/spark/src/main/scala/org/apache/comet/rules/CometExecRule.scala b/spark/src/main/scala/org/apache/comet/rules/CometExecRule.scala index f8ec2e6ab29..5e45380b25a 100644 --- a/spark/src/main/scala/org/apache/comet/rules/CometExecRule.scala +++ b/spark/src/main/scala/org/apache/comet/rules/CometExecRule.scala @@ -911,7 +911,7 @@ case class CometExecRule(session: SparkSession) .flatten .toSet if (reasons.nonEmpty) { - withFallbackReasons(op, reasons) + val _ = withFallbackReasons(op, reasons) } } @@ -943,7 +943,7 @@ case class CometExecRule(session: SparkSession) "operator or any of its expressions. Add a withFallbackReason call stating why " + s"conversion failed. Operator:\n$op") } - withFallbackReason(op, s"${op.nodeName} is not supported") + val _ = withFallbackReason(op, s"${op.nodeName} is not supported") } } @@ -972,7 +972,8 @@ case class CometExecRule(session: SparkSession) CometExplainInfo.collectExprTagValues(allExprs, CometExplainInfo.CODEGEN_DISPATCH_EXPRS) appendTagValues(exec, CometExplainInfo.CODEGEN_DISPATCH_EXPRS, routedNames) if (routedNames.nonEmpty && CometConf.COMET_EXPLAIN_CODEGEN_ENABLED.get()) { - withInfo(exec, s"JVM codegen dispatcher: ${routedNames.toSeq.sorted.mkString(", ")}") + val _ = + withInfo(exec, s"JVM codegen dispatcher: ${routedNames.toSeq.sorted.mkString(", ")}") } } diff --git a/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala b/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala index e3b6f77eb8e..fb2582b2335 100644 --- a/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala +++ b/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala @@ -31,7 +31,7 @@ import scala.jdk.CollectionConverters._ import org.apache.hadoop.conf.Configuration import org.apache.spark.internal.Logging import org.apache.spark.sql.SparkSession -import org.apache.spark.sql.catalyst.expressions.{Attribute, DynamicPruningExpression, Expression, GenericInternalRow, InputFileBlockLength, InputFileBlockStart, InputFileName, PlanExpression} +import org.apache.spark.sql.catalyst.expressions.{Attribute, DynamicPruningExpression, Expression, GenericInternalRow, InputFileBlockLength, InputFileBlockStart, InputFileName} import org.apache.spark.sql.catalyst.rules.Rule import org.apache.spark.sql.catalyst.util.{sideBySide, ArrayBasedMapData, GenericArrayData, MetadataColumnHelper} import org.apache.spark.sql.catalyst.util.ResolveDefaultColumns.getExistenceDefaultValues @@ -1043,9 +1043,6 @@ case class CometScanRule(session: SparkSession) } } - private def isDynamicPruningFilter(e: Expression): Boolean = - e.exists(_.isInstanceOf[PlanExpression[_]]) - /** * Detects AQE DPP (SubqueryAdaptiveBroadcastExec), as opposed to non-AQE DPP. * diff --git a/spark/src/main/scala/org/apache/comet/serde/CometScalaUDF.scala b/spark/src/main/scala/org/apache/comet/serde/CometScalaUDF.scala index 9249ab280f0..57d609c158b 100644 --- a/spark/src/main/scala/org/apache/comet/serde/CometScalaUDF.scala +++ b/spark/src/main/scala/org/apache/comet/serde/CometScalaUDF.scala @@ -181,7 +181,7 @@ object CometScalaUDF extends CometExpressionSerde[ScalaUDF] { target.foreach { case _: AttributeReference | _: Literal => case node if !(node eq target) => - withCodegenDispatchExpr(expr, CometExplainInfo.exprDisplayName(node)) + val _ = withCodegenDispatchExpr(expr, CometExplainInfo.exprDisplayName(node)) case _ => } Some( diff --git a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala index be4bc9c3412..1da6937357a 100644 --- a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala +++ b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala @@ -911,7 +911,7 @@ object QueryPlanSerde extends Logging with CometExprShim with CometTypeShim { e.getTagValue(CometExplainInfo.FALLBACK_REASONS).foreach(reasons ++= _) } if (reasons.nonEmpty) { - withFallbackReasons(to, reasons.toSet) + val _ = withFallbackReasons(to, reasons.toSet) } } diff --git a/spark/src/main/scala/org/apache/comet/serde/literals.scala b/spark/src/main/scala/org/apache/comet/serde/literals.scala index b368f80320c..98a3757ffc8 100644 --- a/spark/src/main/scala/org/apache/comet/serde/literals.scala +++ b/spark/src/main/scala/org/apache/comet/serde/literals.scala @@ -86,8 +86,8 @@ object CometLiteral extends CometExpressionSerde[Literal] with CometTypeShim wit exprBuilder.setIsNull(false) dataType match { case _: BooleanType => exprBuilder.setBoolVal(value.asInstanceOf[Boolean]) - case _: ByteType => exprBuilder.setByteVal(value.asInstanceOf[Byte]) - case _: ShortType => exprBuilder.setShortVal(value.asInstanceOf[Short]) + case _: ByteType => exprBuilder.setByteVal(value.asInstanceOf[Byte].toInt) + case _: ShortType => exprBuilder.setShortVal(value.asInstanceOf[Short].toInt) case _: IntegerType | _: DateType => exprBuilder.setIntVal(value.asInstanceOf[Int]) case _: LongType | _: TimestampType | _: TimestampNTZType | _: DayTimeIntervalType => exprBuilder.setLongVal(value.asInstanceOf[Long]) diff --git a/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeScan.scala b/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeScan.scala index ac2538288e2..256a7b09fa3 100644 --- a/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeScan.scala +++ b/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeScan.scala @@ -353,7 +353,7 @@ object CometIcebergNativeScan extends CometOperatorSerde[CometBatchScanExec] wit val equalityIds = equalityIdsMethod .invoke(deleteFile) .asInstanceOf[java.util.List[Integer]] - equalityIds.forEach(id => deleteBuilder.addEqualityIds(id)) + deleteBuilder.addAllEqualityIds(equalityIds) } catch { case _: Exception => } @@ -515,7 +515,7 @@ object CometIcebergNativeScan extends CometOperatorSerde[CometBatchScanExec] wit commonBuilder.addPartitionTypePool(partitionTypeJson) idx }) - taskBuilder.setPartitionSpecIdx(specIdx) + val _ = taskBuilder.setPartitionSpecIdx(specIdx) } catch { case e: Exception => logWarning(s"Failed to serialize partition spec to JSON: ${e.getMessage}") @@ -575,7 +575,7 @@ object CometIcebergNativeScan extends CometOperatorSerde[CometBatchScanExec] wit commonBuilder.addPartitionDataPool(partitionDataProto) idx }) - taskBuilder.setPartitionDataIdx(partitionDataIdx) + val _ = taskBuilder.setPartitionDataIdx(partitionDataIdx) } else { // Defensive: ContentScanTask.partition() returns an empty struct (never null) for // unpartitioned tables in practice. If it is ever null we cannot compute values, so diff --git a/spark/src/main/scala/org/apache/comet/shuffle/CelebornShufflePusherFactory.scala b/spark/src/main/scala/org/apache/comet/shuffle/CelebornShufflePusherFactory.scala index 37d67f55ca4..27ecec8f559 100644 --- a/spark/src/main/scala/org/apache/comet/shuffle/CelebornShufflePusherFactory.scala +++ b/spark/src/main/scala/org/apache/comet/shuffle/CelebornShufflePusherFactory.scala @@ -246,7 +246,7 @@ object CelebornShufflePusherFactory { client: AnyRef, taskContext: TaskContext, handle: ShuffleHandle): Unit = { - sparkUtilsClass + val _ = sparkUtilsClass .getMethod( "addFailureListenerIfBarrierTask", shuffleClientClass, @@ -336,7 +336,7 @@ object CelebornShufflePusherFactory { /** Remove task-independent state for one unregistered Celeborn generation. */ def cleanupShuffle(client: AnyRef, celebornShuffleId: Int): Unit = { try { - client.getClass + val _ = client.getClass .getMethod("cleanupShuffle", java.lang.Integer.TYPE) .invoke(client, Int.box(celebornShuffleId)) } catch { @@ -354,7 +354,8 @@ object CelebornShufflePusherFactory { try { val shuffleClientClass = ClassLoaders.loadClass(CELEBORN_SHUFFLE_CLIENT) try { - shuffleClientClass.getMethod("removeInstance", shuffleClientClass).invoke(null, client) + val _ = + shuffleClientClass.getMethod("removeInstance", shuffleClientClass).invoke(null, client) } catch { case _: NoSuchMethodException => // Celeborn 0.6 owns one shared application client and cannot remove a single instance. diff --git a/spark/src/main/scala/org/apache/comet/udf/codegen/CometScalaUDFCodegen.scala b/spark/src/main/scala/org/apache/comet/udf/codegen/CometScalaUDFCodegen.scala index ef541fb3e2d..1cab882df93 100644 --- a/spark/src/main/scala/org/apache/comet/udf/codegen/CometScalaUDFCodegen.scala +++ b/spark/src/main/scala/org/apache/comet/udf/codegen/CometScalaUDFCodegen.scala @@ -296,7 +296,7 @@ object CometScalaUDFCodegen { private[codegen] def recordCompiledSignature( specs: IndexedSeq[ArrowColumnSpec], outputType: DataType): Unit = { - compiledSignatures.add((specs.map(_.vectorClass), outputType)) + val _ = compiledSignatures.add((specs.map(_.vectorClass), outputType)) } /** diff --git a/spark/src/main/scala/org/apache/spark/CometSource.scala b/spark/src/main/scala/org/apache/spark/CometSource.scala index 95d75236166..6581cb2018a 100644 --- a/spark/src/main/scala/org/apache/spark/CometSource.scala +++ b/spark/src/main/scala/org/apache/spark/CometSource.scala @@ -54,9 +54,9 @@ object CometSource extends Source { }) def recordStats(stats: CometCoverageStats): Unit = { - NATIVE_OPERATORS.inc(stats.cometOperators) - SPARK_OPERATORS.inc(stats.sparkOperators) - TRANSITIONS.inc(stats.transitions) + NATIVE_OPERATORS.inc(stats.cometOperators.toLong) + SPARK_OPERATORS.inc(stats.sparkOperators.toLong) + TRANSITIONS.inc(stats.transitions.toLong) QUERIES_PLANNED.inc() } } diff --git a/spark/src/main/scala/org/apache/spark/Plugins.scala b/spark/src/main/scala/org/apache/spark/Plugins.scala index eaeac316655..3e80b55c7f1 100644 --- a/spark/src/main/scala/org/apache/spark/Plugins.scala +++ b/spark/src/main/scala/org/apache/spark/Plugins.scala @@ -187,13 +187,13 @@ object CometDriverPlugin extends Logging { val listeners = sc.conf.get(listenerKey, "") if (listeners.isEmpty) { logInfo(s"Setting $listenerKey=$listenerClass") - sc.conf.set(listenerKey, listenerClass) + val _ = sc.conf.set(listenerKey, listenerClass) } else { val currentListeners = listeners.split(",").map(_.trim) if (!currentListeners.contains(listenerClass)) { val newValue = s"$listeners,$listenerClass" logInfo(s"Setting $listenerKey=$newValue") - sc.conf.set(listenerKey, newValue) + val _ = sc.conf.set(listenerKey, newValue) } } } else { @@ -208,13 +208,13 @@ object CometDriverPlugin extends Logging { val extensions = conf.get(extensionKey, "") if (extensions.isEmpty) { logInfo(s"Setting $extensionKey=$extensionClass") - conf.set(extensionKey, extensionClass) + val _ = conf.set(extensionKey, extensionClass) } else { val currentExtensions = extensions.split(",").map(_.trim) if (!currentExtensions.contains(extensionClass)) { val newValue = s"$extensions,$extensionClass" logInfo(s"Setting $extensionKey=$newValue") - conf.set(extensionKey, newValue) + val _ = conf.set(extensionKey, newValue) } } } diff --git a/spark/src/main/scala/org/apache/spark/shuffle/sort/RowPartition.scala b/spark/src/main/scala/org/apache/spark/shuffle/sort/RowPartition.scala index d3536850f68..887dae05bca 100644 --- a/spark/src/main/scala/org/apache/spark/shuffle/sort/RowPartition.scala +++ b/spark/src/main/scala/org/apache/spark/shuffle/sort/RowPartition.scala @@ -27,7 +27,7 @@ class RowPartition(initialSize: Int) { def addRow(addr: Long, size: Int): Unit = { rowAddresses += addr - rowSizes += size + val _ = rowSizes += size } def getNumRows: Int = if (rowAddresses == null) { diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/CometBatchScanExec.scala b/spark/src/main/scala/org/apache/spark/sql/comet/CometBatchScanExec.scala index d900889e176..4f807dc1c79 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/CometBatchScanExec.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/CometBatchScanExec.scala @@ -76,7 +76,7 @@ case class CometBatchScanExec( override def next(): ColumnarBatch = { val batch = batches.next() - numOutputRows += batch.numRows() + numOutputRows += batch.numRows().toLong batch } } diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/CometBroadcastExchangeExec.scala b/spark/src/main/scala/org/apache/spark/sql/comet/CometBroadcastExchangeExec.scala index 9ae92845c9c..615f7bada26 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/CometBroadcastExchangeExec.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/CometBroadcastExchangeExec.scala @@ -107,7 +107,7 @@ case class CometBroadcastExchangeExec( private val timeout: Long = conf.broadcastTimeout @transient - private lazy val maxBroadcastRows = 512000000 + private lazy val maxBroadcastRows = 512000000L private def getByteArrayRdd(plan: SparkPlan): RDD[(Long, ChunkedByteBuffer)] = { plan.executeColumnar().mapPartitionsInternal { iter => @@ -195,7 +195,7 @@ case class CometBroadcastExchangeExec( override protected def doPrepare(): Unit = { // Materialize the future. - relationFuture + val _ = relationFuture } override protected def doExecute(): RDD[InternalRow] = { diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/CometCollectLimitExec.scala b/spark/src/main/scala/org/apache/spark/sql/comet/CometCollectLimitExec.scala index 6d43fe156b2..4d560ef8d3f 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/CometCollectLimitExec.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/CometCollectLimitExec.scala @@ -112,7 +112,7 @@ case class CometCollectLimitExec( outputPartitioning, serializer, metrics) - metrics("numPartitions").set(dep.partitioner.numPartitions) + metrics("numPartitions").set(dep.partitioner.numPartitions.toLong) new CometShuffledBatchRDD(dep, readMetrics) } diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/CometColumnarToRowExec.scala b/spark/src/main/scala/org/apache/spark/sql/comet/CometColumnarToRowExec.scala index 2fe870ed069..da1d9ba296f 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/CometColumnarToRowExec.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/CometColumnarToRowExec.scala @@ -80,7 +80,7 @@ case class CometColumnarToRowExec(child: SparkPlan) val toUnsafe = UnsafeProjection.create(localOutput, localOutput) batches.flatMap { batch => numInputBatches += 1 - numOutputRows += batch.numRows() + numOutputRows += batch.numRows().toLong batch.rowIterator().asScala.map(toUnsafe) } } @@ -111,14 +111,14 @@ case class CometColumnarToRowExec(child: SparkPlan) val numOutputRows = longMetric("numOutputRows") val numInputBatches = longMetric("numInputBatches") val localOutput = this.output - val broadcastColumnar = child.executeBroadcast() + val broadcastColumnar = child.executeBroadcast[Any]() val serializedBatches = broadcastColumnar.value.asInstanceOf[Array[ChunkedByteBuffer]] val toUnsafe = UnsafeProjection.create(localOutput, localOutput) val rows = serializedBatches.iterator .flatMap(CometUtils.decodeBatches(_, this.getClass.getSimpleName)) .flatMap { batch => numInputBatches += 1 - numOutputRows += batch.numRows() + numOutputRows += batch.numRows().toLong batch.rowIterator().asScala.map(toUnsafe) } diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/CometIcebergWriteExec.scala b/spark/src/main/scala/org/apache/spark/sql/comet/CometIcebergWriteExec.scala index 00a5047625f..c95c0d41e46 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/CometIcebergWriteExec.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/CometIcebergWriteExec.scala @@ -373,7 +373,8 @@ object CometIcebergWriteExec { /** Take ownership of the locations the native writer reported. */ def own(written: Seq[String]): Unit = locations = written - override def onTaskFailure(context: TaskContext, error: Throwable): Unit = - IcebergReflection.deleteFilesQuietly(io, locations, describeTask) + override def onTaskFailure(context: TaskContext, error: Throwable): Unit = { + val _ = IcebergReflection.deleteFilesQuietly(io, locations, describeTask) + } } } diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/CometInMemoryTableScanExec.scala b/spark/src/main/scala/org/apache/spark/sql/comet/CometInMemoryTableScanExec.scala index 0f6f531fdc3..103f95f2e53 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/CometInMemoryTableScanExec.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/CometInMemoryTableScanExec.scala @@ -123,7 +123,7 @@ case class CometInMemoryTableScanExec( serializer .convertCachedBatchToColumnarBatch(filteredBuffers, relationOutput, scanOutput, conf) .map { cb => - numOutputRows += cb.numRows() + numOutputRows += cb.numRows().toLong cb } } diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/CometMapInBatchExec.scala b/spark/src/main/scala/org/apache/spark/sql/comet/CometMapInBatchExec.scala index 8ac9a70de3e..8e6ff6c2ad3 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/CometMapInBatchExec.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/CometMapInBatchExec.scala @@ -91,7 +91,7 @@ case class CometMapInBatchExec( def processPartition(batches: Iterator[ColumnarBatch]): Iterator[ColumnarBatch] = { val context = TaskContext.get() - val counting = batches.map { b => numInputRows += b.numRows(); b } + val counting = batches.map { b => numInputRows += b.numRows().toLong; b } val columnarBatchIter = computeArrowPython( resolvedRunnerInputs, @@ -113,7 +113,7 @@ case class CometMapInBatchExec( outputAttrs.indices.map(i => structVector.getChild(i)).toArray val flattenedBatch = new ColumnarBatch(outputVectors) flattenedBatch.setNumRows(batch.numRows()) - numOutputRows += flattenedBatch.numRows() + numOutputRows += flattenedBatch.numRows().toLong numOutputBatches += 1 flattenedBatch } diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/CometMetricNode.scala b/spark/src/main/scala/org/apache/spark/sql/comet/CometMetricNode.scala index 336cbf1a6ee..2d4dfd8b5bc 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/CometMetricNode.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/CometMetricNode.scala @@ -110,7 +110,7 @@ case class CometMetricNode(metrics: Map[String, SQLMetric], children: Seq[CometM * be called in a TaskCompletionListener after the iterator is fully consumed. */ def reportScanInputMetrics(ctx: TaskContext): Unit = { - ctx.addTaskCompletionListener[Unit] { _ => + val _ = ctx.addTaskCompletionListener[Unit] { _ => val scanLeaves = leafNodes.filter(_.metrics.contains("bytes_scanned")) if (scanLeaves.nonEmpty) { val totalBytes = scanLeaves.map(_.metrics("bytes_scanned").value).sum @@ -137,7 +137,7 @@ case class CometMetricNode(metrics: Map[String, SQLMetric], children: Seq[CometM * before this listener runs. */ def reportNativeWriteOutputMetrics(ctx: TaskContext): Unit = { - ctx.addTaskCompletionListener[Unit] { _ => + val _ = ctx.addTaskCompletionListener[Unit] { _ => metrics.get("bytes_written").foreach { m => ctx.taskMetrics().outputMetrics.setBytesWritten(m.value) } @@ -162,7 +162,7 @@ case class CometMetricNode(metrics: Map[String, SQLMetric], children: Seq[CometM */ def reportSpillMetrics(ctx: TaskContext): Unit = { val seenMetrics = CometMetricNode.taskSeenSpillMetrics(ctx) - ctx.addTaskCompletionListener[Unit] { _ => + val _ = ctx.addTaskCompletionListener[Unit] { _ => val diskBytesSpilled = sumMetricValues("spilled_bytes", seenMetrics.disk) if (diskBytesSpilled > 0L) { ctx.taskMetrics().incDiskBytesSpilled(diskBytesSpilled) @@ -244,7 +244,9 @@ object CometMetricNode { // The task thread is the only registrant for its attempt id, so there is no put race. val created = SeenSpillMetrics(new IdentityHashMap(), new IdentityHashMap()) seenSpillMetricsByTask.put(attemptId, created) - ctx.addTaskCompletionListener[Unit](_ => seenSpillMetricsByTask.remove(attemptId)) + ctx.addTaskCompletionListener[Unit](_ => { + val _ = seenSpillMetricsByTask.remove(attemptId) + }) created } } diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/CometNativeColumnarToRowExec.scala b/spark/src/main/scala/org/apache/spark/sql/comet/CometNativeColumnarToRowExec.scala index ce800ddf098..0c3eea71f8b 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/CometNativeColumnarToRowExec.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/CometNativeColumnarToRowExec.scala @@ -97,7 +97,7 @@ case class CometNativeColumnarToRowExec(child: SparkPlan) val numInputBatches = longMetric("numInputBatches") val localSchema = this.schema val batchSize = CometConf.COMET_BATCH_SIZE.get() - val broadcastColumnar = child.executeBroadcast() + val broadcastColumnar = child.executeBroadcast[Any]() val serializedBatches = broadcastColumnar.value.asInstanceOf[Array[org.apache.spark.util.io.ChunkedByteBuffer]] @@ -108,7 +108,7 @@ case class CometNativeColumnarToRowExec(child: SparkPlan) .flatMap(CometUtils.decodeBatches(_, this.getClass.getSimpleName)) .flatMap { batch => numInputBatches += 1 - numOutputRows += batch.numRows() + numOutputRows += batch.numRows().toLong val result = converter.convert(batch) // Wrap iterator to close batch after consumption new Iterator[InternalRow] { @@ -202,7 +202,7 @@ case class CometNativeColumnarToRowExec(child: SparkPlan) batches.flatMap { batch => numInputBatches += 1 val numRows = batch.numRows() - numOutputRows += numRows + numOutputRows += numRows.toLong val startTime = System.nanoTime() val result = converter.convert(batch) diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/CometNativeWriteExec.scala b/spark/src/main/scala/org/apache/spark/sql/comet/CometNativeWriteExec.scala index f0d10b17667..ced525c1869 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/CometNativeWriteExec.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/CometNativeWriteExec.scala @@ -188,7 +188,7 @@ case class CometNativeWriteExec( val taskAttemptId = org.apache.spark.TaskContext.get().taskAttemptId() // Setup task-level commit protocol if provided - val (workDir, taskContext, commitMsg) = capturedCommitter + val (workDir, taskContext, _) = capturedCommitter .map { committer => val taskContext = createTaskContext(capturedJobTrackerID, partitionId, taskAttemptId.toInt) diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/CometScanExec.scala b/spark/src/main/scala/org/apache/spark/sql/comet/CometScanExec.scala index f88e3d380e9..3e1beec716d 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/CometScanExec.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/CometScanExec.scala @@ -228,7 +228,7 @@ case class CometScanExec( driverMetrics("staticFilesSize") = filesSize } if (relation.partitionSchema.nonEmpty) { - driverMetrics("numPartitions") = partitions.length + driverMetrics("numPartitions") = partitions.length.toLong } } diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/CometSparkToColumnarExec.scala b/spark/src/main/scala/org/apache/spark/sql/comet/CometSparkToColumnarExec.scala index ed4dc84c9cc..9b74b6ddb69 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/CometSparkToColumnarExec.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/CometSparkToColumnarExec.scala @@ -105,7 +105,7 @@ case class CometSparkToColumnarExec(child: SparkPlan) CometArrowStream .countingIterator( sparkBatches, - (b: ColumnarBatch) => numInputRows.add(b.numRows())), + (b: ColumnarBatch) => numInputRows.add(b.numRows().toLong)), maxBatchInt, onConversionNs)) } diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/CometSubqueryBroadcastExec.scala b/spark/src/main/scala/org/apache/spark/sql/comet/CometSubqueryBroadcastExec.scala index 61254794489..419b99aef70 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/CometSubqueryBroadcastExec.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/CometSubqueryBroadcastExec.scala @@ -128,7 +128,7 @@ case class CometSubqueryBroadcastExec( val beforeBuild = System.nanoTime() longMetric("collectTime") += (beforeBuild - beforeCollect) / 1000000 - longMetric("numOutputRows") += rows.length + longMetric("numOutputRows") += rows.length.toLong val dataSize = rows.map(_.asInstanceOf[UnsafeRow].getSizeInBytes.toLong).sum longMetric("dataSize") += dataSize SQLMetrics.postDriverMetricUpdates(sparkContext, executionId, metrics.values.toSeq) @@ -139,7 +139,7 @@ case class CometSubqueryBroadcastExec( } protected override def doPrepare(): Unit = { - relationFuture + val _ = relationFuture } protected override def doExecute(): RDD[InternalRow] = { diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/CometTakeOrderedAndProjectExec.scala b/spark/src/main/scala/org/apache/spark/sql/comet/CometTakeOrderedAndProjectExec.scala index 09dd944d93e..4cb5047f7d7 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/CometTakeOrderedAndProjectExec.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/CometTakeOrderedAndProjectExec.scala @@ -160,7 +160,7 @@ case class CometTakeOrderedAndProjectExec( outputPartitioning, serializer, metrics) - metrics("numPartitions").set(dep.partitioner.numPartitions) + metrics("numPartitions").set(dep.partitioner.numPartitions.toLong) new CometShuffledBatchRDD(dep, readMetrics) } diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/IcebergCommitExec.scala b/spark/src/main/scala/org/apache/spark/sql/comet/IcebergCommitExec.scala index df3c38b3de3..90a8d0dd370 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/IcebergCommitExec.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/IcebergCommitExec.scala @@ -102,7 +102,7 @@ case class IcebergCommitExec( } throw cause } - longMetric("numCommittedMessages").add(messages.length) + longMetric("numCommittedMessages").add(messages.length.toLong) try { messages.foreach(batchWrite.onDataWriterCommit) @@ -132,7 +132,7 @@ case class IcebergCommitExec( .flatMap(IcebergReflection.getTableIO) io match { case Some(fileIO) => - IcebergReflection.deleteFilesQuietly( + val _ = IcebergReflection.deleteFilesQuietly( fileIO, locations, s"job abort, ${completed.length} completed task(s)") diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometBlockStoreShuffleReader.scala b/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometBlockStoreShuffleReader.scala index 3048456ea78..3592095a37c 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometBlockStoreShuffleReader.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometBlockStoreShuffleReader.scala @@ -121,7 +121,7 @@ class CometBlockStoreShuffleReader[K, C]( // Update the context task metrics for each record read. val metricIter = CompletionIterator[(Any, Any), Iterator[(Any, Any)]]( recordIter.map { record => - readMetrics.incRecordsRead(record._2.numRows()) + readMetrics.incRecordsRead(record._2.numRows().toLong) record }, context.taskMetrics().mergeShuffleReadMetrics()) diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShuffleManager.scala b/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShuffleManager.scala index 71fec591ee8..2fdda64582b 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShuffleManager.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShuffleManager.scala @@ -23,6 +23,7 @@ import java.lang.reflect.InvocationTargetException import java.util.Locale import java.util.concurrent.ConcurrentHashMap +import scala.annotation.nowarn import scala.collection.mutable import scala.concurrent.ExecutionContext import scala.jdk.CollectionConverters._ @@ -124,6 +125,7 @@ class CometCelebornShuffleManager private[shuffle] ( } } + @nowarn("msg=references private") override def getWriter[K, V]( handle: ShuffleHandle, mapId: Long, @@ -212,6 +214,7 @@ class CometCelebornShuffleManager private[shuffle] ( } // Spark's final all-mapper reader overload delegates to this mapper-range overload. + @nowarn("msg=references private") override def getReader[K, C]( handle: ShuffleHandle, startMapIndex: Int, @@ -256,9 +259,9 @@ class CometCelebornShuffleManager private[shuffle] ( endPartition), context, metrics, - client => ownedNativeClients.put(client, java.lang.Boolean.TRUE), + client => { val _ = ownedNativeClients.put(client, java.lang.Boolean.TRUE) }, (client, celebornShuffleId) => { - nativeShuffleClients + val _ = nativeShuffleClients .computeIfAbsent(handle.shuffleId, _ => new ConcurrentHashMap[Int, AnyRef]()) .put(celebornShuffleId, client) }, @@ -392,7 +395,7 @@ class CometCelebornShuffleManager private[shuffle] ( conf, handle, context, - client => ownedNativeClients.put(client, java.lang.Boolean.TRUE), + client => { val _ = ownedNativeClients.put(client, java.lang.Boolean.TRUE) }, onGenerationResolved, onGenerationInvalidated, onInvalidationUnsafe) @@ -411,7 +414,7 @@ class CometCelebornShuffleManager private[shuffle] ( } private[shuffle] def removeSizeLimitFallback(shuffleId: Int): Unit = { - sizeLimitFallbacks.remove(shuffleId) + val _ = sizeLimitFallbacks.remove(shuffleId) } private def isLocalNativeHandle(handle: ShuffleHandle): Boolean = handle match { @@ -1070,7 +1073,9 @@ private[shuffle] final class CelebornShuffleGenerationCoordinator( invalidatedGenerations.remove(shuffleId) generationEpochs.remove(shuffleId) claimOwners.retain { case ((ownerShuffleId, _, _, _), _) => ownerShuffleId != shuffleId } - deniedAttempts.retain { case ((ownerShuffleId, _, _, _), _) => ownerShuffleId != shuffleId } + val _ = deniedAttempts.retain { case ((ownerShuffleId, _, _, _), _) => + ownerShuffleId != shuffleId + } } } diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShuffleReader.scala b/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShuffleReader.scala index 891290a441e..b7deac55b49 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShuffleReader.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShuffleReader.scala @@ -143,7 +143,7 @@ private[shuffle] final class CometCelebornShuffleReader[K, C]( } val rows = decoder.map { batch: ColumnarBatch => - readMetrics.incRecordsRead(batch.numRows()) + readMetrics.incRecordsRead(batch.numRows().toLong) (0, batch) } val completed = CompletionIterator[(Int, ColumnarBatch), Iterator[(Int, ColumnarBatch)]]( @@ -511,7 +511,7 @@ private[shuffle] object CelebornRawPartitionReader { java.util.Objects.checkFromIndexSize(offset, length, buffer.length) if (length == 0) return 0 val readingHeader = bodyRemaining == 0 - val remaining = if (readingHeader) header.length - headerBytes else bodyRemaining + val remaining = if (readingHeader) (header.length - headerBytes).toLong else bodyRemaining val count = in.read(buffer, offset, math.min(length.toLong, remaining).toInt) if (count < 0) { if (headerBytes != 0 || bodyRemaining != 0) { diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativeShuffleWriter.scala b/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativeShuffleWriter.scala index 21dd7686302..7fe3ac0ef7f 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativeShuffleWriter.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativeShuffleWriter.scala @@ -72,7 +72,10 @@ class CometNativeShuffleWriter[K, V]( private val OFFSET_LENGTH = 8 var partitionLengths: Array[Long] = _ - var mapStatus: MapStatus = _ + // `private[shuffle]` rather than public: `MapStatus` is `private[spark]`, and a public + // var exposing it trips `-Xlint:inaccessible`. The shuffle package, which includes the + // tests that read it, is the only thing that ever touches this. + private[shuffle] var mapStatus: MapStatus = _ private var stopped = false private lazy val effectivePartitionCount = remoteDestination.map(_.numPartitions).getOrElse(outputPartitioning.numPartitions) @@ -118,7 +121,7 @@ class CometNativeShuffleWriter[K, V]( val dataFile = resolver.getDataFile(shuffleId, mapId) val indexFile = resolver.getIndexFile(shuffleId, mapId) Some( - LocalShuffleOutput( + CometNativeShuffleWriter.LocalShuffleOutput( resolver, dataFile.getPath.replace(".data", ".data.tmp"), indexFile.getPath.replace(".index", ".index.tmp"))) @@ -479,14 +482,19 @@ class CometNativeShuffleWriter[K, V]( } override def getPartitionLengths(): Array[Long] = partitionLengths +} + +private[shuffle] object CometNativeShuffleWriter { - private final case class LocalShuffleOutput( + /** + * Declared here rather than inside the class: as an inner case class every type test against it + * carries an outer reference that cannot be checked at run time, which `-Xlint` reports. + */ + private[shuffle] final case class LocalShuffleOutput( resolver: IndexShuffleBlockResolver, dataFile: String, indexFile: String) -} -private[shuffle] object CometNativeShuffleWriter { private[shuffle] def isSizeLimitFailure(failure: Throwable): Boolean = { var cause = failure val visited = new java.util.IdentityHashMap[Throwable, java.lang.Boolean]() diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometShuffleExchangeExec.scala b/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometShuffleExchangeExec.scala index 1f5d06a53af..020edaa4c37 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometShuffleExchangeExec.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometShuffleExchangeExec.scala @@ -215,7 +215,7 @@ case class CometShuffleExchangeExec( serializer, metrics) } - metrics("numPartitions").set(dep.partitioner.numPartitions) + metrics("numPartitions").set(dep.partitioner.numPartitions.toLong) val executionId = sparkContext.getLocalProperty(SQLExecution.EXECUTION_ID_KEY) SQLMetrics.postDriverMetricUpdates( sparkContext, @@ -229,7 +229,7 @@ case class CometShuffleExchangeExec( outputPartitioning, serializer, metrics) - metrics("numPartitions").set(dep.partitioner.numPartitions) + metrics("numPartitions").set(dep.partitioner.numPartitions.toLong) val executionId = sparkContext.getLocalProperty(SQLExecution.EXECUTION_ID_KEY) SQLMetrics.postDriverMetricUpdates( sparkContext, @@ -1043,7 +1043,7 @@ object CometShuffleExchangeExec // end up being almost the same regardless of the index. substantially scrambling the // seed by hashing will help. Refer to SPARK-21782 for more details. val partitionId = TaskContext.get().partitionId() - var position = new XORShiftRandom(partitionId).nextInt(numPartitions) + var position = new XORShiftRandom(partitionId.toLong).nextInt(numPartitions) (_: InternalRow) => { // The HashPartitioner will handle the `mod` by the number of partitions position += 1 @@ -1090,7 +1090,7 @@ object CometShuffleExchangeExec row: InternalRow): UnsafeExternalRowSorter.PrefixComputer.Prefix = { // The hashcode generated from the binary form of a [[UnsafeRow]] should not be null. result.isNull = false - result.value = row.hashCode() + result.value = row.hashCode().toLong result } } diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometShuffleManager.scala b/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometShuffleManager.scala index 398ed66b6a5..546a1d40427 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometShuffleManager.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometShuffleManager.scala @@ -21,6 +21,7 @@ package org.apache.spark.sql.comet.execution.shuffle import java.util.concurrent.ConcurrentHashMap +import scala.annotation.nowarn import scala.jdk.CollectionConverters._ import org.apache.spark.ShuffleDependency @@ -164,6 +165,7 @@ class CometShuffleManager(conf: SparkConf) extends ShuffleManager with Logging { } } + @nowarn("msg=references private") override def getReader[K, C]( handle: ShuffleHandle, startMapIndex: Int, @@ -216,6 +218,7 @@ class CometShuffleManager(conf: SparkConf) extends ShuffleManager with Logging { } /** Get a writer for a given partition. Called on executors by map tasks. */ + @nowarn("msg=references private") override def getWriter[K, V]( handle: ShuffleHandle, mapId: Long, diff --git a/spark/src/main/spark-3.4/org/apache/spark/sql/comet/shims/ShimCometUnionExec.scala b/spark/src/main/spark-3.4/org/apache/spark/sql/comet/shims/ShimCometUnionExec.scala index 15c480b057c..57db09d5f38 100644 --- a/spark/src/main/spark-3.4/org/apache/spark/sql/comet/shims/ShimCometUnionExec.scala +++ b/spark/src/main/spark-3.4/org/apache/spark/sql/comet/shims/ShimCometUnionExec.scala @@ -37,7 +37,7 @@ object ShimCometUnionExec { def unionRDDs[T: ClassTag]( sc: SparkContext, rdds: Seq[RDD[T]], - @annotation.nowarn("cat=unused") outputPartitioning: Partitioning): RDD[T] = { + outputPartitioning: Partitioning): RDD[T] = { sc.union(rdds) } } diff --git a/spark/src/main/spark-3.5/org/apache/spark/sql/comet/shims/ShimCometUnionExec.scala b/spark/src/main/spark-3.5/org/apache/spark/sql/comet/shims/ShimCometUnionExec.scala index 15c480b057c..57db09d5f38 100644 --- a/spark/src/main/spark-3.5/org/apache/spark/sql/comet/shims/ShimCometUnionExec.scala +++ b/spark/src/main/spark-3.5/org/apache/spark/sql/comet/shims/ShimCometUnionExec.scala @@ -37,7 +37,7 @@ object ShimCometUnionExec { def unionRDDs[T: ClassTag]( sc: SparkContext, rdds: Seq[RDD[T]], - @annotation.nowarn("cat=unused") outputPartitioning: Partitioning): RDD[T] = { + outputPartitioning: Partitioning): RDD[T] = { sc.union(rdds) } } diff --git a/spark/src/main/spark-3.x/org/apache/comet/shims/CometTypeShim.scala b/spark/src/main/spark-3.x/org/apache/comet/shims/CometTypeShim.scala index 1e5d1686f96..567999fc1c6 100644 --- a/spark/src/main/spark-3.x/org/apache/comet/shims/CometTypeShim.scala +++ b/spark/src/main/spark-3.x/org/apache/comet/shims/CometTypeShim.scala @@ -22,40 +22,37 @@ package org.apache.comet.shims import java.nio.ByteBuffer import java.nio.charset.{CharacterCodingException, CodingErrorAction, StandardCharsets} -import scala.annotation.nowarn - import org.apache.spark.sql.catalyst.expressions.aggregate.Mode import org.apache.spark.sql.types.{DataType, StructType} import org.apache.spark.unsafe.types.UTF8String trait CometTypeShim { - @nowarn // Spark 4 feature; stubbed to false in Spark 3.x for compatibility. + // Spark 4 feature; stubbed to false in Spark 3.x for compatibility. def isStringCollationType(dt: DataType): Boolean = false // `mode() WITHIN GROUP (ORDER BY ...)` and the deterministic-flag form (which set `reverseOpt`) // are Spark 4.0 features; Spark 3.x `Mode` is always the plain `mode(col)` form. - @nowarn def modeHasUnsupportedOrdering(expr: Mode): Boolean = false - @nowarn // Spark 4 feature; stubbed to false in Spark 3.x for compatibility. + // Spark 4 feature; stubbed to false in Spark 3.x for compatibility. def hasNonDefaultStringCollation(dt: DataType): Boolean = false - @nowarn // Spark 4 feature; collation does not exist in Spark 3.x. + // Spark 4 feature; collation does not exist in Spark 3.x. def hasCollationSupport: Boolean = false - @nowarn // Spark 4 feature; Variant shredding doesn't exist in Spark 3.x. + // Spark 4 feature; Variant shredding doesn't exist in Spark 3.x. def isVariantStruct(s: StructType): Boolean = false - @nowarn // Spark 4 feature; VariantType doesn't exist in Spark 3.x. + // Spark 4 feature; VariantType doesn't exist in Spark 3.x. def isVariantType(dt: DataType): Boolean = false - @nowarn // Spark 4 feature; VariantType doesn't exist in Spark 3.x. + // Spark 4 feature; VariantType doesn't exist in Spark 3.x. def containsVariantType(dt: DataType): Boolean = false - @nowarn // Spark 4 feature; VariantType doesn't exist in Spark 3.x. + // Spark 4 feature; VariantType doesn't exist in Spark 3.x. def variantType: Option[DataType] = None - @nowarn // Spark 4.1 feature; TimeType doesn't exist in Spark 3.x. + // Spark 4.1 feature; TimeType doesn't exist in Spark 3.x. def isTimeType(dt: DataType): Boolean = false /** diff --git a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala index 4ab86b4f373..0ee1b4b5a13 100644 --- a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala @@ -1262,7 +1262,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { def makeDecimalRDD(num: Int, decimal: DecimalType, useDictionary: Boolean): DataFrame = { val div = if (useDictionary) 5 else num // narrow the space to make it dictionary encoded spark - .range(num) + .range(num.toLong) .map(_ % div) // Parquet doesn't allow column names with spaces, have to add an alias here. // Minus 500 here so that negative decimals are also tested. @@ -1766,7 +1766,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { // must stay native; interval types are unsupported and fall back to Spark. withParquetTable( (1 to 5).map(i => - (i.toByte, i.toShort, i, i.toLong, i.toFloat, i.toDouble, BigDecimal(i * 3, 2))), + (i.toByte, i.toShort, i, i.toLong, i.toFloat, i.toDouble, BigDecimal((i * 3).toLong, 2))), "umt") { checkSparkAnswerAndOperator("SELECT -_1, -_2, -_3, -_4, -_5, -_6, -_7 FROM umt") } diff --git a/spark/src/test/scala/org/apache/comet/CometIcebergNativeSuite.scala b/spark/src/test/scala/org/apache/comet/CometIcebergNativeSuite.scala index d92723ecf83..1dfd62ee2e2 100644 --- a/spark/src/test/scala/org/apache/comet/CometIcebergNativeSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometIcebergNativeSuite.scala @@ -5902,7 +5902,8 @@ class CometIcebergNativeSuite conflictingField.put("name", "id_as_region") conflictingField.put("transform", "identity") conflictingFields.add(conflictingField) - conflictingSpec.set("fields", conflictingFields) + conflictingSpec + .set[com.fasterxml.jackson.databind.node.ObjectNode]("fields", conflictingFields) specs.add(conflictingSpec) // Round-trip through Iceberg's own parser before writing, so a malformed hand-edit diff --git a/spark/src/test/scala/org/apache/comet/CometIcebergSystemFunctionSuite.scala b/spark/src/test/scala/org/apache/comet/CometIcebergSystemFunctionSuite.scala index e7dd961150a..118b2371d53 100644 --- a/spark/src/test/scala/org/apache/comet/CometIcebergSystemFunctionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometIcebergSystemFunctionSuite.scala @@ -483,7 +483,7 @@ class CometIcebergSystemFunctionSuite maybeNull(randomDecimal38()), maybeNull(randomString()), maybeNull(randomBinary()), - maybeNull(LocalDate.ofEpochDay(random.nextInt(40000) - 20000)), + maybeNull(LocalDate.ofEpochDay((random.nextInt(40000) - 20000).toLong)), maybeNull(instant(randomMicros())), maybeNull(localDateTime(randomMicros()))) } diff --git a/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala b/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala index 3c9dbacd156..9909aeb6120 100644 --- a/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala @@ -2706,28 +2706,7 @@ class CometNativeCastSuite extends CometTestBase with AdaptiveSparkPlanHelper { values.map(v => Some(v)) ++ Seq(None) } - private def castFallbackTest( - input: DataFrame, - toType: DataType, - expectedMessage: String): Unit = { - withTempPath { dir => - val data = roundtripParquet(input, dir).coalesce(1) - data.createOrReplaceTempView("t") - - withSQLConf((SQLConf.ANSI_ENABLED.key, "false")) { - val df = data.withColumn("converted", col("a").cast(toType)) - df.collect() - val str = - new ExtendedExplainInfo().generateExtendedInfo(df.queryExecution.executedPlan) - assert(str.contains(expectedMessage)) - } - } - } - - private def castTimestampTest( - input: DataFrame, - toType: DataType, - assertNative: Boolean = false) = { + private def castTimestampTest(input: DataFrame, toType: DataType, assertNative: Boolean) = { withTempPath { dir => val data = roundtripParquet(input, dir).coalesce(1) data.createOrReplaceTempView("t") diff --git a/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala index 5b5d43dbe8a..5b32b137501 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala @@ -3385,19 +3385,11 @@ class CometExecSuite extends CometTestBase { val df2 = (0 until 50).map(i => (i % 7, i % 11, i.toString)).toDF("i", "j", "k").as("df2") - val BucketedTableTestSpec( - bucketSpecLeft, - numPartitionsLeft, - shuffleLeft, - sortLeft, - numOutputPartitionsLeft) = bucketedTableTestSpecLeft - - val BucketedTableTestSpec( - bucketSpecRight, - numPartitionsRight, - shuffleRight, - sortRight, - numOutputPartitionsRight) = bucketedTableTestSpecRight + val BucketedTableTestSpec(bucketSpecLeft, numPartitionsLeft, _, _, _) = + bucketedTableTestSpecLeft + + val BucketedTableTestSpec(bucketSpecRight, numPartitionsRight, _, _, _) = + bucketedTableTestSpecRight withTable("bucketed_table1", "bucketed_table2") { withBucket(df1.repartition(numPartitionsLeft).write.format("parquet"), bucketSpecLeft) @@ -3585,7 +3577,7 @@ class CometExecSuite extends CometTestBase { withTable("t1") { val numRows = 10 spark - .range(numRows) + .range(numRows.toLong) .selectExpr("if (id % 2 = 0, null, id) AS a", s"$numRows - id AS b") .repartition(3) // Move data across multiple partitions .write @@ -3622,7 +3614,7 @@ class CometExecSuite extends CometTestBase { withTable("t1") { val numRows = 10 spark - .range(numRows) + .range(numRows.toLong) .selectExpr("if (id % 2 = 0, null, id) AS a", s"$numRows - id AS b") .repartition(3) // Force repartition to test data will come to single partition .write @@ -3653,7 +3645,7 @@ class CometExecSuite extends CometTestBase { withTable("t1") { val numRows = 10 spark - .range(numRows) + .range(numRows.toLong) .selectExpr("if (id % 2 = 0, null, id) AS a", s"$numRows - id AS b") .repartition(3) // Force repartition to test data will come to single partition .write diff --git a/spark/src/test/scala/org/apache/comet/exec/CometJoinSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometJoinSuite.scala index 5af9a37c267..e658b9623ea 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometJoinSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometJoinSuite.scala @@ -473,7 +473,7 @@ class CometJoinSuite extends CometTestBase { SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false", SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1") { withParquetTable( - (0 until 100).map(i => (Some(i), i.toLong)) :+ (None, -1L), + (0 until 100).map(i => (Some(i), i.toLong)) :+ ((None, -1L)), "dynamic_probe") { for (build <- Seq( Seq((Some(10), 1L), (None, 2L), (Some(10), 3L), (Some(90), 4L)), diff --git a/spark/src/test/scala/org/apache/comet/exec/CometNativeColumnarToRowSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometNativeColumnarToRowSuite.scala index 800264bb141..818be0b1ecd 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometNativeColumnarToRowSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometNativeColumnarToRowSuite.scala @@ -357,7 +357,7 @@ class CometNativeColumnarToRowSuite extends CometTestBase with AdaptiveSparkPlan if (isNull) null else s"string_$i", if (isNull) null else new Date(baseDate.getTime + i * 24 * 60 * 60 * 1000L), if (isNull) null else new Timestamp(baseTs.getTime + i * 1000L), - if (isNull) null else BigDecimal(i * 100 + i, 2).bigDecimal) + if (isNull) null else BigDecimal((i * 100 + i).toLong, 2).bigDecimal) } val schema = StructType( diff --git a/spark/src/test/scala/org/apache/comet/exec/CometNativeShuffleSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometNativeShuffleSuite.scala index ff6ef3b10f0..763dfdfcea4 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometNativeShuffleSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometNativeShuffleSuite.scala @@ -434,7 +434,7 @@ class CometNativeShuffleSuite extends CometTestBase with AdaptiveSparkPlanHelper withTempDir { dir => val path = new Path(dir.toURI.toString, "test.parquet") makeParquetFileAllPrimitiveTypes(path, dictionaryEnabled = dictionaryEnabled, 1000) - var allTypes: Seq[Int] = (1 to 20) + val allTypes: Seq[Int] = (1 to 20) allTypes.map(i => s"_$i").foreach { c => withSQLConf("parquet.enable.dictionary" -> dictionaryEnabled.toString) { readParquetFile(path.toString) { df => diff --git a/spark/src/test/scala/org/apache/comet/objectstore/NativeConfigSuite.scala b/spark/src/test/scala/org/apache/comet/objectstore/NativeConfigSuite.scala index 8e5a60d63b0..1cfd1d36b92 100644 --- a/spark/src/test/scala/org/apache/comet/objectstore/NativeConfigSuite.scala +++ b/spark/src/test/scala/org/apache/comet/objectstore/NativeConfigSuite.scala @@ -30,6 +30,14 @@ import org.apache.comet.CometConf.COMET_S3_COMPLIANT_SCHEMES_KEY class NativeConfigSuite extends AnyFunSuite with Matchers { + /** + * A Hadoop `Configuration` variable reference to `key`, as `Configuration#get` expands it. + * Built by concatenation: written as a literal, it reads to scalac as a string missing its `s` + * interpolator, and adding the `s` (escaping the dollar) is then flagged by scalafix as + * redundant. + */ + private def varRef(key: String): String = "${" + key + "}" + test("extractObjectStoreOptions - multiple cloud provider configurations") { val hadoopConf = new Configuration() // S3A configs @@ -108,13 +116,15 @@ class NativeConfigSuite extends AnyFunSuite with Matchers { } } - test("extractObjectStoreOptions - forwards the substituted value of a ${...} reference") { + test( + "extractObjectStoreOptions - forwards the substituted value of a " + + s"${varRef("...")} reference") { // Hadoop's own consumers read values through Configuration#get, which expands a ${...} // reference against another conf entry. Forwarding the raw, unexpanded literal here would // give native a different credential than every Hadoop-side consumer sees. val hadoopConf = new Configuration() hadoopConf.set("my.custom.access.key", "expanded-access-key") - hadoopConf.set("fs.s3a.access.key", "${my.custom.access.key}") + hadoopConf.set("fs.s3a.access.key", varRef("my.custom.access.key")) val options = NativeConfig.extractObjectStoreOptions(hadoopConf, new URI("s3a://test-bucket/test-object")) @@ -122,19 +132,19 @@ class NativeConfigSuite extends AnyFunSuite with Matchers { } test( - "extractObjectStoreOptions - a cyclic ${...} reference falls back to the raw value " + - "instead of throwing") { + s"extractObjectStoreOptions - a cyclic ${varRef("...")} reference falls back to the raw " + + "value instead of throwing") { // Configuration#get raises IllegalStateException once ${...} expansion recurses past // Hadoop's MAX_SUBST bound; a two-key mutual cycle triggers this on every call. Extraction // must still return a full options map rather than aborting for the whole object store. val hadoopConf = new Configuration() - hadoopConf.set("fs.s3a.access.key", "${fs.s3a.secret.key}") - hadoopConf.set("fs.s3a.secret.key", "${fs.s3a.access.key}") + hadoopConf.set("fs.s3a.access.key", varRef("fs.s3a.secret.key")) + hadoopConf.set("fs.s3a.secret.key", varRef("fs.s3a.access.key")) val options = NativeConfig.extractObjectStoreOptions(hadoopConf, new URI("s3a://test-bucket/test-object")) - assert(options("fs.s3a.access.key") == "${fs.s3a.secret.key}") - assert(options("fs.s3a.secret.key") == "${fs.s3a.access.key}") + assert(options("fs.s3a.access.key") == varRef("fs.s3a.secret.key")) + assert(options("fs.s3a.secret.key") == varRef("fs.s3a.access.key")) } test( diff --git a/spark/src/test/scala/org/apache/comet/parquet/CometParquetWriterSuite.scala b/spark/src/test/scala/org/apache/comet/parquet/CometParquetWriterSuite.scala index eef77d88246..29ccfcb0162 100644 --- a/spark/src/test/scala/org/apache/comet/parquet/CometParquetWriterSuite.scala +++ b/spark/src/test/scala/org/apache/comet/parquet/CometParquetWriterSuite.scala @@ -949,7 +949,7 @@ class CometParquetWriterSuite extends CometTestBase { var iterations = 0 while (capturedPlan.isEmpty && iterations < maxIterations) { - Thread.sleep(checkIntervalMs) + Thread.sleep(checkIntervalMs.toLong) iterations += 1 } diff --git a/spark/src/test/scala/org/apache/comet/parquet/ParquetReadSuite.scala b/spark/src/test/scala/org/apache/comet/parquet/ParquetReadSuite.scala index d412918b572..552fd98c218 100644 --- a/spark/src/test/scala/org/apache/comet/parquet/ParquetReadSuite.scala +++ b/spark/src/test/scala/org/apache/comet/parquet/ParquetReadSuite.scala @@ -736,8 +736,8 @@ abstract class ParquetReadSuite extends CometTestBase { opt match { case Some(i) => record.add(0, i % 2 == 0) - record.add(1, i.toByte) - record.add(2, i.toShort) + record.add(1, i.toByte.toInt) + record.add(2, i.toShort.toInt) record.add(3, i) record.add(4, i.toLong) record.add(5, i.toFloat) @@ -1153,7 +1153,7 @@ abstract class ParquetReadSuite extends CometTestBase { var b = record.addGroup("b") b.add("b1", 1) b.add("b2", 1) - var c = record.addGroup("c") + val c = record.addGroup("c") c.add("c1", 1) c.add("c2", 1) writer.write(record) @@ -1223,7 +1223,7 @@ abstract class ParquetReadSuite extends CometTestBase { var b = record.addGroup("b") b.add("b1", 1) b.add("b2", 1) - var c = record.addGroup("c") + val c = record.addGroup("c") c.add("c1", 1) c.add("c2", 1) writer.write(record) @@ -1900,7 +1900,7 @@ abstract class ParquetReadSuite extends CometTestBase { } private def withId(id: Int) = - new MetadataBuilder().putLong(ParquetUtils.FIELD_ID_METADATA_KEY, id).build() + new MetadataBuilder().putLong(ParquetUtils.FIELD_ID_METADATA_KEY, id.toLong).build() // Based on Spark ParquetIOSuite.test("vectorized reader: array of nested struct") test("array of nested struct with and without field id") { diff --git a/spark/src/test/scala/org/apache/comet/rules/CometScanContribSuite.scala b/spark/src/test/scala/org/apache/comet/rules/CometScanContribSuite.scala index c0f3c0a6ce4..c848501a71a 100644 --- a/spark/src/test/scala/org/apache/comet/rules/CometScanContribSuite.scala +++ b/spark/src/test/scala/org/apache/comet/rules/CometScanContribSuite.scala @@ -25,6 +25,7 @@ import java.nio.charset.StandardCharsets import java.nio.file.Files import java.util.ServiceLoader +import scala.annotation.nowarn import scala.jdk.CollectionConverters._ import org.scalatest.funsuite.AnyFunSuite @@ -331,6 +332,7 @@ class RecordingClaimingScanContrib extends ClaimingScanContrib { class NotAContribAtAll /** Implements the service but cannot be constructed; `next` throws. */ +@nowarn("msg=dead code") class ThrowingCtorScanContrib extends CometScanContrib { throw new IllegalStateException("contrib constructor blew up") } diff --git a/spark/src/test/scala/org/apache/spark/CometExecIteratorLifecycleSuite.scala b/spark/src/test/scala/org/apache/spark/CometExecIteratorLifecycleSuite.scala index a5b98dd5988..cfe80c032ad 100644 --- a/spark/src/test/scala/org/apache/spark/CometExecIteratorLifecycleSuite.scala +++ b/spark/src/test/scala/org/apache/spark/CometExecIteratorLifecycleSuite.scala @@ -93,10 +93,10 @@ class CometExecIteratorLifecycleSuite extends CometTestBase { // Unique synthetic task attempt ids keep each iteration's pool entry independent. val taskAttemptId = 4200000L + i withTaskContext(taskAttemptId) { - val manager = new CometTaskMemoryManager(i, taskAttemptId) + val manager = new CometTaskMemoryManager(i.toLong, taskAttemptId) val thrown = intercept[Throwable] { nativeLib.createPlan( - i, + i.toLong, Array.empty[Object], emptyPlan, badConfigs, diff --git a/spark/src/test/scala/org/apache/spark/shuffle/sort/SpillSorterSuite.scala b/spark/src/test/scala/org/apache/spark/shuffle/sort/SpillSorterSuite.scala index dfbe38b6484..92f7d03a5af 100644 --- a/spark/src/test/scala/org/apache/spark/shuffle/sort/SpillSorterSuite.scala +++ b/spark/src/test/scala/org/apache/spark/shuffle/sort/SpillSorterSuite.scala @@ -75,7 +75,8 @@ class SpillSorterSuite extends AnyFunSuite with BeforeAndAfterEach { spills: java.util.LinkedList[org.apache.spark.sql.comet.execution.shuffle.SpillInfo] = new java.util.LinkedList[org.apache.spark.sql.comet.execution.shuffle.SpillInfo](), partitionChecksums: Array[Long] = new Array[Long](10)): SpillSorter = { - val allocator = CometShuffleMemoryAllocator.getInstance(conf, taskMemoryManager, PAGE_SIZE) + val allocator = + CometShuffleMemoryAllocator.getInstance(conf, taskMemoryManager, PAGE_SIZE.toLong) val schema = createTestSchema() val writeMetrics = new ShuffleWriteMetrics() val taskContext = TaskContext.empty() @@ -115,7 +116,11 @@ class SpillSorterSuite extends AnyFunSuite with BeforeAndAfterEach { val partitionId = 0 sorter.initialCurrentPage(recordData.length + UAO_SIZE) - sorter.insertRecord(recordData, Platform.BYTE_ARRAY_OFFSET, recordData.length, partitionId) + sorter.insertRecord( + recordData, + Platform.BYTE_ARRAY_OFFSET.toLong, + recordData.length, + partitionId) assert(sorter.numRecords() === 1) } finally { @@ -136,7 +141,7 @@ class SpillSorterSuite extends AnyFunSuite with BeforeAndAfterEach { val partitionId = i % 10 sorter.insertRecord( recordData, - Platform.BYTE_ARRAY_OFFSET, + Platform.BYTE_ARRAY_OFFSET.toLong, recordData.length, partitionId) } @@ -153,7 +158,7 @@ class SpillSorterSuite extends AnyFunSuite with BeforeAndAfterEach { try { val recordData = Array[Byte](1, 2, 3, 4) sorter.initialCurrentPage(recordData.length + UAO_SIZE) - sorter.insertRecord(recordData, Platform.BYTE_ARRAY_OFFSET, recordData.length, 0) + sorter.insertRecord(recordData, Platform.BYTE_ARRAY_OFFSET.toLong, recordData.length, 0) assert(sorter.numRecords() === 1) @@ -190,7 +195,7 @@ class SpillSorterSuite extends AnyFunSuite with BeforeAndAfterEach { try { sorter.initialCurrentPage(1024) val recordData = Array[Byte](1, 2, 3, 4) - sorter.insertRecord(recordData, Platform.BYTE_ARRAY_OFFSET, recordData.length, 0) + sorter.insertRecord(recordData, Platform.BYTE_ARRAY_OFFSET.toLong, recordData.length, 0) assert(spillCount.get() === 0, "Spill callback should not be triggered during normal ops") } finally { @@ -225,8 +230,9 @@ class SpillSorterSuite extends AnyFunSuite with BeforeAndAfterEach { val sorter = createSpillSorter() try { val initialMemory = sorter.getMemoryUsage() - val allocator = CometShuffleMemoryAllocator.getInstance(conf, taskMemoryManager, PAGE_SIZE) - val newArray = allocator.allocateArray(INITIAL_SIZE * 2) + val allocator = + CometShuffleMemoryAllocator.getInstance(conf, taskMemoryManager, PAGE_SIZE.toLong) + val newArray = allocator.allocateArray((INITIAL_SIZE * 2).toLong) sorter.expandPointerArray(newArray) assert(sorter.getMemoryUsage() >= initialMemory) @@ -248,7 +254,7 @@ class SpillSorterSuite extends AnyFunSuite with BeforeAndAfterEach { for (p <- 0 until numPartitions) { for (_ <- 0 until recordsPerPartition) { - sorter.insertRecord(recordData, Platform.BYTE_ARRAY_OFFSET, recordData.length, p) + sorter.insertRecord(recordData, Platform.BYTE_ARRAY_OFFSET.toLong, recordData.length, p) } } diff --git a/spark/src/test/scala/org/apache/spark/sql/CometTPCQueryBase.scala b/spark/src/test/scala/org/apache/spark/sql/CometTPCQueryBase.scala index a3437559757..c07006c23d2 100644 --- a/spark/src/test/scala/org/apache/spark/sql/CometTPCQueryBase.scala +++ b/spark/src/test/scala/org/apache/spark/sql/CometTPCQueryBase.scala @@ -99,7 +99,7 @@ trait CometTPCQueryBase extends Logging { // Recover partitions but don't fail if a table is not partitioned. Try { cometSpark.sql(s"ALTER TABLE $tableName RECOVER PARTITIONS") - }.getOrElse { + }.failed.foreach { _ => logInfo(s"Recovering partitions of table $tableName failed") } } diff --git a/spark/src/test/scala/org/apache/spark/sql/CometTestBase.scala b/spark/src/test/scala/org/apache/spark/sql/CometTestBase.scala index a2bbe415cf5..0156ddfbdbc 100644 --- a/spark/src/test/scala/org/apache/spark/sql/CometTestBase.scala +++ b/spark/src/test/scala/org/apache/spark/sql/CometTestBase.scala @@ -792,9 +792,7 @@ abstract class CometTestBase .builder(path) .withDictionaryEncoding(dictionaryEnabled) .withType(schema) - // TODO we need to shim this and use withRowGroupSize(Long) with later parquet-hadoop versions to remove - // the deprecated warning here - .withRowGroupSize(rowGroupSize.toInt) + .withRowGroupSize(rowGroupSize) .withPageSize(pageSize) .withDictionaryPageSize(dictionaryPageSize) .withPageRowCountLimit(pageRowCountLimit) @@ -910,15 +908,15 @@ abstract class CometTestBase opt match { case Some(i) => record.add(0, i % 2 == 0) - record.add(1, i.toByte) - record.add(2, i.toShort) + record.add(1, i.toByte.toInt) + record.add(2, i.toShort.toInt) record.add(3, i) record.add(4, i.toLong) record.add(5, i.toFloat) record.add(6, i.toDouble) record.add(7, i.toString * 48) - record.add(8, (-i).toByte) - record.add(9, (-i).toShort) + record.add(8, (-i).toByte.toInt) + record.add(9, (-i).toShort.toInt) record.add(10, -i) record.add(11, (-i).toLong) record.add(12, i.toString) @@ -939,15 +937,15 @@ abstract class CometTestBase val i = rand.nextLong() val record = new SimpleGroup(schema) record.add(0, i % 2 == 0) - record.add(1, i.toByte) - record.add(2, i.toShort) + record.add(1, i.toByte.toInt) + record.add(2, i.toShort.toInt) record.add(3, i.toInt) record.add(4, i) record.add(5, java.lang.Float.intBitsToFloat(i.toInt)) record.add(6, java.lang.Double.longBitsToDouble(i)) record.add(7, i.toString * 24) - record.add(8, (-i).toByte) - record.add(9, (-i).toShort) + record.add(8, (-i).toByte.toInt) + record.add(9, (-i).toShort.toInt) record.add(10, (-i).toInt) record.add(11, -i) record.add(12, i.toString) @@ -996,7 +994,7 @@ abstract class CometTestBase if (rand.nextBoolean()) { None } else { - Some(getValue(i, div)) + Some(getValue(i.toLong, div.toLong)) } } expected.foreach { opt => @@ -1050,7 +1048,7 @@ abstract class CometTestBase if (rand.nextBoolean()) { None } else { - Some(getValue(i, div)) + Some(getValue(i.toLong, div.toLong)) } } expected.foreach { opt => @@ -1228,7 +1226,7 @@ abstract class CometTestBase val div = if (dictionaryEnabled) 10 else n // maps value to a small range for dict to kick in val expected = (0 until n).map { i => - Some(getValue(i, div)) + Some(getValue(i.toLong, div.toLong)) } expected.foreach { opt => val timestampFormats = List( @@ -1276,7 +1274,7 @@ abstract class CometTestBase def makeDecimalRDD(num: Int, decimal: DecimalType, useDictionary: Boolean): DataFrame = { val div = if (useDictionary) 5 else num // narrow the space to make it dictionary encoded spark - .range(num) + .range(num.toLong) .map(_ % div) // Parquet doesn't allow column names with spaces, have to add an alias here. // Minus 500 here so that negative decimals are also tested. @@ -1456,8 +1454,8 @@ abstract class CometTestBase val record = new SimpleGroup(schema) opt match { case Some(i) => - record.add(0, i.toByte) - record.add(1, i.toShort) + record.add(0, i.toByte.toInt) + record.add(1, i.toShort.toInt) record.add(2, i) record.add(3, i.toLong) record.add(4, rand.nextFloat()) diff --git a/spark/src/test/scala/org/apache/spark/sql/GenTPCHData.scala b/spark/src/test/scala/org/apache/spark/sql/GenTPCHData.scala index e25d4e51e46..6e8ce177d62 100644 --- a/spark/src/test/scala/org/apache/spark/sql/GenTPCHData.scala +++ b/spark/src/test/scala/org/apache/spark/sql/GenTPCHData.scala @@ -65,7 +65,7 @@ object GenTPCHData { // Install the data generators in all nodes // TODO: think a better way to install on each worker node // such as https://stackoverflow.com/a/40876671 - spark.range(0, workers, 1, workers).foreach(worker => installDBGEN(baseDir)(worker)) + spark.range(0, workers.toLong, 1, workers).foreach(worker => installDBGEN(baseDir)(worker)) s"${baseDir}/dbgen" } else { config.dbgenDir @@ -91,7 +91,7 @@ object GenTPCHData { // Clean up if (defaultDbgenDir != null) { - spark.range(0, workers, 1, workers).foreach { _ => + spark.range(0, workers.toLong, 1, workers).foreach { _ => val _ = FileUtils.deleteQuietly(defaultDbgenDir) } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometAggregateExpressionBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometAggregateExpressionBenchmark.scala index 2772d37ffbf..f577ed102d5 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometAggregateExpressionBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometAggregateExpressionBenchmark.scala @@ -217,7 +217,7 @@ object CometAggregateExpressionBenchmark extends CometBenchmarkBase { approxCountDistinctAggregates ++ maxMinByAggregates allAggregates.foreach { config => - runExpressionBenchmark(config.name, v, config.query, config.extraCometConfigs) + runExpressionBenchmark(config.name, v.toLong, config.query, config.extraCometConfigs) } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometArithmeticBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometArithmeticBenchmark.scala index a513aa1a77b..7594ab62fb5 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometArithmeticBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometArithmeticBenchmark.scala @@ -44,7 +44,7 @@ object CometArithmeticBenchmark extends CometBenchmarkBase { val name = s"Binary op ${dataType.sql}, dictionary = $useDictionary" val query = s"SELECT c1 ${op.sig} c2 FROM $table" - runExpressionBenchmark(name, values, query) + runExpressionBenchmark(name, values.toLong, query) } } } @@ -64,7 +64,7 @@ object CometArithmeticBenchmark extends CometBenchmarkBase { val name = s"Binary op ${dataType.sql}, dictionary = $useDictionary" val query = s"SELECT c1 ${op.sig} c2 FROM $table" - runExpressionBenchmark(name, values, query) + runExpressionBenchmark(name, values.toLong, query) } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometArrayExpressionBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometArrayExpressionBenchmark.scala index 624abdebe88..90c247a9996 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometArrayExpressionBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometArrayExpressionBenchmark.scala @@ -65,7 +65,7 @@ object CometArrayExpressionBenchmark extends CometBenchmarkBase { prepareSortArrayTable(width) { runExpressionBenchmark( s"sort_array int ascending (width=$width)", - values, + values.toLong, "SELECT sort_array(int_arr) FROM parquetV1Table") } } @@ -74,7 +74,7 @@ object CometArrayExpressionBenchmark extends CometBenchmarkBase { prepareSortArrayTable(width) { runExpressionBenchmark( s"sort_array int descending (width=$width)", - values, + values.toLong, "SELECT sort_array(int_arr, false) FROM parquetV1Table") } } @@ -83,7 +83,7 @@ object CometArrayExpressionBenchmark extends CometBenchmarkBase { prepareSortArrayTable(width) { runExpressionBenchmark( s"element_at(sort_array(int_arr), 1) (width=$width)", - values, + values.toLong, "SELECT element_at(sort_array(int_arr), 1) FROM parquetV1Table") } } @@ -112,7 +112,7 @@ object CometArrayExpressionBenchmark extends CometBenchmarkBase { runExpressionBenchmark( "array_position - int array", - values, + values.toLong, "SELECT array_position(int_arr, search_val) FROM parquetV1Table") } } @@ -140,7 +140,7 @@ object CometArrayExpressionBenchmark extends CometBenchmarkBase { runExpressionBenchmark( "array_position - string array", - values, + values.toLong, "SELECT array_position(str_arr, search_val) FROM parquetV1Table") } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometArrowWriterBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometArrowWriterBenchmark.scala index 11033703afc..c8073e04c66 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometArrowWriterBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometArrowWriterBenchmark.scala @@ -97,7 +97,10 @@ object CometArrowWriterBenchmark extends BenchmarkBase { val root = VectorSchemaRoot.create(arrowSchema, allocator) try { val benchmark = - new Benchmark(s"Spark columnar to Arrow ($numRows rows)", numRows, output = output) + new Benchmark( + s"Spark columnar to Arrow ($numRows rows)", + numRows.toLong, + output = output) benchmark.addCase("on-heap optimized path") { _ => writeBatch(onHeap, bulkCopy = true, root) } @@ -133,7 +136,7 @@ object CometArrowWriterBenchmark extends BenchmarkBase { arrowSchema, Iterator.from(0).map(i => if ((i & 1) == 0) nullRow else row), numRows) - val rowBenchmark = new Benchmark("Spark rows to Arrow", numRows, output = output) + val rowBenchmark = new Benchmark("Spark rows to Arrow", numRows.toLong, output = output) try { rowBenchmark.addCase("fixed-width, no nulls") { _ => noNullRowReader.loadNextBatch() } rowBenchmark.addCase("fixed-width, 50% nulls") { _ => nullableRowReader.loadNextBatch() } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometBenchmarkBase.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometBenchmarkBase.scala index 487e4797d12..2a957f2d0a2 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometBenchmarkBase.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometBenchmarkBase.scala @@ -103,7 +103,7 @@ trait CometBenchmarkBase // generator, so that results are comparable across runs. Seeding a driver-side `Random` // would not work here: the closure runs per row on the executor. spark - .range(values) + .range(values.toLong) .map(i => if (useDictionary) CometBenchmarkBase.mix64(i) % 5 else CometBenchmarkBase.mix64(i)) .createOrReplaceTempView(tbl) @@ -374,7 +374,7 @@ trait CometBenchmarkBase val div = if (useDictionary) 5 else values spark - .range(values) + .range(values.toLong) .map(_ % div) .select((($"value" - 500) / 100.0) cast decimal as Symbol("dec")) } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometBroadcastHashJoinBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometBroadcastHashJoinBenchmark.scala index 152d79dbb98..81f5f4d7a39 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometBroadcastHashJoinBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometBroadcastHashJoinBenchmark.scala @@ -65,12 +65,12 @@ object CometBroadcastHashJoinBenchmark extends CometBenchmarkBase { withTempPath { dir => withTempTable("probe", "build") { spark - .range(probeRows) + .range(probeRows.toLong) .selectExpr("id AS k", "id % 100 AS v") .write .parquet(s"${dir.getAbsolutePath}/probe") spark - .range(buildRows) + .range(buildRows.toLong) .selectExpr("id AS k", "id * 10 AS w") .write .parquet(s"${dir.getAbsolutePath}/build") @@ -81,7 +81,7 @@ object CometBroadcastHashJoinBenchmark extends CometBenchmarkBase { runBenchmark("BroadcastHashJoin - inner count") { runExpressionBenchmark( "inner count", - probeRows, + probeRows.toLong, "SELECT /*+ BROADCAST(b) */ count(*) FROM probe p JOIN build b ON p.k = b.k", cometConfigs) } @@ -89,7 +89,7 @@ object CometBroadcastHashJoinBenchmark extends CometBenchmarkBase { runBenchmark("BroadcastHashJoin - inner projected") { runExpressionBenchmark( "inner projected", - probeRows, + probeRows.toLong, "SELECT /*+ BROADCAST(b) */ p.k, p.v, b.w FROM probe p JOIN build b ON p.k = b.k", cometConfigs) } @@ -97,7 +97,7 @@ object CometBroadcastHashJoinBenchmark extends CometBenchmarkBase { runBenchmark("BroadcastHashJoin - left outer") { runExpressionBenchmark( "left outer", - probeRows, + probeRows.toLong, "SELECT /*+ BROADCAST(b) */ count(*) FROM probe p LEFT JOIN build b ON p.k = b.k", cometConfigs) } @@ -105,7 +105,7 @@ object CometBroadcastHashJoinBenchmark extends CometBenchmarkBase { runBenchmark("BroadcastHashJoin - left semi") { runExpressionBenchmark( "left semi", - probeRows, + probeRows.toLong, "SELECT /*+ BROADCAST(b) */ count(*) FROM probe p LEFT SEMI JOIN build b ON p.k = b.k", cometConfigs) } @@ -113,7 +113,7 @@ object CometBroadcastHashJoinBenchmark extends CometBenchmarkBase { runBenchmark("BroadcastHashJoin - right outer") { runExpressionBenchmark( "right outer", - probeRows, + probeRows.toLong, "SELECT /*+ BROADCAST(p) */ count(*) FROM probe p RIGHT JOIN build b ON p.k = b.k", cometConfigs) } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometBroadcastNestedLoopJoinBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometBroadcastNestedLoopJoinBenchmark.scala index f8fe6100d4d..dcfc7d44a21 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometBroadcastNestedLoopJoinBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometBroadcastNestedLoopJoinBenchmark.scala @@ -70,12 +70,12 @@ object CometBroadcastNestedLoopJoinBenchmark extends CometBenchmarkBase { withTempPath { dir => withTempTable("probe", "build") { spark - .range(probeRows) + .range(probeRows.toLong) .selectExpr("id AS k", "id % 100 AS v") .write .parquet(s"${dir.getAbsolutePath}/probe") spark - .range(buildRows) + .range(buildRows.toLong) .selectExpr("id * 1000 AS lo", "id * 1000 + 500 AS hi") .write .parquet(s"${dir.getAbsolutePath}/build") @@ -86,7 +86,7 @@ object CometBroadcastNestedLoopJoinBenchmark extends CometBenchmarkBase { runBenchmark("BroadcastNestedLoopJoin - range") { runExpressionBenchmark( "range join (BETWEEN)", - probeRows, + probeRows.toLong, "SELECT /*+ BROADCAST(b) */ count(*) FROM probe p " + "JOIN build b ON p.k BETWEEN b.lo AND b.hi", cometConfigs) @@ -95,7 +95,7 @@ object CometBroadcastNestedLoopJoinBenchmark extends CometBenchmarkBase { runBenchmark("BroadcastNestedLoopJoin - inequality") { runExpressionBenchmark( "inequality join (>)", - probeRows, + probeRows.toLong, "SELECT /*+ BROADCAST(b) */ count(*) FROM probe p " + "JOIN build b ON p.k > b.lo", cometConfigs) @@ -104,7 +104,7 @@ object CometBroadcastNestedLoopJoinBenchmark extends CometBenchmarkBase { runBenchmark("BroadcastNestedLoopJoin - left outer with non-equi") { runExpressionBenchmark( "left outer non-equi", - probeRows, + probeRows.toLong, "SELECT /*+ BROADCAST(b) */ count(*) FROM probe p " + "LEFT OUTER JOIN build b ON p.k BETWEEN b.lo AND b.hi", cometConfigs) @@ -113,7 +113,7 @@ object CometBroadcastNestedLoopJoinBenchmark extends CometBenchmarkBase { runBenchmark("BroadcastNestedLoopJoin - range, materialized rows") { runExpressionBenchmark( "range join (BETWEEN, projected)", - probeRows, + probeRows.toLong, "SELECT /*+ BROADCAST(b) */ p.k, p.v, b.lo, b.hi FROM probe p " + "JOIN build b ON p.k BETWEEN b.lo AND b.hi", cometConfigs) diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastBooleanBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastBooleanBenchmark.scala index 57b8e88a7b5..8419ade9e63 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastBooleanBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastBooleanBenchmark.scala @@ -89,7 +89,7 @@ object CometCastBooleanBenchmark extends CometBenchmarkBase { """)) (boolToStringConfigs ++ boolToNumericConfigs).foreach { config => - runExpressionBenchmark(config.name, v, config.query, config.extraCometConfigs) + runExpressionBenchmark(config.name, v.toLong, config.query, config.extraCometConfigs) } } } @@ -115,7 +115,7 @@ object CometCastBooleanBenchmark extends CometBenchmarkBase { """)) numericToBoolConfigs.foreach { config => - runExpressionBenchmark(config.name, v, config.query, config.extraCometConfigs) + runExpressionBenchmark(config.name, v.toLong, config.query, config.extraCometConfigs) } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastNumericToNumericBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastNumericToNumericBenchmark.scala index 2141b1bea08..12ae85374d0 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastNumericToNumericBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastNumericToNumericBenchmark.scala @@ -148,7 +148,7 @@ object CometCastNumericToNumericBenchmark extends CometBenchmarkBase { generateConfigs(floatToIntPairs) ++ generateConfigs(decimalPairs) ++ generateConfigs(floatToDecimalPairs)).foreach { config => - runExpressionBenchmark(config.name, v, config.query, config.extraCometConfigs) + runExpressionBenchmark(config.name, v.toLong, config.query, config.extraCometConfigs) } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastNumericToStringBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastNumericToStringBenchmark.scala index 1fd2138c581..cdf719b0406 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastNumericToStringBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastNumericToStringBenchmark.scala @@ -95,7 +95,7 @@ object CometCastNumericToStringBenchmark extends CometBenchmarkBase { """)) castConfigs.foreach { config => - runExpressionBenchmark(config.name, v, config.query, config.extraCometConfigs) + runExpressionBenchmark(config.name, v.toLong, config.query, config.extraCometConfigs) } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastNumericToTemporalBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastNumericToTemporalBenchmark.scala index ec2d9ab12f1..e54127c9dc6 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastNumericToTemporalBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastNumericToTemporalBenchmark.scala @@ -70,7 +70,7 @@ object CometCastNumericToTemporalBenchmark extends CometBenchmarkBase { """)) intToDateConfigs.foreach { config => - runExpressionBenchmark(config.name, v, config.query, config.extraCometConfigs) + runExpressionBenchmark(config.name, v.toLong, config.query, config.extraCometConfigs) } } } @@ -92,7 +92,7 @@ object CometCastNumericToTemporalBenchmark extends CometBenchmarkBase { """)) longToTimestampConfigs.foreach { config => - runExpressionBenchmark(config.name, v, config.query, config.extraCometConfigs) + runExpressionBenchmark(config.name, v.toLong, config.query, config.extraCometConfigs) } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastStringToNumericBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastStringToNumericBenchmark.scala index c71eadad8c4..ac9c172e5b8 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastStringToNumericBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastStringToNumericBenchmark.scala @@ -89,7 +89,7 @@ object CometCastStringToNumericBenchmark extends CometBenchmarkBase { """)) castConfigs.foreach { config => - runExpressionBenchmark(config.name, v, config.query, config.extraCometConfigs) + runExpressionBenchmark(config.name, v.toLong, config.query, config.extraCometConfigs) } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastStringToTemporalBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastStringToTemporalBenchmark.scala index 77cc009ae10..31520824b73 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastStringToTemporalBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastStringToTemporalBenchmark.scala @@ -71,7 +71,7 @@ object CometCastStringToTemporalBenchmark extends CometBenchmarkBase { // Run date cast benchmarks with the same data dateCastConfigs.foreach { config => - runExpressionBenchmark(config.name, v, config.query, config.extraCometConfigs) + runExpressionBenchmark(config.name, v.toLong, config.query, config.extraCometConfigs) } } } @@ -94,7 +94,7 @@ object CometCastStringToTemporalBenchmark extends CometBenchmarkBase { // Run timestamp cast benchmarks with the same data timestampCastConfigs.foreach { config => - runExpressionBenchmark(config.name, v, config.query, config.extraCometConfigs) + runExpressionBenchmark(config.name, v.toLong, config.query, config.extraCometConfigs) } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastTemporalToNumericBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastTemporalToNumericBenchmark.scala index 1468cbe086f..6a4b9f81b06 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastTemporalToNumericBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastTemporalToNumericBenchmark.scala @@ -74,7 +74,7 @@ object CometCastTemporalToNumericBenchmark extends CometBenchmarkBase { """)) dateToNumericConfigs.foreach { config => - runExpressionBenchmark(config.name, v, config.query, config.extraCometConfigs) + runExpressionBenchmark(config.name, v.toLong, config.query, config.extraCometConfigs) } } } @@ -96,7 +96,7 @@ object CometCastTemporalToNumericBenchmark extends CometBenchmarkBase { """)) timestampToNumericConfigs.foreach { config => - runExpressionBenchmark(config.name, v, config.query, config.extraCometConfigs) + runExpressionBenchmark(config.name, v.toLong, config.query, config.extraCometConfigs) } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastTemporalToStringBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastTemporalToStringBenchmark.scala index 1ef3e7711d2..5e77742a345 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastTemporalToStringBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastTemporalToStringBenchmark.scala @@ -67,7 +67,7 @@ object CometCastTemporalToStringBenchmark extends CometBenchmarkBase { """)) dateCastConfigs.foreach { config => - runExpressionBenchmark(config.name, v, config.query, config.extraCometConfigs) + runExpressionBenchmark(config.name, v.toLong, config.query, config.extraCometConfigs) } } } @@ -89,7 +89,7 @@ object CometCastTemporalToStringBenchmark extends CometBenchmarkBase { """)) timestampCastConfigs.foreach { config => - runExpressionBenchmark(config.name, v, config.query, config.extraCometConfigs) + runExpressionBenchmark(config.name, v.toLong, config.query, config.extraCometConfigs) } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastTemporalToTemporalBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastTemporalToTemporalBenchmark.scala index f2e25724873..c030846861a 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastTemporalToTemporalBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastTemporalToTemporalBenchmark.scala @@ -69,7 +69,7 @@ object CometCastTemporalToTemporalBenchmark extends CometBenchmarkBase { """)) dateToTimestampConfigs.foreach { config => - runExpressionBenchmark(config.name, v, config.query, config.extraCometConfigs) + runExpressionBenchmark(config.name, v.toLong, config.query, config.extraCometConfigs) } } } @@ -91,7 +91,7 @@ object CometCastTemporalToTemporalBenchmark extends CometBenchmarkBase { """)) timestampToDateConfigs.foreach { config => - runExpressionBenchmark(config.name, v, config.query, config.extraCometConfigs) + runExpressionBenchmark(config.name, v.toLong, config.query, config.extraCometConfigs) } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCodegenDispatchBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCodegenDispatchBenchmark.scala index 53a8fefa7f3..05370a7afc7 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCodegenDispatchBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCodegenDispatchBenchmark.scala @@ -294,7 +294,7 @@ object CometCodegenDispatchBenchmark extends CometBenchmarkBase { private def runSteadyState(c: DispatchCase, rows: Int): Unit = { runBenchmark(s"${c.name} -- $rows rows") { - val benchmark = new Benchmark(s"${c.name} -- $rows rows", rows, output = output) + val benchmark = new Benchmark(s"${c.name} -- $rows rows", rows.toLong, output = output) checkPlans(benchmark, c) // The dispatch-off arm goes first so the `Relative` column reads as the speedup this // change buys over the behaviour that shipped before it. @@ -364,7 +364,7 @@ object CometCodegenDispatchBenchmark extends CometBenchmarkBase { private def withCorpus(rows: Int)(f: => Unit): Unit = { withTempPath { dir => withTempTable(tbl, "parquetV1Table") { - spark.range(rows).createOrReplaceTempView(tbl) + spark.range(rows.toLong).createOrReplaceTempView(tbl) prepareTable(dir, spark.sql(corpusQuery)) f } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometColumnarToRowBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometColumnarToRowBenchmark.scala index 22e7dbb54dd..1a05bdbcd73 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometColumnarToRowBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometColumnarToRowBenchmark.scala @@ -150,13 +150,13 @@ object CometColumnarToRowBenchmark extends CometBenchmarkBase { */ def primitiveTypesBenchmark(values: Int): Unit = { val benchmark = - new Benchmark("Columnar to Row - Primitive Types", values, output = output) + new Benchmark("Columnar to Row - Primitive Types", values.toLong, output = output) withTempPath { dir => withTempTable("parquetV1Table") { // Create a table with various primitive types (includes strings) val df = spark - .range(values) + .range(values.toLong) .selectExpr( "id as long_col", "cast(id as int) as int_col", @@ -182,13 +182,16 @@ object CometColumnarToRowBenchmark extends CometBenchmarkBase { */ def fixedWidthOnlyBenchmark(values: Int): Unit = { val benchmark = - new Benchmark("Columnar to Row - Fixed Width Only (no strings)", values, output = output) + new Benchmark( + "Columnar to Row - Fixed Width Only (no strings)", + values.toLong, + output = output) withTempPath { dir => withTempTable("parquetV1Table") { // Create a table with ONLY fixed-width primitive types (no strings!) val df = spark - .range(values) + .range(values.toLong) .selectExpr( "id as long_col", "cast(id as int) as int_col", @@ -214,12 +217,12 @@ object CometColumnarToRowBenchmark extends CometBenchmarkBase { */ def stringTypesBenchmark(values: Int): Unit = { val benchmark = - new Benchmark("Columnar to Row - String Types", values, output = output) + new Benchmark("Columnar to Row - String Types", values.toLong, output = output) withTempPath { dir => withTempTable("parquetV1Table") { val df = spark - .range(values) + .range(values.toLong) .selectExpr( "id", "concat('short_', cast(id % 100 as string)) as short_str", @@ -239,12 +242,12 @@ object CometColumnarToRowBenchmark extends CometBenchmarkBase { */ def structTypesBenchmark(values: Int): Unit = { val benchmark = - new Benchmark("Columnar to Row - Struct Types", values, output = output) + new Benchmark("Columnar to Row - Struct Types", values.toLong, output = output) withTempPath { dir => withTempTable("parquetV1Table") { val df = spark - .range(values) + .range(values.toLong) .selectExpr( "id", // Simple struct @@ -279,12 +282,12 @@ object CometColumnarToRowBenchmark extends CometBenchmarkBase { */ def arrayTypesBenchmark(values: Int): Unit = { val benchmark = - new Benchmark("Columnar to Row - Array Types", values, output = output) + new Benchmark("Columnar to Row - Array Types", values.toLong, output = output) withTempPath { dir => withTempTable("parquetV1Table") { val df = spark - .range(values) + .range(values.toLong) .selectExpr( "id", // Array of primitives @@ -310,12 +313,12 @@ object CometColumnarToRowBenchmark extends CometBenchmarkBase { */ def mapTypesBenchmark(values: Int): Unit = { val benchmark = - new Benchmark("Columnar to Row - Map Types", values, output = output) + new Benchmark("Columnar to Row - Map Types", values.toLong, output = output) withTempPath { dir => withTempTable("parquetV1Table") { val df = spark - .range(values) + .range(values.toLong) .selectExpr( "id", // Map with string keys and int values @@ -343,12 +346,12 @@ object CometColumnarToRowBenchmark extends CometBenchmarkBase { */ def complexNestedTypesBenchmark(values: Int): Unit = { val benchmark = - new Benchmark("Columnar to Row - Complex Nested Types", values, output = output) + new Benchmark("Columnar to Row - Complex Nested Types", values.toLong, output = output) withTempPath { dir => withTempTable("parquetV1Table") { val df = spark - .range(values) + .range(values.toLong) .selectExpr( "id", // Array of structs @@ -380,7 +383,7 @@ object CometColumnarToRowBenchmark extends CometBenchmarkBase { */ def wideRowsBenchmark(values: Int): Unit = { val benchmark = - new Benchmark("Columnar to Row - Wide Rows (50 columns)", values, output = output) + new Benchmark("Columnar to Row - Wide Rows (50 columns)", values.toLong, output = output) withTempPath { dir => withTempTable("parquetV1Table") { @@ -395,7 +398,7 @@ object CometColumnarToRowBenchmark extends CometBenchmarkBase { } } - val df = spark.range(values).selectExpr(columns: _*) + val df = spark.range(values.toLong).selectExpr(columns: _*) prepareTable(dir, df) val query = "SELECT * FROM parquetV1Table" @@ -411,12 +414,15 @@ object CometColumnarToRowBenchmark extends CometBenchmarkBase { */ def dictionaryEncodedBenchmark(values: Int): Unit = { val benchmark = - new Benchmark("Columnar to Row - Dictionary-encoded strings", values, output = output) + new Benchmark( + "Columnar to Row - Dictionary-encoded strings", + values.toLong, + output = output) withTempPath { dir => withTempTable("parquetV1Table") { val df = spark - .range(values) + .range(values.toLong) .selectExpr( "id", "concat('val_', cast(id % 1000 as string)) as low_card_str", @@ -446,14 +452,14 @@ object CometColumnarToRowBenchmark extends CometBenchmarkBase { */ def jvmConsumerBenchmark(values: Int): Unit = { val benchmark = - new Benchmark("Columnar to Row - JVM UDF consumer", values, output = output) + new Benchmark("Columnar to Row - JVM UDF consumer", values.toLong, output = output) spark.udf.register("plus_one", (x: Long) => x + 1) withTempPath { dir => withTempTable("parquetV1Table") { val df = spark - .range(values) + .range(values.toLong) .selectExpr( "id as long_col", "cast(id as int) as int_col", diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometComparisonExpressionBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometComparisonExpressionBenchmark.scala index 56ce2ac2640..060b96b8ea4 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometComparisonExpressionBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometComparisonExpressionBenchmark.scala @@ -85,7 +85,7 @@ object CometComparisonExpressionBenchmark extends CometBenchmarkBase { """)) comparisonExpressions.foreach { config => - runExpressionBenchmark(config.name, v, config.query, config.extraCometConfigs) + runExpressionBenchmark(config.name, v.toLong, config.query, config.extraCometConfigs) } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometConcatWsBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometConcatWsBenchmark.scala index 6db28b3072b..54391073bbd 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometConcatWsBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometConcatWsBenchmark.scala @@ -98,7 +98,7 @@ object CometConcatWsBenchmark extends CometBenchmarkBase { // scalastyle:on println } runBenchmark(name) { - runExpressionBenchmark(name, rows, query, cometConfigs) + runExpressionBenchmark(name, rows.toLong, query, cometConfigs) } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometConditionalExpressionBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometConditionalExpressionBenchmark.scala index 5b3532d5f23..877318e96b4 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometConditionalExpressionBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometConditionalExpressionBenchmark.scala @@ -57,7 +57,7 @@ object CometConditionalExpressionBenchmark extends CometBenchmarkBase { prepareTestTable(values) { val query = "SELECT CASE WHEN c1 < 0 THEN '<0' WHEN c1 = 0 THEN '=0' ELSE '>0' END FROM parquetV1Table" - runExpressionBenchmark("Case When Literal (3 branches)", values, query) + runExpressionBenchmark("Case When Literal (3 branches)", values.toLong, query) } } @@ -78,7 +78,7 @@ object CometConditionalExpressionBenchmark extends CometBenchmarkBase { ELSE 'j' END FROM parquetV1Table """ - runExpressionBenchmark("Case When Literal (10 branches)", values, query) + runExpressionBenchmark("Case When Literal (10 branches)", values.toLong, query) } } @@ -87,7 +87,7 @@ object CometConditionalExpressionBenchmark extends CometBenchmarkBase { // Result expressions are column references, not literals val query = "SELECT CASE WHEN c1 < 0 THEN c3 WHEN c1 = 0 THEN c1 ELSE c3 + c1 END FROM parquetV1Table" - runExpressionBenchmark("Case When Column Result (3 branches)", values, query) + runExpressionBenchmark("Case When Column Result (3 branches)", values.toLong, query) } } @@ -108,14 +108,14 @@ object CometConditionalExpressionBenchmark extends CometBenchmarkBase { ELSE c1 + c2 + c3 END FROM parquetV1Table """ - runExpressionBenchmark("Case When Column Result (10 branches)", values, query) + runExpressionBenchmark("Case When Column Result (10 branches)", values.toLong, query) } } def ifLiteralBenchmark(values: Int): Unit = { prepareTestTable(values) { val query = "SELECT IF(c1 < 0, '<0', '>=0') FROM parquetV1Table" - runExpressionBenchmark("If Literal", values, query) + runExpressionBenchmark("If Literal", values.toLong, query) } } @@ -123,7 +123,7 @@ object CometConditionalExpressionBenchmark extends CometBenchmarkBase { prepareTestTable(values) { // Result expressions are column references val query = "SELECT IF(c1 < 0, c3, c1 + c3) FROM parquetV1Table" - runExpressionBenchmark("If Column Result", values, query) + runExpressionBenchmark("If Column Result", values.toLong, query) } } @@ -136,7 +136,7 @@ object CometConditionalExpressionBenchmark extends CometBenchmarkBase { IF(c2 < 75, 'c', 'd'))) FROM parquetV1Table """ - runExpressionBenchmark("Nested If Literal (4 outcomes)", values, query) + runExpressionBenchmark("Nested If Literal (4 outcomes)", values.toLong, query) } } @@ -148,7 +148,7 @@ object CometConditionalExpressionBenchmark extends CometBenchmarkBase { IF(c2 < 75, c1 + c3, c3 * 2))) FROM parquetV1Table """ - runExpressionBenchmark("Nested If Column Result (4 outcomes)", values, query) + runExpressionBenchmark("Nested If Column Result (4 outcomes)", values.toLong, query) } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCsvExpressionBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCsvExpressionBenchmark.scala index 1495b0320e2..3e0e02a4cc8 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCsvExpressionBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCsvExpressionBenchmark.scala @@ -63,7 +63,7 @@ object CometCsvExpressionBenchmark extends CometBenchmarkBase { CometConf.getExprAllowIncompatConfigKey( classOf[CsvToStructs]) -> "true") ++ config.extraCometConfigs - runExpressionBenchmark(config.name, values, config.query, extraConfigs) + runExpressionBenchmark(config.name, values.toLong, config.query, extraConfigs) } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometDatetimeExpressionBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometDatetimeExpressionBenchmark.scala index 8e2978714aa..1ff82a6d450 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometDatetimeExpressionBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometDatetimeExpressionBenchmark.scala @@ -45,7 +45,7 @@ object CometDatetimeExpressionBenchmark extends CometBenchmarkBase { Seq("YEAR", "MONTH").foreach { level => val name = s"Date Truncate - $level" val query = s"select trunc(dt, '$level') from parquetV1Table" - runExpressionBenchmark(name, values, query) + runExpressionBenchmark(name, values.toLong, query) } } } @@ -70,7 +70,7 @@ object CometDatetimeExpressionBenchmark extends CometBenchmarkBase { "MICROSECOND").foreach { level => val name = s"Timestamp Truncate - $level" val query = s"select date_trunc('$level', ts) from parquetV1Table" - runExpressionBenchmark(name, values, query) + runExpressionBenchmark(name, values.toLong, query) } } } @@ -85,7 +85,7 @@ object CometDatetimeExpressionBenchmark extends CometBenchmarkBase { withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> timeZone) { val name = s"Unix Timestamp from Timestamp ($timeZone)" val query = "select unix_timestamp(ts) from parquetV1Table" - runExpressionBenchmark(name, values, query) + runExpressionBenchmark(name, values.toLong, query) } } } @@ -101,7 +101,7 @@ object CometDatetimeExpressionBenchmark extends CometBenchmarkBase { withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> timeZone) { val name = s"Unix Timestamp from Date ($timeZone)" val query = "select unix_timestamp(dt) from parquetV1Table" - runExpressionBenchmark(name, values, query) + runExpressionBenchmark(name, values.toLong, query) } } } @@ -116,7 +116,7 @@ object CometDatetimeExpressionBenchmark extends CometBenchmarkBase { s"select concat(cast(abs(value) % 24 as string), ':', lpad(cast(abs(value) % 60 as string), 2, '0'), ':', lpad(cast(abs(value) % 60 as string), 2, '0')) as s FROM $tbl")) val name = "to_time" val query = "select to_time(s) from parquetV1Table" - runExpressionBenchmark(name, values, query) + runExpressionBenchmark(name, values.toLong, query) } } } @@ -130,7 +130,7 @@ object CometDatetimeExpressionBenchmark extends CometBenchmarkBase { s"select cast(abs(value) % 24 as int) as h, cast(abs(value) % 60 as int) as m, cast(abs(value) % 60 as decimal(16,6)) as s FROM $tbl")) val name = "make_time" val query = "select make_time(h, m, s) from parquetV1Table" - runExpressionBenchmark(name, values, query) + runExpressionBenchmark(name, values.toLong, query) } } } @@ -154,7 +154,7 @@ object CometDatetimeExpressionBenchmark extends CometBenchmarkBase { def consumeIntervals(): Unit = { spark.sql(query).queryExecution.toRdd.foreachPartition(_.foreach(_.getInterval(0))) } - val benchmark = new Benchmark("MakeInterval", values, output = output) + val benchmark = new Benchmark("MakeInterval", values.toLong, output = output) val cometConfigs = Map( CometConf.COMET_ENABLED.key -> "true", CometConf.COMET_EXEC_ENABLED.key -> "true", @@ -209,11 +209,14 @@ object CometDatetimeExpressionBenchmark extends CometBenchmarkBase { END AS dow FROM $tbl """)) - runExpressionBenchmark("NextDay", values, "select next_day(dt, dow) from parquetV1Table") + runExpressionBenchmark( + "NextDay", + values.toLong, + "select next_day(dt, dow) from parquetV1Table") if (isSpark40Plus) { runExpressionBenchmark( "NextDay - collated dayOfWeek", - values, + values.toLong, "select next_day(dt, dow collate utf8_lcase) from parquetV1Table") } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometExecBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometExecBenchmark.scala index 2e7e143a472..3e25e408004 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometExecBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometExecBenchmark.scala @@ -70,7 +70,10 @@ object CometExecBenchmark extends CometBenchmarkBase { def numericFilterExecBenchmark(values: Int, fractionOfZeros: Double): Unit = { val percentageOfZeros = fractionOfZeros * 100 val benchmark = - new Benchmark(s"Project + Filter Exec ($percentageOfZeros% zeros)", values, output = output) + new Benchmark( + s"Project + Filter Exec ($percentageOfZeros% zeros)", + values.toLong, + output = output) withTempPath { dir => withTempTable("parquetV1Table") { @@ -108,7 +111,7 @@ object CometExecBenchmark extends CometBenchmarkBase { } def subqueryExecBenchmark(values: Int): Unit = { - val benchmark = new Benchmark("Subquery", values, output = output) + val benchmark = new Benchmark("Subquery", values.toLong, output = output) withTempPath { dir => withTempTable("parquetV1Table") { @@ -140,7 +143,7 @@ object CometExecBenchmark extends CometBenchmarkBase { } def sortExecBenchmark(values: Int): Unit = { - val benchmark = new Benchmark("Sort Exec", values, output = output) + val benchmark = new Benchmark("Sort Exec", values.toLong, output = output) withTempPath { dir => withTempTable("parquetV1Table") { @@ -164,7 +167,8 @@ object CometExecBenchmark extends CometBenchmarkBase { } def sampleExecBenchmark(values: Int, fraction: Double): Unit = { - val benchmark = new Benchmark(s"Sample Exec (fraction $fraction)", values, output = output) + val benchmark = + new Benchmark(s"Sample Exec (fraction $fraction)", values.toLong, output = output) withTempPath { dir => withTempTable("parquetV1Table") { @@ -194,7 +198,7 @@ object CometExecBenchmark extends CometBenchmarkBase { } def expandExecBenchmark(values: Int): Unit = { - val benchmark = new Benchmark("Expand Exec", values, output = output) + val benchmark = new Benchmark("Expand Exec", values.toLong, output = output) withTempPath { dir => withTempTable("parquetV1Table") { @@ -232,7 +236,7 @@ object CometExecBenchmark extends CometBenchmarkBase { val benchmark = new Benchmark( s"BloomFilterAggregate Exec (cardinality $cardinality)", - values, + values.toLong, output = output) val funcId_bloom_filter_agg = new FunctionIdentifier("bloom_filter_agg") diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometExplodeBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometExplodeBenchmark.scala index aa16417ca2c..86909f839c5 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometExplodeBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometExplodeBenchmark.scala @@ -79,7 +79,7 @@ object CometExplodeBenchmark extends CometBenchmarkBase { * Rows of an `rows`-row dataset whose [[arrayColumn]] is neither NULL nor empty, which is how * many rows reach a non-outer generator with something to emit. */ - private def nonEmptyRows(rows: Int): Long = rows - (rows + 9) / 10 - (rows + 8) / 10 + private def nonEmptyRows(rows: Int): Long = (rows - (rows + 9) / 10 - (rows + 8) / 10).toLong /** * A SQL expression for an array of `len` elements of `elementExpr`, where `elementExpr` may @@ -88,7 +88,7 @@ object CometExplodeBenchmark extends CometBenchmarkBase { * Elements are wrapped in a never-taken null branch so the array types as `containsNull`; see * [[nullableExpr]]. */ - private def fullArray(elementExpr: String, len: Int, v: String = "x"): String = + private def fullArray(elementExpr: String, len: Int, v: String): String = s"transform(sequence(1, $len), $v -> ${nullableExpr(elementExpr, s"$v = 0")})" /** @@ -214,7 +214,7 @@ object CometExplodeBenchmark extends CometBenchmarkBase { /** Writes a view's rows to Parquet and registers it. */ private def createView(dir: File, view: TempView): Unit = { val path = s"${dir.getAbsolutePath}/${view.name}" - spark.range(view.rows).selectExpr(view.columns: _*).write.parquet(path) + spark.range(view.rows.toLong).selectExpr(view.columns: _*).write.parquet(path) spark.read.parquet(path).createOrReplaceTempView(view.name) } @@ -306,7 +306,7 @@ object CometExplodeBenchmark extends CometBenchmarkBase { /** Verifies a case's aggregate under both engines, then times it. */ private def runCase(name: String, benchmarkCase: Case, rows: Int = numRows): Unit = { verifySink(name, benchmarkCase.query, benchmarkCase.expected) - runExpressionBenchmark(name, rows, benchmarkCase.query) + runExpressionBenchmark(name, rows.toLong, benchmarkCase.query) } override def runCometBenchmark(mainArgs: Array[String]): Unit = { diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometGetJsonObjectBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometGetJsonObjectBenchmark.scala index fa639453475..6b18aed5dee 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometGetJsonObjectBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometGetJsonObjectBenchmark.scala @@ -38,7 +38,7 @@ object CometGetJsonObjectBenchmark extends CometBenchmarkBase { prepareTable( dir, spark - .range(numRows) + .range(numRows.toLong) .map { i => val name = s"user_$i" val age = (i % 80 + 18).toInt @@ -76,7 +76,11 @@ object CometGetJsonObjectBenchmark extends CometBenchmarkBase { benchmarks.foreach { config => runBenchmark(config.name) { - runExpressionBenchmark(config.name, v, config.query, config.extraCometConfigs) + runExpressionBenchmark( + config.name, + v.toLong, + config.query, + config.extraCometConfigs) } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometHashExpressionBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometHashExpressionBenchmark.scala index 68ee3c1d1e0..238d8b662b1 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometHashExpressionBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometHashExpressionBenchmark.scala @@ -65,7 +65,7 @@ object CometHashExpressionBenchmark extends CometBenchmarkBase { """)) hashExpressions.foreach { config => - runExpressionBenchmark(config.name, v, config.query, config.extraCometConfigs) + runExpressionBenchmark(config.name, v.toLong, config.query, config.extraCometConfigs) } } } @@ -124,7 +124,7 @@ object CometHashExpressionBenchmark extends CometBenchmarkBase { HashExprConfig("hash_decimal", "SELECT hash(c_decimal) FROM parquetV1Table")) primitiveHashBenchmarks.foreach { config => - runExpressionBenchmark(config.name, v, config.query, config.extraCometConfigs) + runExpressionBenchmark(config.name, v.toLong, config.query, config.extraCometConfigs) } } } @@ -175,7 +175,7 @@ object CometHashExpressionBenchmark extends CometBenchmarkBase { "SELECT hash(c_struct_multi) FROM parquetV1Table")) complexHashBenchmarks.foreach { config => - runExpressionBenchmark(config.name, v, config.query, config.extraCometConfigs) + runExpressionBenchmark(config.name, v.toLong, config.query, config.extraCometConfigs) } } } @@ -232,7 +232,7 @@ object CometHashExpressionBenchmark extends CometBenchmarkBase { "SELECT hash(c_array_of_struct) FROM parquetV1Table")) nestedHashBenchmarks.foreach { config => - runExpressionBenchmark(config.name, v, config.query, config.extraCometConfigs) + runExpressionBenchmark(config.name, v.toLong, config.query, config.extraCometConfigs) } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometHashJoinBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometHashJoinBenchmark.scala index ef2c325c547..8963451d809 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometHashJoinBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometHashJoinBenchmark.scala @@ -72,12 +72,12 @@ object CometHashJoinBenchmark extends CometBenchmarkBase { withTempPath { dir => withTempTable("probe", "build") { spark - .range(probeRows) + .range(probeRows.toLong) .selectExpr("id AS k", "id % 100 AS v") .write .parquet(s"${dir.getAbsolutePath}/probe") spark - .range(buildRows) + .range(buildRows.toLong) .selectExpr("id AS k", "id * 10 AS w") .write .parquet(s"${dir.getAbsolutePath}/build") @@ -88,7 +88,7 @@ object CometHashJoinBenchmark extends CometBenchmarkBase { runBenchmark("ShuffledHashJoin - inner count") { runExpressionBenchmark( "inner count", - probeRows, + probeRows.toLong, "SELECT /*+ SHUFFLE_HASH(b) */ count(*) FROM probe p JOIN build b ON p.k = b.k", cometConfigs) } @@ -96,7 +96,7 @@ object CometHashJoinBenchmark extends CometBenchmarkBase { runBenchmark("ShuffledHashJoin - inner projected") { runExpressionBenchmark( "inner projected", - probeRows, + probeRows.toLong, "SELECT /*+ SHUFFLE_HASH(b) */ p.k, p.v, b.w FROM probe p JOIN build b ON p.k = b.k", cometConfigs) } @@ -104,7 +104,7 @@ object CometHashJoinBenchmark extends CometBenchmarkBase { runBenchmark("ShuffledHashJoin - left outer") { runExpressionBenchmark( "left outer", - probeRows, + probeRows.toLong, "SELECT /*+ SHUFFLE_HASH(b) */ count(*) FROM probe p LEFT JOIN build b ON p.k = b.k", cometConfigs) } @@ -112,7 +112,7 @@ object CometHashJoinBenchmark extends CometBenchmarkBase { runBenchmark("ShuffledHashJoin - left semi") { runExpressionBenchmark( "left semi", - probeRows, + probeRows.toLong, "SELECT /*+ SHUFFLE_HASH(b) */ count(*) FROM probe p LEFT SEMI JOIN build b ON p.k = b.k", cometConfigs) } @@ -120,7 +120,7 @@ object CometHashJoinBenchmark extends CometBenchmarkBase { runBenchmark("ShuffledHashJoin - right outer") { runExpressionBenchmark( "right outer", - probeRows, + probeRows.toLong, "SELECT /*+ SHUFFLE_HASH(p) */ count(*) FROM probe p RIGHT JOIN build b ON p.k = b.k", cometConfigs) } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometIcebergReadBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometIcebergReadBenchmark.scala index b90b893712b..1a171a5adac 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometIcebergReadBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometIcebergReadBenchmark.scala @@ -34,7 +34,10 @@ object CometIcebergReadBenchmark extends CometBenchmarkBase { def icebergScanBenchmark(values: Int, dataType: DataType): Unit = { val sqlBenchmark = - new Benchmark(s"SQL Single ${dataType.sql} Iceberg Column Scan", values, output = output) + new Benchmark( + s"SQL Single ${dataType.sql} Iceberg Column Scan", + values.toLong, + output = output) withTempPath { dir => withTempTable("icebergTable") { diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometIcebergSystemFunctionBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometIcebergSystemFunctionBenchmark.scala index 7369a532304..97b11a54f3c 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometIcebergSystemFunctionBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometIcebergSystemFunctionBenchmark.scala @@ -154,7 +154,7 @@ object CometIcebergSystemFunctionBenchmark extends CometBenchmarkBase { cases.foreach { case (name, query) => verifyOutputsMatch(name, query) runBenchmark(name) { - runExpressionBenchmark(name, v, query) + runExpressionBenchmark(name, v.toLong, query) } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometInMemoryCacheBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometInMemoryCacheBenchmark.scala index 7d171ac26ee..af06f0bc10c 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometInMemoryCacheBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometInMemoryCacheBenchmark.scala @@ -37,7 +37,10 @@ object CometInMemoryCacheBenchmark extends CometBenchmarkBase { private val numRows = 5 * 1000 * 1000 private val cacheTable = "comet_cache_bench" private val sourceTable = "comet_cache_bench_src" - @volatile private var statsResult: (Array[Any], Array[Any], Array[Int]) = _ + // A sink for the benchmarked call's result: written but never read, so that neither the + // compiler nor the JIT can treat `gatherColumnStats` as dead code. Not `private`, because + // a private field that is only ever written is what `-Ywarn-unused:privates` reports. + @volatile var statsResult: (Array[Any], Array[Any], Array[Int]) = _ override def getSparkSession: SparkSession = { val conf = new SparkConf() @@ -75,7 +78,7 @@ object CometInMemoryCacheBenchmark extends CometBenchmarkBase { withTempTable(sourceTable, cacheTable) { spark - .range(0, numRows, 1, 16) + .range(0, numRows.toLong, 1, 16) .selectExpr( "id", "id % 1000 AS k", @@ -124,15 +127,15 @@ object CometInMemoryCacheBenchmark extends CometBenchmarkBase { var r = 0 while (r < batchSize) { columns(0).putLong(r, r.toLong) - columns(1).putLong(r, r % 1000) - columns(2).putLong(r, r + 1) + columns(1).putLong(r, (r % 1000).toLong) + columns(2).putLong(r, (r + 1).toLong) columns(3).putByteArray(r, s"str_a_${r % 100000}".getBytes(StandardCharsets.UTF_8)) columns(4).putByteArray(r, s"str_b_${r % 7919}".getBytes(StandardCharsets.UTF_8)) columns(5).putByteArray(r, s"str_c_$r".getBytes(StandardCharsets.UTF_8)) r += 1 } val serializer = new ArrowCachedBatchSerializer - val benchmark = new Benchmark("in-memory cache statistics", numRows, output = output) + val benchmark = new Benchmark("in-memory cache statistics", numRows.toLong, output = output) // One case measures this collector across commits; Spark's default cache has its own collector. benchmark.addCase("Comet statistics collector") { _ => var i = 0 @@ -154,7 +157,7 @@ object CometInMemoryCacheBenchmark extends CometBenchmarkBase { verifyPlan(query, nativeCacheEnabled = true) } - val benchmark = new Benchmark(name, numRows, output = output) + val benchmark = new Benchmark(name, numRows.toLong, output = output) benchmark.addCase("Spark cache scan + CometSparkColumnarToColumnar") { _ => withSQLConf(cacheConf(nativeCacheEnabled = false): _*) { diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometJsonExpressionBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometJsonExpressionBenchmark.scala index 82aae3d7b93..8a422bb27ba 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometJsonExpressionBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometJsonExpressionBenchmark.scala @@ -159,7 +159,7 @@ object CometJsonExpressionBenchmark extends CometBenchmarkBase { CometConf.getExprAllowIncompatConfigKey( classOf[StructsToJson]) -> "true") ++ config.extraCometConfigs - runExpressionBenchmark(config.name, values, config.query, extraConfigs) + runExpressionBenchmark(config.name, values.toLong, config.query, extraConfigs) } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometLengthOfJsonArrayBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometLengthOfJsonArrayBenchmark.scala index 8c09ce01cf1..53acadbc729 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometLengthOfJsonArrayBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometLengthOfJsonArrayBenchmark.scala @@ -41,7 +41,7 @@ object CometLengthOfJsonArrayBenchmark extends CometBenchmarkBase { prepareTable( dir, spark - .range(numRows) + .range(numRows.toLong) .map { i => val arrayLength = (i % 100).toInt (0 until arrayLength) @@ -61,7 +61,11 @@ object CometLengthOfJsonArrayBenchmark extends CometBenchmarkBase { benchmarks.foreach { config => runBenchmark(config.name) { - runExpressionBenchmark(config.name, v, config.query, config.extraCometConfigs) + runExpressionBenchmark( + config.name, + v.toLong, + config.query, + config.extraCometConfigs) } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometOperatorSerdeBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometOperatorSerdeBenchmark.scala index 2f3904c5df6..fa54e0dfe3b 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometOperatorSerdeBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometOperatorSerdeBenchmark.scala @@ -219,7 +219,7 @@ object CometOperatorSerdeBenchmark extends CometBenchmarkBase { val iterations = 100 val benchmark = new Benchmark( s"IcebergScan serde ($numPartitions partitions, ${tasks.size()} tasks)", - iterations, + iterations.toLong, output = output) // Benchmark: Convert FileScanTasks to protobuf (the convert() method) @@ -342,7 +342,7 @@ object CometOperatorSerdeBenchmark extends CometBenchmarkBase { val benchmark = new Benchmark( s"CometScanRule apply ($numPartitions partitions)", - iterations, + iterations.toLong, output = output) benchmark.addCase("CometScanRule.apply(sparkPlan)") { _ => diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometPaddingExpressionBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometPaddingExpressionBenchmark.scala index 6fab1ea7a76..58750ee9f4c 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometPaddingExpressionBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometPaddingExpressionBenchmark.scala @@ -34,7 +34,7 @@ object CometPaddingExpressionBenchmark extends CometBenchmarkBase { prepareTable( dir, spark - .range(rows) + .range(rows.toLong) .selectExpr( "CAST(id AS STRING) AS s", "CAST(id % 32 + 8 AS INT) AS len", @@ -48,7 +48,7 @@ object CometPaddingExpressionBenchmark extends CometBenchmarkBase { runBenchmark(name) { runExpressionBenchmark( name, - rows, + rows.toLong, s"SELECT $function($arguments) FROM parquetV1Table") } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometPartitionColumnBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometPartitionColumnBenchmark.scala index a7d170057f4..cdc93a44ff1 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometPartitionColumnBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometPartitionColumnBenchmark.scala @@ -39,7 +39,7 @@ object CometPartitionColumnBenchmark extends CometBenchmarkBase { def partitionColumnScanBenchmark(values: Int, numPartitionCols: Int): Unit = { val sqlBenchmark = new Benchmark( s"Partitioned Scan with $numPartitionCols partition column(s)", - values, + values.toLong, output = output) withTempPath { dir => diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometPredicateExpressionBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometPredicateExpressionBenchmark.scala index 6506c5665d9..a7e22d8e90d 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometPredicateExpressionBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometPredicateExpressionBenchmark.scala @@ -38,7 +38,7 @@ object CometPredicateExpressionBenchmark extends CometBenchmarkBase { val query = "select * from parquetV1Table where c1 in ('positive', 'zero')" - runExpressionBenchmark("in Expr", values, query) + runExpressionBenchmark("in Expr", values.toLong, query) } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometReadBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometReadBenchmark.scala index 1055240cd73..6b809d381b1 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometReadBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometReadBenchmark.scala @@ -77,7 +77,11 @@ class CometReadBaseBenchmark extends CometBenchmarkBase { val operators = 1024 val creation = - new Benchmark("Scan I/O SQL metrics: creation", operators, minNumIters = 5, output = output) + new Benchmark( + "Scan I/O SQL metrics: creation", + operators.toLong, + minNumIters = 5, + output = output) creation.addCase("nine metrics per operator") { _ => var count = 0 while (count < operators) { @@ -91,7 +95,7 @@ class CometReadBaseBenchmark extends CometBenchmarkBase { val driverMetrics = createMetrics() val updates = new Benchmark( "Scan I/O SQL metrics: task snapshots", - tasks, + tasks.toLong, minNumIters = 5, output = output) updates.addCase("copy, update and merge nine metrics") { _ => @@ -112,7 +116,7 @@ class CometReadBaseBenchmark extends CometBenchmarkBase { val partitions = spark.sparkContext.parallelize(0 until tasks, tasks) val jobs = new Benchmark( "Scan I/O SQL metrics: 10000-task Spark job", - tasks, + tasks.toLong, minNumIters = 3, output = output) val cases = if (reverseCases) Seq(9, 0) else Seq(0, 9) @@ -131,7 +135,7 @@ class CometReadBaseBenchmark extends CometBenchmarkBase { def numericScanBenchmark(values: Int, dataType: DataType): Unit = { val sqlBenchmark = - new Benchmark(s"SQL Single ${dataType.sql} Column Scan", values, output = output) + new Benchmark(s"SQL Single ${dataType.sql} Column Scan", values.toLong, output = output) withTempPath { dir => withTempTable("parquetV1Table") { @@ -150,7 +154,10 @@ class CometReadBaseBenchmark extends CometBenchmarkBase { def encryptedScanBenchmark(values: Int, dataType: DataType): Unit = { val sqlBenchmark = - new Benchmark(s"SQL Single ${dataType.sql} Encrypted Column Scan", values, output = output) + new Benchmark( + s"SQL Single ${dataType.sql} Encrypted Column Scan", + values.toLong, + output = output) val encoder = Base64.getEncoder val footerKey = @@ -191,7 +198,7 @@ class CometReadBaseBenchmark extends CometBenchmarkBase { def decimalScanBenchmark(values: Int, precision: Int, scale: Int): Unit = { val sqlBenchmark = new Benchmark( s"SQL Single Decimal(precision: $precision, scale: $scale) Column Scan", - values, + values.toLong, output = output) withTempPath { dir => @@ -210,7 +217,7 @@ class CometReadBaseBenchmark extends CometBenchmarkBase { def readerBenchmark(values: Int, dataType: DataType): Unit = { val sqlBenchmark = - new Benchmark(s"Parquet reader benchmark for $dataType", values, output = output) + new Benchmark(s"Parquet reader benchmark for $dataType", values.toLong, output = output) withTempPath { dir => withTempTable("parquetV1Table") { @@ -268,7 +275,10 @@ class CometReadBaseBenchmark extends CometBenchmarkBase { def numericFilterScanBenchmark(values: Int, fractionOfZeros: Double): Unit = { val percentageOfZeros = fractionOfZeros * 100 val benchmark = - new Benchmark(s"Numeric Filter Scan ($percentageOfZeros% zeros)", values, output = output) + new Benchmark( + s"Numeric Filter Scan ($percentageOfZeros% zeros)", + values.toLong, + output = output) withTempPath { dir => withTempTable("parquetV1Table", "parquetV2Table") { @@ -286,7 +296,7 @@ class CometReadBaseBenchmark extends CometBenchmarkBase { def stringWithDictionaryScanBenchmark(values: Int): Unit = { val sqlBenchmark = - new Benchmark("String Scan with Dictionary Encoding", values, output = output) + new Benchmark("String Scan with Dictionary Encoding", values.toLong, output = output) withTempPath { dir => withTempTable("parquetV1Table", "parquetV2Table") { @@ -316,7 +326,10 @@ class CometReadBaseBenchmark extends CometBenchmarkBase { def stringWithNullsScanBenchmark(values: Int, fractionOfNulls: Double): Unit = { val percentageOfNulls = fractionOfNulls * 100 val benchmark = - new Benchmark(s"String with Nulls Scan ($percentageOfNulls%)", values, output = output) + new Benchmark( + s"String with Nulls Scan ($percentageOfNulls%)", + values.toLong, + output = output) withTempPath { dir => withTempTable("parquetV1Table") { @@ -337,7 +350,7 @@ class CometReadBaseBenchmark extends CometBenchmarkBase { def columnsBenchmark(values: Int, width: Int): Unit = { val benchmark = - new Benchmark(s"Single Column Scan from $width columns", values, output = output) + new Benchmark(s"Single Column Scan from $width columns", values.toLong, output = output) withTempPath { dir => withTempTable("t1", "parquetV1Table") { @@ -358,7 +371,7 @@ class CometReadBaseBenchmark extends CometBenchmarkBase { val benchmark = new Benchmark( s"Large String Filter Scan ($percentageOfZeros% zeros)", - values, + values.toLong, output = output) withTempPath { dir => @@ -380,7 +393,7 @@ class CometReadBaseBenchmark extends CometBenchmarkBase { val benchmark = new Benchmark( s"Sorted Lg Str Filter Scan ($percentageOfZeros% zeros)", - values, + values.toLong, output = output) withTempPath { dir => diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometRegExpBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometRegExpBenchmark.scala index 5a6741dc2a8..8ab4ecf5996 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometRegExpBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometRegExpBenchmark.scala @@ -71,7 +71,7 @@ object CometRegExpBenchmark extends CometBenchmarkBase { patterns.foreach { p => val query = s"select c1 rlike '${p.pattern}' from parquetV1Table" runBenchmark(p.name) { - runRLikeModes(p.name, v, query) + runRLikeModes(p.name, v.toLong, query) } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometRegExpExtractBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometRegExpExtractBenchmark.scala index 26cc7f73f14..1d8d423f060 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometRegExpExtractBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometRegExpExtractBenchmark.scala @@ -112,14 +112,14 @@ object CometRegExpExtractBenchmark extends CometBenchmarkBase { val extractQuery = s"select regexp_extract(c1, '${p.pattern}', ${p.idx}) from parquetV1Table" runBenchmark(extractName) { - runModes("RegExpExtract", extractName, v, extractQuery) + runModes("RegExpExtract", extractName, v.toLong, extractQuery) } val extractAllName = s"regexp_extract_all / ${p.name}" val extractAllQuery = s"select regexp_extract_all(c1, '${p.pattern}', ${p.idx}) from parquetV1Table" runBenchmark(extractAllName) { - runModes("RegExpExtractAll", extractAllName, v, extractAllQuery) + runModes("RegExpExtractAll", extractAllName, v.toLong, extractAllQuery) } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometSequenceBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometSequenceBenchmark.scala index 93f41ccf6b2..1b559ee6ddc 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometSequenceBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometSequenceBenchmark.scala @@ -70,7 +70,7 @@ object CometSequenceBenchmark extends CometBenchmarkBase { sequenceQueries.foreach { case (name, query) => runBenchmark(name) { - runExpressionBenchmark(name, v, query) + runExpressionBenchmark(name, v.toLong, query) } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometShuffleBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometShuffleBenchmark.scala index 64d9d0237af..db63d2ef942 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometShuffleBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometShuffleBenchmark.scala @@ -110,7 +110,9 @@ object CometShuffleBenchmark extends CometBenchmarkBase { def shuffleArrayBenchmark(values: Int, dataType: DataType, partitionNum: Int): Unit = { val benchmark = - microBenchmark(s"SQL ${dataType.sql} shuffle on array ($partitionNum Partition)", values) + microBenchmark( + s"SQL ${dataType.sql} shuffle on array ($partitionNum Partition)", + values.toLong) withTempPath { dir => withTempTable("parquetV1Table") { @@ -158,7 +160,9 @@ object CometShuffleBenchmark extends CometBenchmarkBase { def shuffleStructBenchmark(values: Int, dataType: DataType, partitionNum: Int): Unit = { val benchmark = - microBenchmark(s"SQL ${dataType.sql} shuffle on struct ($partitionNum Partition)", values) + microBenchmark( + s"SQL ${dataType.sql} shuffle on struct ($partitionNum Partition)", + values.toLong) withTempPath { dir => withTempTable("parquetV1Table") { @@ -213,7 +217,9 @@ object CometShuffleBenchmark extends CometBenchmarkBase { def shuffleDictionaryBenchmark(values: Int, dataType: DataType, partitionNum: Int): Unit = { val benchmark = - microBenchmark(s"SQL ${dataType.sql} Dictionary Shuffle($partitionNum Partition)", values) + microBenchmark( + s"SQL ${dataType.sql} Dictionary Shuffle($partitionNum Partition)", + values.toLong) withTempPath { dir => withTempTable("parquetV1Table") { @@ -298,7 +304,7 @@ object CometShuffleBenchmark extends CometBenchmarkBase { val benchmark = microBenchmark( s"SQL Wide ($width cols) ${dataType.sql} Shuffle($partitionNum Partition)", - values) + values.toLong) val projection = (1 to width) .map(i => s"CAST(CAST(RAND(1) * 100 AS INTEGER) AS ${dataType.sql}) AS c$i") @@ -367,7 +373,7 @@ object CometShuffleBenchmark extends CometBenchmarkBase { val benchmark = microBenchmark( s"SQL Wide ($width cols) ${dataType.sql} Range Partition Shuffle($partitionNum Partition)", - values) + values.toLong) val projection = (1 to width) .map(i => s"CAST(CAST(RAND(1) * 100 AS INTEGER) AS ${dataType.sql}) AS c$i") @@ -434,7 +440,7 @@ object CometShuffleBenchmark extends CometBenchmarkBase { numRows: Int, partitionNum: Int): Unit = { val benchmark = - microBenchmark(s"Shuffle with nested schema ($name)", numRows) + microBenchmark(s"Shuffle with nested schema ($name)", numRows.toLong) val df = spark.read.parquet(filename) withTempTable("deeplyNestedTable") { df.createOrReplaceTempView("deeplyNestedTable") @@ -496,7 +502,7 @@ object CometShuffleBenchmark extends CometBenchmarkBase { values: Int, partitionNum: Int): Unit = { val benchmark = - microBenchmark(s"Nested hash key: $name ($partitionNum Partition)", values) + microBenchmark(s"Nested hash key: $name ($partitionNum Partition)", values.toLong) withTempPath { dir => withTempTable("parquetV1Table") { diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometSortMergeJoinBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometSortMergeJoinBenchmark.scala index a429da58350..9cf3fdb5dbd 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometSortMergeJoinBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometSortMergeJoinBenchmark.scala @@ -71,12 +71,12 @@ object CometSortMergeJoinBenchmark extends CometBenchmarkBase { withTempPath { dir => withTempTable("probe", "build") { spark - .range(probeRows) + .range(probeRows.toLong) .selectExpr("id AS k", "id % 100 AS v") .write .parquet(s"${dir.getAbsolutePath}/probe") spark - .range(buildRows) + .range(buildRows.toLong) .selectExpr("id AS k", "id * 10 AS w") .write .parquet(s"${dir.getAbsolutePath}/build") @@ -87,7 +87,7 @@ object CometSortMergeJoinBenchmark extends CometBenchmarkBase { runBenchmark("SortMergeJoin - inner count") { runExpressionBenchmark( "inner count", - probeRows, + probeRows.toLong, "SELECT /*+ MERGE(b) */ count(*) FROM probe p JOIN build b ON p.k = b.k", cometConfigs) } @@ -95,7 +95,7 @@ object CometSortMergeJoinBenchmark extends CometBenchmarkBase { runBenchmark("SortMergeJoin - inner projected") { runExpressionBenchmark( "inner projected", - probeRows, + probeRows.toLong, "SELECT /*+ MERGE(b) */ p.k, p.v, b.w FROM probe p JOIN build b ON p.k = b.k", cometConfigs) } @@ -103,7 +103,7 @@ object CometSortMergeJoinBenchmark extends CometBenchmarkBase { runBenchmark("SortMergeJoin - left outer") { runExpressionBenchmark( "left outer", - probeRows, + probeRows.toLong, "SELECT /*+ MERGE(b) */ count(*) FROM probe p LEFT JOIN build b ON p.k = b.k", cometConfigs) } @@ -111,7 +111,7 @@ object CometSortMergeJoinBenchmark extends CometBenchmarkBase { runBenchmark("SortMergeJoin - left semi") { runExpressionBenchmark( "left semi", - probeRows, + probeRows.toLong, "SELECT /*+ MERGE(b) */ count(*) FROM probe p LEFT SEMI JOIN build b ON p.k = b.k", cometConfigs) } @@ -119,7 +119,7 @@ object CometSortMergeJoinBenchmark extends CometBenchmarkBase { runBenchmark("SortMergeJoin - right outer") { runExpressionBenchmark( "right outer", - probeRows, + probeRows.toLong, "SELECT /*+ MERGE(p) */ count(*) FROM probe p RIGHT JOIN build b ON p.k = b.k", cometConfigs) } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometStringExpressionBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometStringExpressionBenchmark.scala index fe3c78dac93..aede98a129d 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometStringExpressionBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometStringExpressionBenchmark.scala @@ -127,7 +127,7 @@ object CometStringExpressionBenchmark extends CometBenchmarkBase { (stringExpressions ++ collatedStringExpressions).foreach { config => val allConfigs = extraConfigs ++ config.extraCometConfigs runBenchmark(config.name) { - runExpressionBenchmark(config.name, v, config.query, allConfigs) + runExpressionBenchmark(config.name, v.toLong, config.query, allConfigs) } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometUnBase64Benchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometUnBase64Benchmark.scala index 20ebfb9df8f..e70005fa063 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometUnBase64Benchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometUnBase64Benchmark.scala @@ -88,7 +88,7 @@ object CometUnBase64Benchmark extends CometBenchmarkBase { shapes.foreach { s => val query = s"select unbase64(${s.column}) from parquetV1Table" runBenchmark(s.name) { - runUnBase64Modes(s.name, v, query) + runUnBase64Modes(s.name, v.toLong, query) } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/CometPlanStabilitySuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/CometPlanStabilitySuite.scala index 8f309f5cd27..d7d94ce831c 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/CometPlanStabilitySuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/CometPlanStabilitySuite.scala @@ -28,11 +28,9 @@ import org.apache.commons.io.FileUtils import org.apache.spark.SparkContext import org.apache.spark.internal.config.{MEMORY_OFFHEAP_ENABLED, MEMORY_OFFHEAP_SIZE} import org.apache.spark.sql.TPCDSBase -import org.apache.spark.sql.catalyst.expressions.AttributeSet import org.apache.spark.sql.catalyst.util.resourceToString -import org.apache.spark.sql.execution.{ReusedSubqueryExec, SparkPlan, SubqueryBroadcastExec, SubqueryExec} import org.apache.spark.sql.execution.adaptive.DisableAdaptiveExecutionSuite -import org.apache.spark.sql.execution.exchange.{Exchange, ReusedExchangeExec, ValidateRequirements} +import org.apache.spark.sql.execution.exchange.ValidateRequirements import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.test.TestSparkSession @@ -66,7 +64,6 @@ trait CometPlanStabilitySuite extends DisableAdaptiveExecutionSuite with TPCDSBa getWorkspaceFilePath("spark", "src", "test", "resources", "tpcds-plan-stability").toFile } - private val referenceRegex = "#\\d+".r private val normalizeRegex = "#\\d+L?".r private val planIdRegex = "plan_id=\\d+".r @@ -182,64 +179,6 @@ trait CometPlanStabilitySuite extends DisableAdaptiveExecutionSuite with TPCDSBa } } - /** - * Get the simplified plan for a specific SparkPlan. In the simplified plan, the node only has - * its name and all the sorted reference and produced attributes names(without ExprId) and its - * simplified children as well. And we'll only identify the performance sensitive nodes, e.g., - * Exchange, Subquery, in the simplified plan. Given such a identical but simplified plan, we'd - * expect to avoid frequent plan changing and catch the possible meaningful regression. - */ - private def getSimplifiedPlan(plan: SparkPlan): String = { - val exchangeIdMap = new mutable.HashMap[Int, Int]() - val subqueriesMap = new mutable.HashMap[Int, Int]() - - def getId(plan: SparkPlan): Int = plan match { - case exchange: Exchange => - exchangeIdMap.getOrElseUpdate(exchange.id, exchangeIdMap.size + 1) - case ReusedExchangeExec(_, exchange) => - exchangeIdMap.getOrElseUpdate(exchange.id, exchangeIdMap.size + 1) - case subquery: SubqueryExec => - subqueriesMap.getOrElseUpdate(subquery.id, subqueriesMap.size + 1) - case subquery: SubqueryBroadcastExec => - subqueriesMap.getOrElseUpdate(subquery.id, subqueriesMap.size + 1) - case ReusedSubqueryExec(subquery) => - subqueriesMap.getOrElseUpdate(subquery.id, subqueriesMap.size + 1) - case _ => -1 - } - - /** - * Some expression names have ExprId in them due to using things such as - * "sum(sr_return_amt#14)", so we remove all of these using regex - */ - def cleanUpReferences(references: AttributeSet): String = { - referenceRegex.replaceAllIn(references.map(_.name).mkString(","), "") - } - - /** - * Generate a simplified plan as a string Example output: TakeOrderedAndProject - * [c_customer_id] WholeStageCodegen Project [c_customer_id] - */ - def simplifyNode(node: SparkPlan, depth: Int): String = { - val padding = " " * depth - var thisNode = node.nodeName - if (node.references.nonEmpty) { - thisNode += s" [${cleanUpReferences(node.references)}]" - } - if (node.producedAttributes.nonEmpty) { - thisNode += s" [${cleanUpReferences(node.producedAttributes)}]" - } - val id = getId(node) - if (id > 0) { - thisNode += s" #$id" - } - val childrenSimplified = node.children.map(simplifyNode(_, depth + 1)) - val subqueriesSimplified = node.subqueries.map(simplifyNode(_, depth + 1)) - s"$padding$thisNode\n${subqueriesSimplified.mkString("")}${childrenSimplified.mkString("")}" - } - - simplifyNode(plan, 0) - } - private def normalizeIds(plan: String): String = { val map = new mutable.HashMap[String, String]() normalizeRegex diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/CometTaskMetricsSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/CometTaskMetricsSuite.scala index 1c3500426c3..d2fb23c90b9 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/CometTaskMetricsSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/CometTaskMetricsSuite.scala @@ -308,11 +308,11 @@ class CometTaskMetricsSuite extends CometTestBase with AdaptiveSparkPlanHelper { def insert(value: Int): Unit = { val bytes = new Array[Byte](4 + 16) - Platform.putInt(bytes, Platform.BYTE_ARRAY_OFFSET, value) + Platform.putInt(bytes, Platform.BYTE_ARRAY_OFFSET.toLong, value) val row = new UnsafeRow(1) - row.pointTo(bytes, Platform.BYTE_ARRAY_OFFSET + 4, 16) + row.pointTo(bytes, (Platform.BYTE_ARRAY_OFFSET + 4).toLong, 16) row.setInt(0, value) - sorter.insertRecord(bytes, Platform.BYTE_ARRAY_OFFSET, bytes.length, value % 2) + sorter.insertRecord(bytes, Platform.BYTE_ARRAY_OFFSET.toLong, bytes.length, value % 2) } try { diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/arrow/CometArrowStreamSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/arrow/CometArrowStreamSuite.scala index d95a4c718e5..767a0a9d9a8 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/arrow/CometArrowStreamSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/arrow/CometArrowStreamSuite.scala @@ -122,7 +122,7 @@ class CometArrowStreamSuite extends AnyFunSuite with Matchers { } requiredInput.putInt(i, -i) booleanInput.putBoolean(i, (i & 1) == 0) - val decimal = Decimal(i % 10000, decimalType.precision, decimalType.scale) + val decimal = Decimal((i % 10000).toLong, decimalType.precision, decimalType.scale) decimalInput.putDecimal(i, decimal, decimalType.precision) i += 1 } @@ -147,7 +147,7 @@ class CometArrowStreamSuite extends AnyFunSuite with Matchers { requiredArrow.get(i) shouldBe -i booleanArrow.get(i) shouldBe (if ((i & 1) == 0) 1 else 0) decimalArrow.getObject(i) shouldBe - Decimal(i % 10000, decimalType.precision, decimalType.scale).toJavaBigDecimal + Decimal((i % 10000).toLong, decimalType.precision, decimalType.scale).toJavaBigDecimal i += 1 } // A realloc frees the old buffers, so cumulative allocations would exceed live memory. diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornConcurrentMaterializationSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornConcurrentMaterializationSuite.scala index 60132c984fd..7603c22d784 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornConcurrentMaterializationSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornConcurrentMaterializationSuite.scala @@ -315,7 +315,9 @@ private[shuffle] class ConcurrentRemoteMapStarts { } } -class CometCelebornConcurrentMaterializationTestManager(conf: SparkConf, isDriver: Boolean) +private[shuffle] class CometCelebornConcurrentMaterializationTestManager( + conf: SparkConf, + isDriver: Boolean) extends CometCelebornFallbackTestShuffleManager(conf, isDriver) { @volatile private[shuffle] var remoteMapStarts: Option[ConcurrentRemoteMapStarts] = None diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornLocalFetchFailureSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornLocalFetchFailureSuite.scala index 85829401d4d..a2f8f1e41e5 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornLocalFetchFailureSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornLocalFetchFailureSuite.scala @@ -88,7 +88,9 @@ class CometCelebornLocalFetchFailureSuite extends CometTestBase { } /** Injects one local block loss only after Spark accepts another result partition. */ -class CometCelebornLocalFetchFailureTestManager(conf: SparkConf, isDriver: Boolean) +private[shuffle] class CometCelebornLocalFetchFailureTestManager( + conf: SparkConf, + isDriver: Boolean) extends CometCelebornFallbackTestShuffleManager(conf, isDriver) { // This local-mode fixture keeps state in its SparkContext-owned shuffle manager. diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShuffleFallbackSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShuffleFallbackSuite.scala index 826ac7e86df..de3381febfe 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShuffleFallbackSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShuffleFallbackSuite.scala @@ -542,7 +542,7 @@ private[shuffle] class CompletedMaterializationJobs extends SparkListener { } } -class CometCelebornFallbackTestShuffleManager(conf: SparkConf, isDriver: Boolean) +private[shuffle] class CometCelebornFallbackTestShuffleManager(conf: SparkConf, isDriver: Boolean) extends CometCelebornShuffleManager( conf, isDriver,