From d4c79e3a17d5aa5eef0d3b07c0d1ef0275849bdf Mon Sep 17 00:00:00 2001 From: litiliu <38579068+litiliu@users.noreply.github.com> Date: Tue, 21 Jul 2026 10:31:16 +0800 Subject: [PATCH] [ci] Split Spark lake tests from compatibility lane --- .github/workflows/ci-template.yaml | 4 ++- .github/workflows/stage.sh | 10 ++++-- .../fluss/spark/lake/SparkLakeTest.java | 32 +++++++++++++++++++ .../lake/SparkLakeLogTableReadTest.scala | 2 ++ ...SparkLakePrimaryKeyTableReadTestBase.scala | 1 + .../paimon/SparkLakePaimonCatalogTest.scala | 2 ++ 6 files changed, 48 insertions(+), 3 deletions(-) create mode 100644 fluss-spark/fluss-spark-ut/src/test/java/org/apache/fluss/spark/lake/SparkLakeTest.java diff --git a/.github/workflows/ci-template.yaml b/.github/workflows/ci-template.yaml index 94404db15b..3b98e20432 100644 --- a/.github/workflows/ci-template.yaml +++ b/.github/workflows/ci-template.yaml @@ -36,12 +36,14 @@ jobs: strategy: fail-fast: false matrix: - module: [ core, flink1, flink2, spark3, spark3-scala213, lake ] + module: [ core, flink1, flink2, spark3, spark3-lake, spark3-scala213, lake ] include: - module: flink1 test-profile: "test-flink1" - module: flink2 test-profile: "test-flink2" + - module: spark3-lake + test-profile: "test-spark3" - module: spark3-scala213 scala-profile: "-Pscala-2.13" test-profile: "test-spark3" diff --git a/.github/workflows/stage.sh b/.github/workflows/stage.sh index a3c1ad5777..462e8389dc 100755 --- a/.github/workflows/stage.sh +++ b/.github/workflows/stage.sh @@ -21,9 +21,12 @@ STAGE_CORE="core" STAGE_FLINK1="flink1" STAGE_FLINK2="flink2" STAGE_SPARK="spark3" +STAGE_SPARK_LAKE="spark3-lake" STAGE_SPARK_SCALA213="spark3-scala213" STAGE_LAKE="lake" +SPARK_LAKE_TEST_TAG="org.apache.fluss.spark.lake.SparkLakeTest" + MODULES_FLINK1="\ fluss-flink/fluss-flink-1.20,\ fluss-flink/fluss-flink-1.18\ @@ -85,10 +88,13 @@ function get_test_modules_for_stage() { echo "-pl fluss-test-coverage,$modules_flink2" ;; (${STAGE_SPARK}) - echo "-pl fluss-test-coverage,$modules_spark3" + echo "-DtagsToExclude=$SPARK_LAKE_TEST_TAG -pl fluss-test-coverage,$modules_spark3" + ;; + (${STAGE_SPARK_LAKE}) + echo "-DtagsToInclude=$SPARK_LAKE_TEST_TAG -pl fluss-test-coverage,$modules_spark3" ;; (${STAGE_SPARK_SCALA213}) - echo "-Pscala-2.13 -pl fluss-test-coverage,$modules_spark3" + echo "-Pscala-2.13 -DtagsToExclude=$SPARK_LAKE_TEST_TAG -pl fluss-test-coverage,$modules_spark3" ;; (${STAGE_LAKE}) echo "-pl fluss-test-coverage,$modules_lake" diff --git a/fluss-spark/fluss-spark-ut/src/test/java/org/apache/fluss/spark/lake/SparkLakeTest.java b/fluss-spark/fluss-spark-ut/src/test/java/org/apache/fluss/spark/lake/SparkLakeTest.java new file mode 100644 index 0000000000..53592c4cb6 --- /dev/null +++ b/fluss-spark/fluss-spark-ut/src/test/java/org/apache/fluss/spark/lake/SparkLakeTest.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.spark.lake; + +import org.scalatest.TagAnnotation; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** Identifies Spark test suites that exercise lake storage integrations. */ +@TagAnnotation +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.TYPE) +public @interface SparkLakeTest {} diff --git a/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/lake/SparkLakeLogTableReadTest.scala b/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/lake/SparkLakeLogTableReadTest.scala index 5fd9109684..30b043c265 100644 --- a/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/lake/SparkLakeLogTableReadTest.scala +++ b/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/lake/SparkLakeLogTableReadTest.scala @@ -685,6 +685,7 @@ abstract class SparkLakeLogTableReadTest extends SparkLakeTableReadTestBase { } +@SparkLakeTest class SparkLakePaimonLogTableReadTest extends SparkLakeLogTableReadTest { override protected def dataLakeFormat: DataLakeFormat = DataLakeFormat.PAIMON @@ -707,6 +708,7 @@ class SparkLakePaimonLogTableReadTest extends SparkLakeLogTableReadTest { } } +@SparkLakeTest class SparkLakeIcebergLogTableReadTest extends SparkLakeLogTableReadTest { override protected def dataLakeFormat: DataLakeFormat = DataLakeFormat.ICEBERG diff --git a/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/lake/SparkLakePrimaryKeyTableReadTestBase.scala b/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/lake/SparkLakePrimaryKeyTableReadTestBase.scala index e8d7b1795c..b4e379f919 100644 --- a/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/lake/SparkLakePrimaryKeyTableReadTestBase.scala +++ b/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/lake/SparkLakePrimaryKeyTableReadTestBase.scala @@ -551,6 +551,7 @@ abstract class SparkLakePrimaryKeyTableReadTestBase extends SparkLakeTableReadTe } +@SparkLakeTest class SparkLakePaimonPrimaryKeyTableReadTest extends SparkLakePrimaryKeyTableReadTestBase { override protected def dataLakeFormat: DataLakeFormat = DataLakeFormat.PAIMON diff --git a/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/lake/paimon/SparkLakePaimonCatalogTest.scala b/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/lake/paimon/SparkLakePaimonCatalogTest.scala index d5a29e163e..fe5d3eb5d6 100644 --- a/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/lake/paimon/SparkLakePaimonCatalogTest.scala +++ b/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/lake/paimon/SparkLakePaimonCatalogTest.scala @@ -25,6 +25,7 @@ import org.apache.fluss.metadata.TableDescriptor.{BUCKET_COLUMN_NAME, OFFSET_COL import org.apache.fluss.server.utils.LakeStorageUtils import org.apache.fluss.spark.SparkCatalogTest import org.apache.fluss.spark.SparkConnectorOptions.{BUCKET_KEY, BUCKET_NUMBER, PRIMARY_KEY} +import org.apache.fluss.spark.lake.SparkLakeTest import org.scalatest.matchers.must.Matchers.contain import org.scalatest.matchers.should.Matchers.{a, convertToAnyShouldWrapper} @@ -33,6 +34,7 @@ import java.nio.file.Files import scala.collection.JavaConverters._ +@SparkLakeTest class SparkLakePaimonCatalogTest extends SparkCatalogTest { private var paimonCatalog: org.apache.paimon.catalog.Catalog = _