From 9de7b1d8c98c978170d220a30c892d6abc6893b0 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Thu, 10 Sep 2026 13:26:18 -0600 Subject: [PATCH] ci: retry the Maven wrapper bootstrap in every job that calls ./mvnw directly `Verify TPC-H Results` failed on #5850 at its `Build project` step, 105 seconds in and before anything was compiled. The check-run annotation gives the cause: https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.6/apache-maven-3.9.6-bin.zip That is `./mvnw` downloading the Maven distribution itself, not a dependency and not a test. `./.github/actions/java-test` already handles this: it caches the distribution under `~/.m2/wrapper/dists` and retries `./mvnw --version` four times with exponential backoff. But five jobs in pr_build_linux.yml never go through java-test -- they invoke `./mvnw` directly -- so none of them had either the cache or the retry: lint-java scalafix check build-spark-4-1 compile, skip tests celeborn-reflection-compatibility reflected-internals check verify-benchmark-results-tpch the job that failed verify-benchmark-results-tpcds same shape as TPC-H Extract the cache/retry/save sequence into `./.github/actions/maven-bootstrap` and call it from all five before their first Maven use. java-test keeps its inline copy: a local action invoking another local action is deliberately avoided in this repository, and the workflows README already says so about the artifact-upload wrapper. Register the new action in the Linux change filter. Without it, a later edit confined to `.github/actions/maven-bootstrap/**` routes to nothing: the `changes` gate reports `build_linux=false`, ci.yml skips the whole Linux workflow, and the edit merges without any of the five consumers having run it. This PR does not expose the gap, because it also edits pr_build_linux.yml. Pin the routing with a case in dev/ci/check-ci-config.py so a filter deletion cannot pass unnoticed either. The workflows README claimed this failure mode was handled. It was, but only for jobs routed through java-test; the wording is corrected. Worth noting for #5838: under a merge queue these jobs gate the queue, so this failure mode would block every merge rather than costing one PR a re-run. Follow-up, not covered here: `ci.yml`'s RAT check and the direct `./mvnw` calls in `pr_benchmark_check.yml`, `pyarrow_udf_test.yml` and `iceberg_spark_test_reusable.yml` have the same gap. Left out to keep this reviewable against the failure that prompted it. --- .github/actions/maven-bootstrap/action.yaml | 64 +++++++++++++++++++++ .github/workflows/README.md | 15 ++++- .github/workflows/pr_build_linux.yml | 19 ++++++ dev/ci/check-ci-config.py | 2 + dev/ci/compute-changes.py | 1 + 5 files changed, 98 insertions(+), 3 deletions(-) create mode 100644 .github/actions/maven-bootstrap/action.yaml diff --git a/.github/actions/maven-bootstrap/action.yaml b/.github/actions/maven-bootstrap/action.yaml new file mode 100644 index 0000000000..7aab0b6030 --- /dev/null +++ b/.github/actions/maven-bootstrap/action.yaml @@ -0,0 +1,64 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: "Bootstrap Maven" +description: > + Make ./mvnw usable, tolerating a flaky download of the Maven distribution + itself. Caches the distribution outside the dependency repository and retries + only the wrapper bootstrap, never compilation or test execution. + +runs: + using: "composite" + steps: + # Maven itself is stored outside the dependency repository. Keep its cache + # independent of pom.xml changes and preserve the macOS cache workaround. + - name: Restore Maven distribution + id: maven-distribution + if: ${{ runner.os != 'macOS' }} + uses: actions/cache/restore@v5 + with: + path: | + ~/.m2/wrapper/dists + /root/.m2/wrapper/dists + key: ${{ runner.os }}-${{ runner.arch }}-maven-wrapper-${{ hashFiles('.mvn/wrapper/maven-wrapper.properties') }} + + # Delays use exponential backoff (10s, 20s, 40s) plus 0-4s of random jitter. + - name: Bootstrap Maven + shell: bash + run: | + for attempt in 1 2 3 4; do + if ./mvnw -B --version; then + break + fi + if [ "$attempt" -eq 4 ]; then + echo "::error::Maven bootstrap failed after $attempt attempts; the job was not started." + exit 1 + fi + delay=$((10 * (1 << (attempt - 1)) + RANDOM % 5)) + echo "::warning::Maven bootstrap attempt $attempt failed; retrying in ${delay}s." + sleep "$delay" + done + + # Save a successful bootstrap even when the work that follows fails. + - name: Save Maven distribution + if: ${{ runner.os != 'macOS' && steps.maven-distribution.outputs.cache-hit != 'true' }} + uses: actions/cache/save@v5 + with: + path: | + ~/.m2/wrapper/dists + /root/.m2/wrapper/dists + key: ${{ steps.maven-distribution.outputs.cache-primary-key }} diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 34765f74a3..f294b37454 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -280,9 +280,18 @@ before any test has started. Once `Required Checks` is a required context which is why plain network flakes are worth retrying rather than re-running the whole pipeline by hand. -**Maven wrapper bootstrap.** `./.github/actions/java-test` retries -`./mvnw --version` with exponential backoff, so a failed download of the Maven -distribution does not surface as a test failure. +**Maven wrapper bootstrap.** `./mvnw` downloads the Maven distribution itself on +a cold runner, and a blip from `repo.maven.apache.org` fails the job before +anything is compiled. `./.github/actions/maven-bootstrap` caches that +distribution under `~/.m2/wrapper/dists` (keyed on +`.mvn/wrapper/maven-wrapper.properties`, not `pom.xml`) and retries +`./mvnw --version` four times with exponential backoff. It retries only the +bootstrap, never compilation or test execution. + +Any job whose first Maven use is a bare `./mvnw` needs this step before it. +`./.github/actions/java-test` carries its own inline copy rather than calling +the composite, because a local action invoking another local action is +deliberately avoided here (see the artifact-upload note above). ## Merge queue diff --git a/.github/workflows/pr_build_linux.yml b/.github/workflows/pr_build_linux.yml index a019e9a721..6814d75769 100644 --- a/.github/workflows/pr_build_linux.yml +++ b/.github/workflows/pr_build_linux.yml @@ -138,6 +138,9 @@ jobs: restore-keys: | ${{ runner.os }}-java-maven- + - name: Bootstrap Maven + uses: ./.github/actions/maven-bootstrap + - name: Run scalafix check run: | ./mvnw -B package -DskipTests scalafix:scalafix -Dscalafix.mode=CHECK -Psemanticdb ${{ matrix.profile.maven_opts }} @@ -192,6 +195,9 @@ jobs: restore-keys: | ${{ runner.os }}-java-maven- + - name: Bootstrap Maven + uses: ./.github/actions/maven-bootstrap + - name: Compile (skip tests) run: ./mvnw -B install -DskipTests -Dmaven.test.skip=true -Pspark-4.1 @@ -226,6 +232,9 @@ jobs: restore-keys: | ${{ runner.os }}-java-maven- + - name: Bootstrap Maven + uses: ./.github/actions/maven-bootstrap + - name: Verify reflected Celeborn internals env: SPARK_LOCAL_HOSTNAME: localhost @@ -584,6 +593,11 @@ jobs: path: ./tpch key: tpch-${{ hashFiles('.github/workflows/pr_build_linux.yml') }} + # Without this, a transient repo.maven.apache.org blip fails the whole + # job before anything is compiled. + - name: Bootstrap Maven + uses: ./.github/actions/maven-bootstrap + - name: Build project run: | ./mvnw -B -Prelease install -DskipTests @@ -639,6 +653,11 @@ jobs: path: ./tpcds-sf-1 key: tpcds-${{ hashFiles('.github/workflows/pr_build_linux.yml') }} + # Without this, a transient repo.maven.apache.org blip fails the whole + # job before anything is compiled. + - name: Bootstrap Maven + uses: ./.github/actions/maven-bootstrap + - name: Build project run: | ./mvnw -B -Prelease install -DskipTests diff --git a/dev/ci/check-ci-config.py b/dev/ci/check-ci-config.py index 0d30910f29..cb59bbe099 100644 --- a/dev/ci/check-ci-config.py +++ b/dev/ci/check-ci-config.py @@ -103,6 +103,8 @@ # to nothing at all and merges having been exercised by no consumer. ([".github/actions/upload-artifact-retry/action.yaml"], BUILD_JOBS), ([".github/actions/download-artifact-retry/action.yaml"], BUILD_JOBS), + # The Maven bootstrap composite is called only from pr_build_linux.yml. + ([".github/actions/maven-bootstrap/action.yaml"], {"build_linux"}), # Spot checks that the additions above did not widen unrelated routes. (["docs/source/user-guide/overview.md"], {"docs"}), (["native/core/benches/parquet_read.rs"], {"benchmark"}), diff --git a/dev/ci/compute-changes.py b/dev/ci/compute-changes.py index 9716ced234..7697e136bb 100644 --- a/dev/ci/compute-changes.py +++ b/dev/ci/compute-changes.py @@ -55,6 +55,7 @@ ".github/workflows/pr_build_linux.yml", ".github/actions/setup-builder/**", ".github/actions/java-test/**", + ".github/actions/maven-bootstrap/**", ".github/actions/rust-test/**", ".github/actions/upload-artifact-retry/**", ".github/actions/download-artifact-retry/**",