diff --git a/.github/actions/maven-bootstrap/action.yaml b/.github/actions/maven-bootstrap/action.yaml new file mode 100644 index 0000000000..7aab0b6030 --- /dev/null +++ b/.github/actions/maven-bootstrap/action.yaml @@ -0,0 +1,64 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: "Bootstrap Maven" +description: > + Make ./mvnw usable, tolerating a flaky download of the Maven distribution + itself. Caches the distribution outside the dependency repository and retries + only the wrapper bootstrap, never compilation or test execution. + +runs: + using: "composite" + steps: + # Maven itself is stored outside the dependency repository. Keep its cache + # independent of pom.xml changes and preserve the macOS cache workaround. + - name: Restore Maven distribution + id: maven-distribution + if: ${{ runner.os != 'macOS' }} + uses: actions/cache/restore@v5 + with: + path: | + ~/.m2/wrapper/dists + /root/.m2/wrapper/dists + key: ${{ runner.os }}-${{ runner.arch }}-maven-wrapper-${{ hashFiles('.mvn/wrapper/maven-wrapper.properties') }} + + # Delays use exponential backoff (10s, 20s, 40s) plus 0-4s of random jitter. + - name: Bootstrap Maven + shell: bash + run: | + for attempt in 1 2 3 4; do + if ./mvnw -B --version; then + break + fi + if [ "$attempt" -eq 4 ]; then + echo "::error::Maven bootstrap failed after $attempt attempts; the job was not started." + exit 1 + fi + delay=$((10 * (1 << (attempt - 1)) + RANDOM % 5)) + echo "::warning::Maven bootstrap attempt $attempt failed; retrying in ${delay}s." + sleep "$delay" + done + + # Save a successful bootstrap even when the work that follows fails. + - name: Save Maven distribution + if: ${{ runner.os != 'macOS' && steps.maven-distribution.outputs.cache-hit != 'true' }} + uses: actions/cache/save@v5 + with: + path: | + ~/.m2/wrapper/dists + /root/.m2/wrapper/dists + key: ${{ steps.maven-distribution.outputs.cache-primary-key }} diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 34765f74a3..f294b37454 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -280,9 +280,18 @@ before any test has started. Once `Required Checks` is a required context which is why plain network flakes are worth retrying rather than re-running the whole pipeline by hand. -**Maven wrapper bootstrap.** `./.github/actions/java-test` retries -`./mvnw --version` with exponential backoff, so a failed download of the Maven -distribution does not surface as a test failure. +**Maven wrapper bootstrap.** `./mvnw` downloads the Maven distribution itself on +a cold runner, and a blip from `repo.maven.apache.org` fails the job before +anything is compiled. `./.github/actions/maven-bootstrap` caches that +distribution under `~/.m2/wrapper/dists` (keyed on +`.mvn/wrapper/maven-wrapper.properties`, not `pom.xml`) and retries +`./mvnw --version` four times with exponential backoff. It retries only the +bootstrap, never compilation or test execution. + +Any job whose first Maven use is a bare `./mvnw` needs this step before it. +`./.github/actions/java-test` carries its own inline copy rather than calling +the composite, because a local action invoking another local action is +deliberately avoided here (see the artifact-upload note above). ## Merge queue diff --git a/.github/workflows/pr_build_linux.yml b/.github/workflows/pr_build_linux.yml index a019e9a721..6814d75769 100644 --- a/.github/workflows/pr_build_linux.yml +++ b/.github/workflows/pr_build_linux.yml @@ -138,6 +138,9 @@ jobs: restore-keys: | ${{ runner.os }}-java-maven- + - name: Bootstrap Maven + uses: ./.github/actions/maven-bootstrap + - name: Run scalafix check run: | ./mvnw -B package -DskipTests scalafix:scalafix -Dscalafix.mode=CHECK -Psemanticdb ${{ matrix.profile.maven_opts }} @@ -192,6 +195,9 @@ jobs: restore-keys: | ${{ runner.os }}-java-maven- + - name: Bootstrap Maven + uses: ./.github/actions/maven-bootstrap + - name: Compile (skip tests) run: ./mvnw -B install -DskipTests -Dmaven.test.skip=true -Pspark-4.1 @@ -226,6 +232,9 @@ jobs: restore-keys: | ${{ runner.os }}-java-maven- + - name: Bootstrap Maven + uses: ./.github/actions/maven-bootstrap + - name: Verify reflected Celeborn internals env: SPARK_LOCAL_HOSTNAME: localhost @@ -584,6 +593,11 @@ jobs: path: ./tpch key: tpch-${{ hashFiles('.github/workflows/pr_build_linux.yml') }} + # Without this, a transient repo.maven.apache.org blip fails the whole + # job before anything is compiled. + - name: Bootstrap Maven + uses: ./.github/actions/maven-bootstrap + - name: Build project run: | ./mvnw -B -Prelease install -DskipTests @@ -639,6 +653,11 @@ jobs: path: ./tpcds-sf-1 key: tpcds-${{ hashFiles('.github/workflows/pr_build_linux.yml') }} + # Without this, a transient repo.maven.apache.org blip fails the whole + # job before anything is compiled. + - name: Bootstrap Maven + uses: ./.github/actions/maven-bootstrap + - name: Build project run: | ./mvnw -B -Prelease install -DskipTests diff --git a/dev/ci/check-ci-config.py b/dev/ci/check-ci-config.py index 0d30910f29..cb59bbe099 100644 --- a/dev/ci/check-ci-config.py +++ b/dev/ci/check-ci-config.py @@ -103,6 +103,8 @@ # to nothing at all and merges having been exercised by no consumer. ([".github/actions/upload-artifact-retry/action.yaml"], BUILD_JOBS), ([".github/actions/download-artifact-retry/action.yaml"], BUILD_JOBS), + # The Maven bootstrap composite is called only from pr_build_linux.yml. + ([".github/actions/maven-bootstrap/action.yaml"], {"build_linux"}), # Spot checks that the additions above did not widen unrelated routes. (["docs/source/user-guide/overview.md"], {"docs"}), (["native/core/benches/parquet_read.rs"], {"benchmark"}), diff --git a/dev/ci/compute-changes.py b/dev/ci/compute-changes.py index 9716ced234..7697e136bb 100644 --- a/dev/ci/compute-changes.py +++ b/dev/ci/compute-changes.py @@ -55,6 +55,7 @@ ".github/workflows/pr_build_linux.yml", ".github/actions/setup-builder/**", ".github/actions/java-test/**", + ".github/actions/maven-bootstrap/**", ".github/actions/rust-test/**", ".github/actions/upload-artifact-retry/**", ".github/actions/download-artifact-retry/**",