From 46c09fac8666d52db2d7fc227bcfb0b0e4959707 Mon Sep 17 00:00:00 2001 From: cgivre Date: Tue, 8 Sep 2026 10:38:39 -0400 Subject: [PATCH 1/4] Shard CI tests and raise the surefire fork count The unit test suite dominates CI wall clock, and exec/java-exec alone accounts for roughly half of it. Split each matrix entry three ways by test class name and run two surefire forks per job. The shard patterns are exhaustive over every test class in the repo, so a newly added test always lands in a shard instead of being silently skipped. --- .github/workflows/ci.yml | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 23e4cdeb5b5..f93048c9bd7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,7 +27,7 @@ on: [push, pull_request] jobs: build: - name: Main Build + name: Main Build (java ${{ matrix.java }}, shard ${{ matrix.shard }}) runs-on: ubuntu-latest if: github.repository == 'apache/drill' timeout-minutes: 150 @@ -36,6 +36,12 @@ jobs: # Java versions to run unit tests (Jetty 12 requires Java 17+) java: [ '17', '21', '25' ] profile: ['default-hadoop'] + # exec/java-exec holds ~31 minutes of the suite in 577 classes, which dominates + # the build. Splitting it three ways by class name brings the longest shard to + # about 11 minutes. Boundaries were chosen from measured per-class times; the + # patterns are exhaustive, so a newly added test always lands in exactly one + # shard rather than being silently skipped. + shard: [ '1', '2', '3' ] fail-fast: false steps: - name: Checkout @@ -57,9 +63,36 @@ jobs: mkswap /tmp/swapfile swapon /tmp/swapfile " + # Which slice of the suite this job runs. The patterns are spelled out one prefix + # at a time because surefire does NOT support character-class globs: + # -Dtest='Test[A-F]*' silently matches nothing and the build passes green having + # run no tests. Only '*' wildcards and comma separated lists work, so each letter + # gets its own entry, in both cases (TestforBaseTestInheritance has a lowercase f). + # + # Shard 3 repeats surefire's other default includes (*Test, *Tests, *TestCase) to + # pick up classes not named Test*. Verified exhaustive against every test + # class in the repo. TestFrameworkTest matches shards 1 and 3 and runs twice — + # harmless, and far safer than a gap that skips tests without saying so. + - name: Select test shard + id: shard + run: | + case "${{ matrix.shard }}" in + 1) echo 'pattern=TestA*,Testa*,TestB*,Testb*,TestC*,Testc*,TestD*,Testd*,TestE*,Teste*,TestF*,Testf*' >> "$GITHUB_OUTPUT" ;; + 2) echo 'pattern=TestG*,Testg*,TestH*,Testh*,TestI*,Testi*,TestJ*,Testj*,TestK*,Testk*,TestL*,Testl*,TestM*,Testm*,TestN*,Testn*,TestO*,Testo*,TestP*,Testp*' >> "$GITHUB_OUTPUT" ;; + 3) echo 'pattern=TestQ*,Testq*,TestR*,Testr*,TestS*,Tests*,TestT*,Testt*,TestU*,Testu*,TestV*,Testv*,TestW*,Testw*,TestX*,Testx*,TestY*,Testy*,TestZ*,Testz*,*Test,*Tests,*TestCase,Test0*,Test1*,Test2*,Test3*,Test4*,Test5*,Test6*,Test7*,Test8*,Test9*,Test_*' >> "$GITHUB_OUTPUT" ;; + esac + + # -DforkCount=2: several of the slowest classes are wall-clock bound rather than + # CPU bound — TestTimedCallable spends 105 seconds asleep, and the RPC, SSL and + # graceful-shutdown tests wait on timeouts — so a second fork overlaps that idle + # time at little extra cost. Raise further only with memory headroom: the vector + # tests fail with OutOfMemoryException under pressure. - name: Build and test run: | mvn -P${{ matrix.profile }} install --batch-mode --no-transfer-progress \ + -DforkCount=2 \ + -Dsurefire.failIfNoSpecifiedTests=false \ + -Dtest='${{ steps.shard.outputs.pattern }}' \ -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 - name: Remove swap space run : | From 11c533f7aa871d540dcb1d5c331b9cd7fb376f81 Mon Sep 17 00:00:00 2001 From: cgivre Date: Wed, 21 Oct 2026 16:17:48 -0400 Subject: [PATCH 2/4] Fail the CI shard step on an unknown shard value --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f93048c9bd7..b2711787130 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -80,6 +80,7 @@ jobs: 1) echo 'pattern=TestA*,Testa*,TestB*,Testb*,TestC*,Testc*,TestD*,Testd*,TestE*,Teste*,TestF*,Testf*' >> "$GITHUB_OUTPUT" ;; 2) echo 'pattern=TestG*,Testg*,TestH*,Testh*,TestI*,Testi*,TestJ*,Testj*,TestK*,Testk*,TestL*,Testl*,TestM*,Testm*,TestN*,Testn*,TestO*,Testo*,TestP*,Testp*' >> "$GITHUB_OUTPUT" ;; 3) echo 'pattern=TestQ*,Testq*,TestR*,Testr*,TestS*,Tests*,TestT*,Testt*,TestU*,Testu*,TestV*,Testv*,TestW*,Testw*,TestX*,Testx*,TestY*,Testy*,TestZ*,Testz*,*Test,*Tests,*TestCase,Test0*,Test1*,Test2*,Test3*,Test4*,Test5*,Test6*,Test7*,Test8*,Test9*,Test_*' >> "$GITHUB_OUTPUT" ;; + *) echo "::error::Unknown shard '${{ matrix.shard }}'; expected 1, 2 or 3"; exit 1 ;; esac # -DforkCount=2: several of the slowest classes are wall-clock bound rather than From 1bcb2d1a9c7fa5cca4fd628ba74be7cafd2ef207 Mon Sep 17 00:00:00 2001 From: cgivre Date: Wed, 21 Oct 2026 16:20:01 -0400 Subject: [PATCH 3/4] Create the CI swap file with dd and continue without swap if swapon fails --- .github/workflows/ci.yml | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b2711787130..e118264513a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -56,13 +56,16 @@ jobs: # Linux Action Runners come with 7GB of RAM which isn't quite enoough # to run the test suite. Two workarounds are used below: a swap file is # added to the Runner and memory hungry tests are run separately. + # dd rather than fallocate: swapon rejects fallocated files on some runner + # filesystems ("Invalid argument"). A missing swap file only costs headroom, + # so warn instead of failing the job. run: | sudo sh -c " - fallocate -l 2G /tmp/swapfile - chmod 0600 /tmp/swapfile - mkswap /tmp/swapfile - swapon /tmp/swapfile - " + dd if=/dev/zero of=/drill-swapfile bs=1M count=2048 status=none && + chmod 0600 /drill-swapfile && + mkswap /drill-swapfile && + swapon /drill-swapfile + " || echo "::warning::Could not add swap space; continuing without it" # Which slice of the suite this job runs. The patterns are spelled out one prefix # at a time because surefire does NOT support character-class globs: # -Dtest='Test[A-F]*' silently matches nothing and the build passes green having @@ -97,11 +100,9 @@ jobs: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 - name: Remove swap space run : | - sudo sh -c " - free -h - swapoff /tmp/swapfile - rm /tmp/swapfile - " + free -h + sudo swapoff /drill-swapfile 2>/dev/null || true + sudo rm -f /drill-swapfile checkstyle_protobuf: name: Run checkstyle and generate protobufs runs-on: ubuntu-22.04 From c2dad4fcdfd02784609308327f52b3bcc882065e Mon Sep 17 00:00:00 2001 From: cgivre Date: Wed, 21 Oct 2026 20:18:41 -0400 Subject: [PATCH 4/4] Bump Testcontainers to 1.21.4 for newer Docker Engine on CI runners --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 90d833ccd2c..d3b2091a798 100644 --- a/pom.xml +++ b/pom.xml @@ -147,7 +147,7 @@ 3.1.2 2.2.42 ${project.basedir}/target/generated-sources - 1.20.0 + 1.21.4 1.4.2 2.8.3 2.23.2