From 21447cd8bb5adfdde35deb9416a8036c95b74681 Mon Sep 17 00:00:00 2001 From: Nick Dimiduk Date: Tue, 20 Jan 2026 07:57:45 +0100 Subject: [PATCH 1/2] HBASE-29838 Run Hadoop Check as a GitHub Action --- .github/workflows/yetus-general-check.yml | 2 +- .../yetus-jdk17-hadoop3-compile-check.yml | 106 ++++++++++++++ .../yetus-jdk17-hadoop3-unit-check.yml | 129 ++++++++++++++++++ dev-support/hbase-personality.sh | 3 + dev-support/jenkins_precommit_github_yetus.sh | 4 + dev-support/yetus_console_to_md.py | 80 ++++++++++- pom.xml | 85 ++++++++++++ 7 files changed, 406 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/yetus-jdk17-hadoop3-compile-check.yml create mode 100644 .github/workflows/yetus-jdk17-hadoop3-unit-check.yml diff --git a/.github/workflows/yetus-general-check.yml b/.github/workflows/yetus-general-check.yml index f77685036703..bb285a7b0692 100644 --- a/.github/workflows/yetus-general-check.yml +++ b/.github/workflows/yetus-general-check.yml @@ -78,7 +78,7 @@ jobs: PLUGINS: "all,-javadoc,-jira,-shadedjars,-unit" SET_JAVA_HOME: "/usr/lib/jvm/java-17" SOURCEDIR: "${{ github.workspace }}/src" - TESTS_FILTER: "checkstyle,javac,javadoc,pylint,shellcheck,shelldocs,blanks,perlcritic,ruby-lint,rubocop" + TESTS_FILTER: "checkstyle,javac,pylint,shellcheck,shelldocs,blanks,perlcritic,ruby-lint,rubocop" YETUSDIR: "${{ github.workspace }}/yetus" AUTHOR_IGNORE_LIST: "src/main/asciidoc/_chapters/developer.adoc" BLANKS_EOL_IGNORE_FILE: "dev-support/blanks-eol-ignore.txt" diff --git a/.github/workflows/yetus-jdk17-hadoop3-compile-check.yml b/.github/workflows/yetus-jdk17-hadoop3-compile-check.yml new file mode 100644 index 000000000000..8526943c3339 --- /dev/null +++ b/.github/workflows/yetus-jdk17-hadoop3-compile-check.yml @@ -0,0 +1,106 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# yamllint disable rule:line-length +--- +name: Yetus JDK17 Hadoop3 Compile Check + +"on": + pull_request: + types: [opened, synchronize, reopened] + +permissions: + contents: read + statuses: write + +jobs: + jdk17-hadoop3-compile-check: + runs-on: ubuntu-latest + timeout-minutes: 60 + + env: + YETUS_VERSION: '0.15.0' + + steps: + - name: Checkout HBase + uses: actions/checkout@v4 + with: + path: src + fetch-depth: 0 + + - name: Set up JDK 17 + uses: actions/setup-java@v4 + with: + java-version: '17' + distribution: 'temurin' + + - name: Maven cache + uses: actions/cache@v4 + with: + path: ~/.m2 + key: hbase-m2-${{ hashFiles('**/pom.xml') }} + restore-keys: | + hbase-m2- + + - name: Download Yetus + run: | + mkdir -p yetus + cd yetus + bash "${{ github.workspace }}/src/dev-support/jenkins-scripts/cache-apache-project-artifact.sh" \ + --keys 'https://downloads.apache.org/yetus/KEYS' \ + --verify-tar-gz \ + ./apache-yetus-${{ env.YETUS_VERSION }}-bin.tar.gz \ + yetus/${{ env.YETUS_VERSION }}/apache-yetus-${{ env.YETUS_VERSION }}-bin.tar.gz + tar --strip-components=1 -xzf apache-yetus-${{ env.YETUS_VERSION }}-bin.tar.gz + rm apache-yetus-${{ env.YETUS_VERSION }}-bin.tar.gz + + - name: Run Yetus JDK17 Hadoop3 Compile Check + env: + ARCHIVE_PATTERN_LIST: "TEST-*.xml,org.apache.h*.txt,*.dumpstream,*.dump" + DOCKERFILE: "${{ github.workspace }}/src/dev-support/docker/Dockerfile" + GITHUB_PASSWORD: ${{ secrets.GITHUB_TOKEN }} + GITHUB_USER: ${{ github.actor }} + PATCHDIR: "${{ github.workspace }}/yetus-jdk17-hadoop3-compile-check/output" + PLUGINS: "compile,github,htmlout,javac,javadoc,maven,mvninstall,shadedjars" + SET_JAVA_HOME: "/usr/lib/jvm/java-17" + SOURCEDIR: "${{ github.workspace }}/src" + TESTS_FILTER: "javac,javadoc" + YETUSDIR: "${{ github.workspace }}/yetus" + AUTHOR_IGNORE_LIST: "src/main/asciidoc/_chapters/developer.adoc" + BLANKS_EOL_IGNORE_FILE: "dev-support/blanks-eol-ignore.txt" + BLANKS_TABS_IGNORE_FILE: "dev-support/blanks-tabs-ignore.txt" + BUILD_THREAD: "4" + BRANCH_NAME: "${{ github.base_ref }}" + SKIP_ERRORPRONE: 'true' + DEBUG: 'true' + run: | + cd "${{ github.workspace }}" + bash src/dev-support/jenkins_precommit_github_yetus.sh + + - name: Publish Job Summary + if: always() + run: | + cd "${{ github.workspace }}" + python3 src/dev-support/yetus_console_to_md.py yetus-jdk17-hadoop3-compile-check/output/console.txt >> $GITHUB_STEP_SUMMARY + + - name: Publish Results + if: always() + uses: actions/upload-artifact@v4 + with: + name: yetus-jdk17-hadoop3-compile-check-output + path: ${{ github.workspace }}/yetus-jdk17-hadoop3-compile-check/output + retention-days: 7 diff --git a/.github/workflows/yetus-jdk17-hadoop3-unit-check.yml b/.github/workflows/yetus-jdk17-hadoop3-unit-check.yml new file mode 100644 index 000000000000..b4cc992b9a08 --- /dev/null +++ b/.github/workflows/yetus-jdk17-hadoop3-unit-check.yml @@ -0,0 +1,129 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# yamllint disable rule:line-length +--- +name: Yetus JDK17 Hadoop3 Unit Check + +"on": + pull_request: + types: [opened, synchronize, reopened] + +permissions: + contents: read + statuses: write + +jobs: + jdk17-hadoop3-unit-check: + runs-on: ubuntu-latest + timeout-minutes: 360 + + strategy: + fail-fast: false + matrix: + include: + - name: "small" + test_profile: "runSmallTests" + - name: "medium" + test_profile: "runMediumTests" + # Large tests split alphabetically by class name (after "Test" prefix) + # Wave 1: Test[A-H]*, Wave 2: Test[I-R]*, Wave 3: Test[S-Z]* + - name: "large-wave-1" + test_profile: "runLargeTests-wave1" + - name: "large-wave-2" + test_profile: "runLargeTests-wave2" + - name: "large-wave-3" + test_profile: "runLargeTests-wave3" + + name: ${{ matrix.name }} + + env: + YETUS_VERSION: '0.15.0' + + steps: + - name: Checkout HBase + uses: actions/checkout@v4 + with: + path: src + fetch-depth: 0 + + - name: Set up JDK 17 + uses: actions/setup-java@v4 + with: + java-version: '17' + distribution: 'temurin' + + - name: Maven cache + uses: actions/cache@v4 + with: + path: ~/.m2 + key: hbase-m2-${{ hashFiles('**/pom.xml') }} + restore-keys: | + hbase-m2- + + - name: Download Yetus + run: | + mkdir -p yetus + cd yetus + bash "${{ github.workspace }}/src/dev-support/jenkins-scripts/cache-apache-project-artifact.sh" \ + --keys 'https://downloads.apache.org/yetus/KEYS' \ + --verify-tar-gz \ + ./apache-yetus-${{ env.YETUS_VERSION }}-bin.tar.gz \ + yetus/${{ env.YETUS_VERSION }}/apache-yetus-${{ env.YETUS_VERSION }}-bin.tar.gz + tar --strip-components=1 -xzf apache-yetus-${{ env.YETUS_VERSION }}-bin.tar.gz + rm apache-yetus-${{ env.YETUS_VERSION }}-bin.tar.gz + + - name: Run Yetus JDK17 Hadoop3 Unit Check + env: + ARCHIVE_PATTERN_LIST: "TEST-*.xml,org.apache.h*.txt,*.dumpstream,*.dump" + DOCKERFILE: "${{ github.workspace }}/src/dev-support/docker/Dockerfile" + GITHUB_PASSWORD: ${{ secrets.GITHUB_TOKEN }} + GITHUB_USER: ${{ github.actor }} + PATCHDIR: "${{ github.workspace }}/yetus-jdk17-hadoop3-unit-check/output" + PLUGINS: "github,htmlout,maven,unit" + SET_JAVA_HOME: "/usr/lib/jvm/java-17" + SOURCEDIR: "${{ github.workspace }}/src" + TESTS_FILTER: "mvninstall" + YETUSDIR: "${{ github.workspace }}/yetus" + AUTHOR_IGNORE_LIST: "src/main/asciidoc/_chapters/developer.adoc" + BLANKS_EOL_IGNORE_FILE: "dev-support/blanks-eol-ignore.txt" + BLANKS_TABS_IGNORE_FILE: "dev-support/blanks-tabs-ignore.txt" + EXCLUDE_TESTS_URL: "https://ci-hbase.apache.org/job/HBase-Find-Flaky-Tests/job/${{ github.base_ref }}/lastSuccessfulBuild/artifact/output/excludes" + BUILD_THREAD: "4" + SUREFIRE_FIRST_PART_FORK_COUNT: "1.0C" + SUREFIRE_SECOND_PART_FORK_COUNT: "0.5C" + BRANCH_NAME: "${{ github.base_ref }}" + SKIP_ERRORPRONE: 'true' + DEBUG: 'true' + TEST_PROFILE: ${{ matrix.test_profile }} + run: | + cd "${{ github.workspace }}" + bash src/dev-support/jenkins_precommit_github_yetus.sh + + - name: Publish Job Summary + if: always() + run: | + cd "${{ github.workspace }}" + python3 src/dev-support/yetus_console_to_md.py yetus-jdk17-hadoop3-unit-check/output/console.txt >> $GITHUB_STEP_SUMMARY + + - name: Publish Test Results + if: always() + uses: actions/upload-artifact@v4 + with: + name: yetus-jdk17-hadoop3-unit-check-${{ matrix.name }} + path: ${{ github.workspace }}/yetus-jdk17-hadoop3-unit-check/output + retention-days: 7 diff --git a/dev-support/hbase-personality.sh b/dev-support/hbase-personality.sh index 5900bab072e0..1bfd04d9ac47 100755 --- a/dev-support/hbase-personality.sh +++ b/dev-support/hbase-personality.sh @@ -330,6 +330,9 @@ function personality_file_tests ## @audience private ## @stability evolving ## @param name of variable to set with maven arguments +# NOTE: INCLUDE_TESTS_URL uses -Dtest= which conflicts with pom.xml patterns. +# Do not use INCLUDE_TESTS_URL with profiles that define their own patterns +# (e.g., runLargeTests-wave1, runLargeTests-wave2, runLargeTests-wave3). function get_include_exclude_tests_arg { local __resultvar=$1 diff --git a/dev-support/jenkins_precommit_github_yetus.sh b/dev-support/jenkins_precommit_github_yetus.sh index c447dee3b5f8..59d4cf2b82c3 100755 --- a/dev-support/jenkins_precommit_github_yetus.sh +++ b/dev-support/jenkins_precommit_github_yetus.sh @@ -167,6 +167,10 @@ fi if [[ -n "${JAVA8_HOME}" ]]; then YETUS_ARGS+=("--java8-home=${JAVA8_HOME}") fi +# Test profile for running specific test categories (e.g., runDevTests, runLargeTests-wave1) +if [[ -n "${TEST_PROFILE}" ]]; then + YETUS_ARGS+=("--test-profile=${TEST_PROFILE}") +fi echo "Launching yetus with command line:" echo "${TESTPATCHBIN} ${YETUS_ARGS[*]}" diff --git a/dev-support/yetus_console_to_md.py b/dev-support/yetus_console_to_md.py index 3f32995e545c..c85806b3cf7e 100644 --- a/dev-support/yetus_console_to_md.py +++ b/dev-support/yetus_console_to_md.py @@ -22,7 +22,7 @@ import re import sys from pathlib import Path -from typing import List, Tuple +from typing import List, Optional, Tuple # Vote to emoji mapping @@ -170,6 +170,72 @@ def process_first_table(lines: List[str], start_idx: int) -> Tuple[List[str], in return content, i +# TODO: Yetus should support this natively, but docker integration with job summaries doesn't seem +# to work out of the box. +def extract_failed_tests_from_unit_files(output_dir: Path) -> List[Tuple[str, List[str]]]: + """ + Extract failed test names from patch-unit-*.txt files. + + Parses Maven surefire output to find lines like: + [ERROR] org.apache.hadoop.hbase.types.TestPBCell.testRoundTrip + + Returns: + List of (module_name, [failed_test_names]) tuples + """ + results = [] + + for unit_file in output_dir.glob('patch-unit-*.txt'): + module_name = unit_file.stem.replace('patch-unit-', '') + failed_tests = set() + + with open(unit_file, 'r') as f: + in_failures_section = False + for line in f: + stripped = line.strip() + + if stripped == '[ERROR] Failures:': + in_failures_section = True + continue + + if in_failures_section: + if stripped.startswith('[ERROR]') and not stripped.startswith('[ERROR] Run'): + test_name = stripped.replace('[ERROR] ', '').strip() + if test_name and '.' in test_name: + failed_tests.add(test_name) + elif stripped.startswith('[INFO]') or not stripped: + in_failures_section = False + + if failed_tests: + results.append((module_name, sorted(failed_tests))) + + return results + + +def format_failed_tests_section(failed_tests: List[Tuple[str, List[str]]]) -> List[str]: + """ + Format failed tests into markdown. + + Args: + failed_tests: List of (module_name, [test_names]) tuples + + Returns: + List of markdown lines + """ + if not failed_tests: + return [] + + content = [] + content.append('\n## ❌ Failed Tests\n\n') + content.append('| Module | Failed Tests |\n') + content.append('|--------|-------------|\n') + + for module_name, tests in failed_tests: + tests_str = ', '.join(tests) + content.append(f'| {module_name} | {tests_str} |\n') + + return content + + def process_second_table(lines: List[str], start_idx: int) -> Tuple[List[str], int]: """ Process the second table (Subsystem, Report/Notes). @@ -206,13 +272,17 @@ def process_second_table(lines: List[str], start_idx: int) -> Tuple[List[str], i return content, i -def convert_console_to_markdown(input_file: str, output_file: str | None = None) -> str: +def convert_console_to_markdown(input_file: str, output_file: Optional[str] = None) -> str: """Convert console to Markdown format.""" + input_path = Path(input_file) + output_dir = input_path.parent + with open(input_file, 'r') as f: lines = f.readlines() content = [] i = 0 + added_failed_tests = False while i < len(lines): line = lines[i] @@ -233,6 +303,12 @@ def convert_console_to_markdown(input_file: str, output_file: str | None = None) if '| Vote |' in line and 'Subsystem' in line: table_content, i = process_first_table(lines, i + 1) content.extend(table_content) + + # Extract and add failed tests from patch-unit-*.txt files + if not added_failed_tests: + failed_tests = extract_failed_tests_from_unit_files(output_dir) + content.extend(format_failed_tests_section(failed_tests)) + added_failed_tests = True continue # Detect second table start diff --git a/pom.xml b/pom.xml index d6f182377459..9196871e86ff 100644 --- a/pom.xml +++ b/pom.xml @@ -4610,6 +4610,91 @@ + + + runLargeTests-wave1 + + false + + + false + true + org.apache.hadoop.hbase.testclassification.LargeTests + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + %regex[(?i).*/test[a-h].*] + + + + + + + + runLargeTests-wave2 + + false + + + false + true + org.apache.hadoop.hbase.testclassification.LargeTests + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + %regex[(?i).*/test[i-r].*] + + + + + + + + runLargeTests-wave3 + + false + + + false + true + org.apache.hadoop.hbase.testclassification.LargeTests + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + %regex[(?i).*/test[s-z].*] + + + + + + runDevTests From add80960219736e24a6d76c89bc51e687b815caa Mon Sep 17 00:00:00 2001 From: Nick Dimiduk Date: Fri, 23 Jan 2026 17:27:47 +0100 Subject: [PATCH 2/2] force test whole project (do not merge) --- pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/pom.xml b/pom.xml index 9196871e86ff..5a3d1c97d49e 100644 --- a/pom.xml +++ b/pom.xml @@ -28,6 +28,7 @@ https://issues.apache.org/jira/browse/HBASE-6795. --> + 4.0.0 org.apache