HBASE-29838 Run Hadoop Check as a GitHub Action

.github/workflows/yetus-general-check.yml

@@ -78,7 +78,7 @@ jobs:
           PLUGINS: "all,-javadoc,-jira,-shadedjars,-unit"
           SET_JAVA_HOME: "/usr/lib/jvm/java-17"
           SOURCEDIR: "${{ github.workspace }}/src"
-          TESTS_FILTER: "checkstyle,javac,javadoc,pylint,shellcheck,shelldocs,blanks,perlcritic,ruby-lint,rubocop"
+          TESTS_FILTER: "checkstyle,javac,pylint,shellcheck,shelldocs,blanks,perlcritic,ruby-lint,rubocop"
           YETUSDIR: "${{ github.workspace }}/yetus"
           AUTHOR_IGNORE_LIST: "src/main/asciidoc/_chapters/developer.adoc"
           BLANKS_EOL_IGNORE_FILE: "dev-support/blanks-eol-ignore.txt"

.github/workflows/yetus-jdk17-hadoop3-compile-check.yml

@@ -0,0 +1,106 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# yamllint disable rule:line-length
+---
+name: Yetus JDK17 Hadoop3 Compile Check
+
+"on":
+  pull_request:
+    types: [opened, synchronize, reopened]
+
+permissions:
+  contents: read
+  statuses: write
+
+jobs:
+  jdk17-hadoop3-compile-check:
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+
+    env:
+      YETUS_VERSION: '0.15.0'
+
+    steps:
+      - name: Checkout HBase
+        uses: actions/checkout@v4
+        with:
+          path: src
+          fetch-depth: 0
+
+      - name: Set up JDK 17
+        uses: actions/setup-java@v4
+        with:
+          java-version: '17'
+          distribution: 'temurin'
+
+      - name: Maven cache
+        uses: actions/cache@v4
+        with:
+          path: ~/.m2
+          key: hbase-m2-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            hbase-m2-
+
+      - name: Download Yetus
+        run: |
+          mkdir -p yetus
+          cd yetus
+          bash "${{ github.workspace }}/src/dev-support/jenkins-scripts/cache-apache-project-artifact.sh" \
+            --keys 'https://downloads.apache.org/yetus/KEYS' \
+            --verify-tar-gz \
+            ./apache-yetus-${{ env.YETUS_VERSION }}-bin.tar.gz \
+            yetus/${{ env.YETUS_VERSION }}/apache-yetus-${{ env.YETUS_VERSION }}-bin.tar.gz
+          tar --strip-components=1 -xzf apache-yetus-${{ env.YETUS_VERSION }}-bin.tar.gz
+          rm apache-yetus-${{ env.YETUS_VERSION }}-bin.tar.gz
+
+      - name: Run Yetus JDK17 Hadoop3 Compile Check
+        env:
+          ARCHIVE_PATTERN_LIST: "TEST-*.xml,org.apache.h*.txt,*.dumpstream,*.dump"
+          DOCKERFILE: "${{ github.workspace }}/src/dev-support/docker/Dockerfile"
+          GITHUB_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
+          GITHUB_USER: ${{ github.actor }}
+          PATCHDIR: "${{ github.workspace }}/yetus-jdk17-hadoop3-compile-check/output"
+          PLUGINS: "compile,github,htmlout,javac,javadoc,maven,mvninstall,shadedjars"
+          SET_JAVA_HOME: "/usr/lib/jvm/java-17"
+          SOURCEDIR: "${{ github.workspace }}/src"
+          TESTS_FILTER: "javac,javadoc"
+          YETUSDIR: "${{ github.workspace }}/yetus"
+          AUTHOR_IGNORE_LIST: "src/main/asciidoc/_chapters/developer.adoc"
+          BLANKS_EOL_IGNORE_FILE: "dev-support/blanks-eol-ignore.txt"
+          BLANKS_TABS_IGNORE_FILE: "dev-support/blanks-tabs-ignore.txt"
+          BUILD_THREAD: "4"
+          BRANCH_NAME: "${{ github.base_ref }}"
+          SKIP_ERRORPRONE: 'true'
+          DEBUG: 'true'
+        run: |
+          cd "${{ github.workspace }}"
+          bash src/dev-support/jenkins_precommit_github_yetus.sh
+
+      - name: Publish Job Summary
+        if: always()
+        run: |
+          cd "${{ github.workspace }}"
+          python3 src/dev-support/yetus_console_to_md.py yetus-jdk17-hadoop3-compile-check/output/console.txt >> $GITHUB_STEP_SUMMARY
+
+      - name: Publish Results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: yetus-jdk17-hadoop3-compile-check-output
+          path: ${{ github.workspace }}/yetus-jdk17-hadoop3-compile-check/output
+          retention-days: 7

.github/workflows/yetus-jdk17-hadoop3-unit-check.yml

@@ -0,0 +1,129 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# yamllint disable rule:line-length
+---
+name: Yetus JDK17 Hadoop3 Unit Check
+
+"on":
+  pull_request:
+    types: [opened, synchronize, reopened]
+
+permissions:
+  contents: read
+  statuses: write
+
+jobs:
+  jdk17-hadoop3-unit-check:
+    runs-on: ubuntu-latest
+    timeout-minutes: 360
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - name: "small"
+            test_profile: "runSmallTests"
+          - name: "medium"
+            test_profile: "runMediumTests"
+          # Large tests split alphabetically by class name (after "Test" prefix)
+          # Wave 1: Test[A-H]*, Wave 2: Test[I-R]*, Wave 3: Test[S-Z]*
+          - name: "large-wave-1"
+            test_profile: "runLargeTests-wave1"
+          - name: "large-wave-2"
+            test_profile: "runLargeTests-wave2"
+          - name: "large-wave-3"
+            test_profile: "runLargeTests-wave3"
+
+    name: ${{ matrix.name }}
+
+    env:
+      YETUS_VERSION: '0.15.0'
+
+    steps:
+      - name: Checkout HBase
+        uses: actions/checkout@v4
+        with:
+          path: src
+          fetch-depth: 0
+
+      - name: Set up JDK 17
+        uses: actions/setup-java@v4
+        with:
+          java-version: '17'
+          distribution: 'temurin'
+
+      - name: Maven cache
+        uses: actions/cache@v4
+        with:
+          path: ~/.m2
+          key: hbase-m2-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            hbase-m2-
+
+      - name: Download Yetus
+        run: |
+          mkdir -p yetus
+          cd yetus
+          bash "${{ github.workspace }}/src/dev-support/jenkins-scripts/cache-apache-project-artifact.sh" \
+            --keys 'https://downloads.apache.org/yetus/KEYS' \
+            --verify-tar-gz \
+            ./apache-yetus-${{ env.YETUS_VERSION }}-bin.tar.gz \
+            yetus/${{ env.YETUS_VERSION }}/apache-yetus-${{ env.YETUS_VERSION }}-bin.tar.gz
+          tar --strip-components=1 -xzf apache-yetus-${{ env.YETUS_VERSION }}-bin.tar.gz
+          rm apache-yetus-${{ env.YETUS_VERSION }}-bin.tar.gz
+
+      - name: Run Yetus JDK17 Hadoop3 Unit Check
+        env:
+          ARCHIVE_PATTERN_LIST: "TEST-*.xml,org.apache.h*.txt,*.dumpstream,*.dump"
+          DOCKERFILE: "${{ github.workspace }}/src/dev-support/docker/Dockerfile"
+          GITHUB_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
+          GITHUB_USER: ${{ github.actor }}
+          PATCHDIR: "${{ github.workspace }}/yetus-jdk17-hadoop3-unit-check/output"
+          PLUGINS: "github,htmlout,maven,unit"
+          SET_JAVA_HOME: "/usr/lib/jvm/java-17"
+          SOURCEDIR: "${{ github.workspace }}/src"
+          TESTS_FILTER: "mvninstall"
+          YETUSDIR: "${{ github.workspace }}/yetus"
+          AUTHOR_IGNORE_LIST: "src/main/asciidoc/_chapters/developer.adoc"
+          BLANKS_EOL_IGNORE_FILE: "dev-support/blanks-eol-ignore.txt"
+          BLANKS_TABS_IGNORE_FILE: "dev-support/blanks-tabs-ignore.txt"
+          EXCLUDE_TESTS_URL: "https://ci-hbase.apache.org/job/HBase-Find-Flaky-Tests/job/${{ github.base_ref }}/lastSuccessfulBuild/artifact/output/excludes"
+          BUILD_THREAD: "4"
+          SUREFIRE_FIRST_PART_FORK_COUNT: "1.0C"
+          SUREFIRE_SECOND_PART_FORK_COUNT: "0.5C"
+          BRANCH_NAME: "${{ github.base_ref }}"
+          SKIP_ERRORPRONE: 'true'
+          DEBUG: 'true'
+          TEST_PROFILE: ${{ matrix.test_profile }}
+        run: |
+          cd "${{ github.workspace }}"
+          bash src/dev-support/jenkins_precommit_github_yetus.sh
+
+      - name: Publish Job Summary
+        if: always()
+        run: |
+          cd "${{ github.workspace }}"
+          python3 src/dev-support/yetus_console_to_md.py yetus-jdk17-hadoop3-unit-check/output/console.txt >> $GITHUB_STEP_SUMMARY
+
+      - name: Publish Test Results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: yetus-jdk17-hadoop3-unit-check-${{ matrix.name }}
+          path: ${{ github.workspace }}/yetus-jdk17-hadoop3-unit-check/output
+          retention-days: 7

dev-support/hbase-personality.sh

@@ -330,6 +330,9 @@ function personality_file_tests
 ## @audience     private
 ## @stability    evolving
 ## @param        name of variable to set with maven arguments
+# NOTE: INCLUDE_TESTS_URL uses -Dtest= which conflicts with pom.xml <include> patterns.
+#       Do not use INCLUDE_TESTS_URL with profiles that define their own <include> patterns
+#       (e.g., runLargeTests-wave1, runLargeTests-wave2, runLargeTests-wave3).
 function get_include_exclude_tests_arg
 {
   local  __resultvar=$1

dev-support/jenkins_precommit_github_yetus.sh

@@ -167,6 +167,10 @@ fi
 if [[ -n "${JAVA8_HOME}" ]]; then
   YETUS_ARGS+=("--java8-home=${JAVA8_HOME}")
 fi
+# Test profile for running specific test categories (e.g., runDevTests, runLargeTests-wave1)
+if [[ -n "${TEST_PROFILE}" ]]; then
+  YETUS_ARGS+=("--test-profile=${TEST_PROFILE}")
+fi
 
 echo "Launching yetus with command line:"
 echo "${TESTPATCHBIN} ${YETUS_ARGS[*]}"

dev-support/yetus_console_to_md.py

@@ -22,7 +22,7 @@
 import re
 import sys
 from pathlib import Path
-from typing import List, Tuple
+from typing import List, Optional, Tuple
 
 
 # Vote to emoji mapping
@@ -170,6 +170,72 @@ def process_first_table(lines: List[str], start_idx: int) -> Tuple[List[str], in
     return content, i
 
 
+# TODO: Yetus should support this natively, but docker integration with job summaries doesn't seem
+#       to work out of the box.
+def extract_failed_tests_from_unit_files(output_dir: Path) -> List[Tuple[str, List[str]]]:
+    """
+    Extract failed test names from patch-unit-*.txt files.
+
+    Parses Maven surefire output to find lines like:
+    [ERROR] org.apache.hadoop.hbase.types.TestPBCell.testRoundTrip
+
+    Returns:
+        List of (module_name, [failed_test_names]) tuples
+    """
+    results = []
+
+    for unit_file in output_dir.glob('patch-unit-*.txt'):
+        module_name = unit_file.stem.replace('patch-unit-', '')
+        failed_tests = set()
+
+        with open(unit_file, 'r') as f:
+            in_failures_section = False
+            for line in f:
+                stripped = line.strip()
+
+                if stripped == '[ERROR] Failures:':
+                    in_failures_section = True
+                    continue
+
+                if in_failures_section:
+                    if stripped.startswith('[ERROR]') and not stripped.startswith('[ERROR]   Run'):
+                        test_name = stripped.replace('[ERROR] ', '').strip()
+                        if test_name and '.' in test_name:
+                            failed_tests.add(test_name)
+                    elif stripped.startswith('[INFO]') or not stripped:
+                        in_failures_section = False
+
+        if failed_tests:
+            results.append((module_name, sorted(failed_tests)))
+
+    return results
+
+
+def format_failed_tests_section(failed_tests: List[Tuple[str, List[str]]]) -> List[str]:
+    """
+    Format failed tests into markdown.
+
+    Args:
+        failed_tests: List of (module_name, [test_names]) tuples
+
+    Returns:
+        List of markdown lines
+    """
+    if not failed_tests:
+        return []
+
+    content = []
+    content.append('\n## ❌ Failed Tests\n\n')
+    content.append('| Module | Failed Tests |\n')
+    content.append('|--------|-------------|\n')
+
+    for module_name, tests in failed_tests:
+        tests_str = ', '.join(tests)
+        content.append(f'| {module_name} | {tests_str} |\n')
+
+    return content
+
+
 def process_second_table(lines: List[str], start_idx: int) -> Tuple[List[str], int]:
     """
     Process the second table (Subsystem, Report/Notes).
@@ -206,13 +272,17 @@ def process_second_table(lines: List[str], start_idx: int) -> Tuple[List[str], i
     return content, i
 
 
-def convert_console_to_markdown(input_file: str, output_file: str | None = None) -> str:
+def convert_console_to_markdown(input_file: str, output_file: Optional[str] = None) -> str:
     """Convert console to Markdown format."""
+    input_path = Path(input_file)
+    output_dir = input_path.parent
+
     with open(input_file, 'r') as f:
         lines = f.readlines()
 
     content = []
     i = 0
+    added_failed_tests = False
 
     while i < len(lines):
         line = lines[i]
@@ -233,6 +303,12 @@ def convert_console_to_markdown(input_file: str, output_file: str | None = None)
         if '| Vote |' in line and 'Subsystem' in line:
             table_content, i = process_first_table(lines, i + 1)
             content.extend(table_content)
+
+            # Extract and add failed tests from patch-unit-*.txt files
+            if not added_failed_tests:
+                failed_tests = extract_failed_tests_from_unit_files(output_dir)
+                content.extend(format_failed_tests_section(failed_tests))
+                added_failed_tests = True
             continue
 
         # Detect second table start