Run Python tests with Bazel and retain CI diagnostics

.github/workflows/main.yml

@@ -20,19 +20,16 @@ jobs:
       contents: read
     steps:
       - uses: actions/checkout@v7
-      - uses: actions/setup-python@v7
+      - uses: bazel-contrib/setup-bazel@0.19.1
         with:
-          python-version: '3.13'
-      - run: python3 -m unittest discover -s tools -p release_site_test.py
-      - run: python3 -m unittest discover -s tools -p release_notes_test.py
-      - run: python3 -m unittest discover -s tools -p site_artwork_test.py
-      - run: python3 -m unittest discover -s tools -p infrastructure_test.py
+          cache-save: false
+          bazelisk-cache: true
       - name: Verify configured documentation and generated links
         env:
           GH_TOKEN: ${{ github.token }}
         run: |
           # This is a disposable build on the runner, never a Pages publication.
-          python3 tools/release_site.py . "${RUNNER_TEMP}/release-site-check" \
+          bazel run //tools:release_site -- "${GITHUB_WORKSPACE}" "${RUNNER_TEMP}/release-site-check" \
             --repository "${GITHUB_REPOSITORY}" --tag 0.0.0-verification --latest ""
 
   pre-commit:
@@ -58,6 +55,11 @@ jobs:
     runs-on: ${{ matrix.os }}
     steps:
     - uses: actions/checkout@v7
+    - name: Configure report capture
+      shell: bash
+      run: |
+        mkdir -p "${RUNNER_TEMP}/test-results"
+        echo "BASHTEST_CI_REPORTS=${RUNNER_TEMP}/test-results" >> "${GITHUB_ENV}"
     - uses: bazel-contrib/setup-bazel@0.19.1
       with:
         cache-save: ${{ github.ref == 'refs/heads/main' }}
@@ -72,8 +74,10 @@ jobs:
       env:
         MSYS_NO_PATHCONV: "1"
       run: |
-        bazel test //...
-        bazel test --enable_runfiles=no //...
+        bazel test //... --build_event_json_file="${BASHTEST_CI_REPORTS}/root-default.json"
+        bazel run //tools:ci_reports -- collect --events "${BASHTEST_CI_REPORTS}/root-default.json" --output "${BASHTEST_CI_REPORTS}/root-default"
+        bazel test --enable_runfiles=no //... --build_event_json_file="${BASHTEST_CI_REPORTS}/root-manifest.json"
+        bazel run //tools:ci_reports -- collect --events "${BASHTEST_CI_REPORTS}/root-manifest.json" --output "${BASHTEST_CI_REPORTS}/root-manifest"
     - name: Test external module consumer
       if: runner.os != 'Windows'
       env:
@@ -88,6 +92,22 @@ jobs:
       shell: pwsh
       run: ./bashtest/tests/windows_runtime_test.ps1
 
+    - name: Summarize JUnit results
+      if: always()
+      env:
+        MSYS_NO_PATHCONV: "1"
+      shell: bash
+      run: bazel run //tools:ci_reports -- finalize "${BASHTEST_CI_REPORTS}" --destination "${BASHTEST_CI_REPORTS}" --output "${GITHUB_STEP_SUMMARY}"
+
+    - name: Retain test diagnostics
+      if: always()
+      uses: actions/upload-artifact@v7
+      with:
+        name: test-results-${{ matrix.os }}
+        path: ${{ runner.temp }}/test-results
+        if-no-files-found: warn
+        retention-days: 7
+
   done:
     permissions:
       contents: read

.pre-commit-config.yaml

@@ -33,12 +33,6 @@ repos:
 
 - repo: local
   hooks:
-  - id: infrastructure-tests
-    name: Test publishing and CI policy
-    language: system
-    entry: python3 -m unittest discover -s tools -p '*_test.py'
-    files: ^(\.pre-commit/check_version\.sh$|tools/|docs/assets/|release-site\.json$|\.github/workflows/)
-    pass_filenames: false
   - id: uncomment-bazelmod-includes
     name: uncomment-bazelmod-includes
     description: |

AGENTS.md

@@ -5,8 +5,7 @@ compatibility; [STYLE_SH.md](STYLE_SH.md) owns shell conventions;
 [GIT_RULES.md](GIT_RULES.md) owns branch and PR operations.
 [CONTRIBUTING.md](CONTRIBUTING.md) describes contributor onboarding.
 
-Run `bazel test //...`, `python3 -m unittest discover -s tools -p '*_test.py'`, and
-`pre-commit run --all-files` before proposing repository-wide changes. Add regression tests for
+Run `bazel test //...` and `pre-commit run --all-files` before proposing repository-wide changes. Add regression tests for
 behavior changes. Preserve the supported operating systems in CI and keep direct Bazel dependencies
 explicit. Keep developer-only dependencies separate from the published module where supported.
 

BUILD.bazel

@@ -14,3 +14,18 @@
 # limitations under the License.
 
 package(default_visibility = ["//visibility:private"])
+
+filegroup(
+    name = "tool_test_data",
+    srcs = glob([
+        "*.md",
+        ".github/**",
+        ".pre-commit/**",
+        "docs/**",
+    ]) + [
+        ".bazelversion",
+        "MODULE.bazel",
+        "release-site.json",
+    ],
+    visibility = ["//tools:__pkg__"],
+)

CONTRIBUTING.md

@@ -11,6 +11,15 @@ All code must adhere to the [RULES.md](RULES.md) and mostly follows the [Google
 Follow [AGENTS.md](AGENTS.md), [GIT_RULES.md](GIT_RULES.md), and [STYLE_SH.md](STYLE_SH.md).
 Run `bazel test //...` and keep behavior changes covered by regression tests.
 
+CI summaries separate regular tests, deliberately failing consumer fixtures, and builds.
+Regular test counts use the final attempt of each run/shard; earlier retries remain in the
+downloadable `test-results` artifact with the XML, logs, and build events. Expected fixture
+diagnostics are collapsed and excluded from regular failure/error counts. Their observed nonzero
+exit alone does not establish correctness: the consumer scripts check the precise failure behavior,
+and the CI step must pass. Likewise, `bazel run` build events describe the build; its CI step checks
+the program exit. Unexpected final-attempt failures/errors, command outcomes, and reporting errors
+fail summary generation.
+
 # Run pre-commit
 
 All changes will be verified by the pre-commit rules. In order to check these before committing changes install the tool:

MODULE.bazel

@@ -30,3 +30,13 @@ windows_runtime = use_repo_rule("//bashtest/runtime:repositories.bzl", "windows_
 windows_runtime(name = "bashtest_windows_runtime")
 
 register_toolchains("//bashtest/runtime:windows_toolchain")
+
+# Repository tooling and its tests do not add a runtime dependency for consumers.
+bazel_dep(name = "rules_python", version = "2.4.5", dev_dependency = True)
+
+python = use_extension("@rules_python//python/extensions:python.bzl", "python", dev_dependency = True)
+python.defaults(python_version = "3.13")
+python.toolchain(python_version = "3.13")
+
+python_config = use_extension("@rules_python//python/extensions:config.bzl", "config", dev_dependency = True)
+python_config.explicit_init_py(default = True)

bashtest/tests/BUILD.bazel

@@ -129,3 +129,15 @@ sh_test(
         "//conditions:default": [],
     }),
 )
+
+filegroup(
+    name = "consumer_scripts",
+    srcs = [
+        "ci_support.sh",
+        "external_consumer_test.sh",
+        "tool_fixture.bzl",
+        "tools_consumer_test.sh",
+        "windows_runtime_test.ps1",
+    ],
+    visibility = ["//tools:__pkg__"],
+)

bashtest/tests/ci_support.sh

@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+
+# SPDX-FileCopyrightText: Copyright (c) M. Boerger, the MBO Works authors
+# SPDX-License-Identifier: Apache-2.0
+
+# Collection only: callers execute Bazel commands directly.
+ci_collect() {
+    [[ -n "${BASHTEST_CI_REPORTS:-}" ]] || return 0
+    local events="${1}" repo name
+    repo="$(cd "${BASH_SOURCE[0]%/*}/../.." && pwd)" || return 1
+    name="${events##*/}"
+    (cd "${repo}" && bazel run //tools:ci_reports -- collect \
+        --events "${events}" --output "${BASHTEST_CI_REPORTS}/${name%.json}")
+}
+
+# Capture a failed command's results before the consumer workspace is deleted.
+ci_collect_pending() {
+    [[ -n "${BASHTEST_CI_REPORTS:-}" ]] || return 0
+    local events="${1}" logs="${2}" suite="${3}" repo log status=0
+    repo="$(cd "${BASH_SOURCE[0]%/*}/../.." && pwd)" || return 1
+    (cd "${repo}" && bazel run //tools:ci_reports -- finalize \
+        "${events}" --destination "${BASHTEST_CI_REPORTS}") || status=1
+    mkdir -p "${BASHTEST_CI_REPORTS}/consumer-logs/${suite}" || return 1
+    for log in "${logs}"/*.log; do
+        [[ -f "${log}" ]] || continue
+        cp "${log}" "${BASHTEST_CI_REPORTS}/consumer-logs/${suite}/" || status=1
+    done
+    return "${status}"
+}
+
+ci_failure_logs() {
+    local status="${1}" directory="${2}" log
+    [[ "${status}" != 0 ]] || return 0
+    for log in "${directory}"/*.log; do
+        [[ -f "${log}" ]] || continue
+        printf >&2 '\n[UNEXPECTED CONSUMER FAILURE: %s]\n' "${log##*/}"
+        cat "${log}" >&2
+    done
+}

bashtest/tests/external_consumer_test.sh

@@ -18,14 +18,27 @@
 # Run outside a Bazel test: this creates and builds a separate consumer module.
 set -euo pipefail
 repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+# shellcheck source=bashtest/tests/ci_support.sh
+source "${repo}/bashtest/tests/ci_support.sh"
 # The override consumes this checkout, including unreleased development versions.
 version="$(sed -nE 's/.*version = "([^"]+)".*/\1/p' "${repo}/MODULE.bazel" | head -n1)"
 root="$(mktemp -d)"
+events="${root}/events"
+mkdir -p "${events}/expected"
+case "${OSTYPE}" in
+    msys*|cygwin*) events="$(cygpath -am "${events}")" ;;
+esac
 cleanup() {
+    local status="${?}"
+    ci_failure_logs "${status}" "${PWD}"
+    if ! ci_collect_pending "${events}" "${PWD}" "external"; then
+        if [[ "${status}" == 0 ]]; then status=1; fi
+    fi
     # Windows cannot remove a directory while it is a process working directory.
     bazel shutdown >/dev/null 2>&1 || true
     cd "${repo}"
     rm -rf "${root}"
+    exit "${status}"
 }
 trap cleanup EXIT
 case "${OSTYPE}" in
@@ -65,11 +78,14 @@ test_runner
 EOF
 # Also run the framework's own suite as an external dependency, as BCR does.
 # A standalone consumer test alone cannot catch root-workspace assumptions in it.
-bazel test --enable_runfiles=no --test_output=errors //:consumer_test @framework//bashtest/tests/...
+bazel test --build_event_json_file="${events}/external-full.json" --enable_runfiles=no --test_output=errors //:consumer_test @framework//bashtest/tests/...
+ci_collect "${events}/external-full.json"
 # A filtered/sharded meta-test must not leak its settings into verifier fixtures.
-bazel test --enable_runfiles=no --test_filter='^junit_cases_and_failures$' \
+bazel test --build_event_json_file="${events}/external-meta-shards.json" --enable_runfiles=no --test_filter='^junit_cases_and_failures$' \
     --test_sharding_strategy=forced=5 @framework//bashtest/tests:junit_test
-bazel run --enable_runfiles=no //:consumer_test -- --keep-tmpdir=never
+ci_collect "${events}/external-meta-shards.json"
+bazel run --build_event_json_file="${events}/external-run.json" --enable_runfiles=no //:consumer_test -- --keep-tmpdir=never
+ci_collect "${events}/external-run.json"
 
 # Exercise both documented update spellings with a workspace path containing spaces.
 # Separate MSYS2 installations have different /tmp mounts. Cross the native
@@ -79,10 +95,12 @@ case "${OSTYPE}" in
     msys*|cygwin*) workspace="$(cygpath -am "${workspace}")" ;;
 esac
 printf 'stale\n' > golden.txt
-bazel run --enable_runfiles=no //:consumer_test -- --update-golden="${workspace}" --keep-tmpdir=never
+bazel run --build_event_json_file="${events}/external-update-long.json" --enable_runfiles=no //:consumer_test -- --update-golden="${workspace}" --keep-tmpdir=never
+ci_collect "${events}/external-update-long.json"
 [[ "$(cat golden.txt)" == 'hello world' ]]
 printf 'stale again\n' > golden.txt
-bazel run --enable_runfiles=no //:consumer_test -- -u="${workspace}" --keep-tmpdir=never
+bazel run --build_event_json_file="${events}/external-update-short.json" --enable_runfiles=no //:consumer_test -- -u="${workspace}" --keep-tmpdir=never
+ci_collect "${events}/external-update-short.json"
 [[ "$(cat golden.txt)" == 'hello world' ]]
 
 # Verify that Bazel collects per-function failure XML, rather than its fallback
@@ -127,10 +145,11 @@ EOF
         junit_targets+=(//:junit_sh_failure_test)
         ;;
 esac
-if bazel test --enable_runfiles=no "${junit_targets[@]}" > junit_failure.log 2>&1; then
+if bazel test --build_event_json_file="${events}/expected/external-junit-failure.json" --enable_runfiles=no "${junit_targets[@]}" > junit_failure.log 2>&1; then
     echo >&2 'ERROR: intentionally failing JUnit fixtures passed'
     exit 1
 fi
+ci_collect "${events}/expected/external-junit-failure.json"
 junit_logs="$(bazel info bazel-testlogs)"
 case "${OSTYPE}" in
     msys*|cygwin*) junit_logs="$(cygpath -u "${junit_logs}")" ;;
@@ -186,9 +205,11 @@ EOF
         selection_targets+=(//:selection_sh_test)
         ;;
 esac
-bazel test --enable_runfiles=no --test_filter='^(alpha|gamma)$' "${selection_targets[@]}"
-bazel test --enable_runfiles=no --test_filter='^(alpha|gamma)$' \
+bazel test --build_event_json_file="${events}/external-filter.json" --enable_runfiles=no --test_filter='^(alpha|gamma)$' "${selection_targets[@]}"
+ci_collect "${events}/external-filter.json"
+bazel test --build_event_json_file="${events}/external-shards.json" --enable_runfiles=no --test_filter='^(alpha|gamma)$' \
     --test_sharding_strategy=forced=5 "${selection_targets[@]}"
+ci_collect "${events}/external-shards.json"
 for target in "${selection_targets[@]}"; do
     reports=("${junit_logs}/${target#//:}"/shard_*/test.xml)
     [[ "${#reports[@]}" == 5 ]]
@@ -201,11 +222,12 @@ for target in "${selection_targets[@]}"; do
     [[ "${alpha}" == 1 && "${gamma}" == 1 && "${empty}" == 2 ]]
     echo "[SHARDING VERIFIED] ${target}: five reports, each selected case once, two empty reports"
 done
-if bazel test --enable_runfiles=no --test_runner_fail_fast --test_env=FAIL_CASE=alpha \
+if bazel test --build_event_json_file="${events}/expected/external-fail-fast.json" --enable_runfiles=no --test_runner_fail_fast --test_env=FAIL_CASE=alpha \
     "${selection_targets[@]}" > selection_failure.log 2>&1; then
     echo >&2 'ERROR: intentionally failing fail-fast fixtures passed'
     exit 1
 fi
+ci_collect "${events}/expected/external-fail-fast.json"
 for target in "${selection_targets[@]}"; do
     report="${junit_logs}/${target#//:}/test.xml"
     grep -F 'tests="3" failures="1" errors="0" skipped="2"' "${report}" >/dev/null
@@ -258,10 +280,11 @@ EOF
         interrupted_targets+=(//:interrupted_sh_test)
         ;;
 esac
-if bazel test --enable_runfiles=no --test_timeout=10 "${interrupted_targets[@]}" > interrupted.log 2>&1; then
+if bazel test --build_event_json_file="${events}/expected/external-timeout.json" --enable_runfiles=no --test_timeout=10 "${interrupted_targets[@]}" > interrupted.log 2>&1; then
     echo >&2 'ERROR: intentionally hanging fixtures passed'
     exit 1
 fi
+ci_collect "${events}/expected/external-timeout.json"
 for target in "${interrupted_targets[@]}"; do
     grep -F "${target}" interrupted.log | grep -F TIMEOUT >/dev/null
     report="${junit_logs}/${target#//:}/test.xml"

bashtest/tests/tools_consumer_test.sh

@@ -18,13 +18,26 @@
 # Integration test: distinct default and test execution platforms, external module.
 set -euo pipefail
 repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+# shellcheck source=bashtest/tests/ci_support.sh
+source "${repo}/bashtest/tests/ci_support.sh"
 # The override consumes this checkout, including unreleased development versions.
 version="$(sed -nE 's/.*version = "([^"]+)".*/\1/p' "${repo}/MODULE.bazel" | head -n1)"
 root="$(mktemp -d)"
+events="${root}/events"
+mkdir -p "${events}/expected"
+case "${OSTYPE}" in
+    msys*|cygwin*) events="$(cygpath -am "${events}")" ;;
+esac
 cleanup() {
+    local status="${?}"
+    ci_failure_logs "${status}" "${PWD}"
+    if ! ci_collect_pending "${events}" "${PWD}" "tools"; then
+        if [[ "${status}" == 0 ]]; then status=1; fi
+    fi
     bazel shutdown >/dev/null 2>&1 || true
     cd "${repo}"
     rm -rf "${root}"
+    exit "${status}"
 }
 trap cleanup EXIT
 case "${OSTYPE}" in
@@ -98,15 +111,19 @@ test::configuration() {
 test_runner
 EOF
 flags=("--extra_execution_platforms=//:ordinary_exec,//:test_exec" --enable_runfiles=no)
-bazel test "${flags[@]}" --test_output=errors //:consumer_test
-bazel run "${flags[@]}" //:consumer_test -- --keep-tmpdir=never
-if bazel build "${flags[@]}" //:invalid_tool_test >invalid.log 2>&1; then
+bazel test --build_event_json_file="${events}/tools-test.json" "${flags[@]}" --test_output=errors //:consumer_test
+ci_collect "${events}/tools-test.json"
+bazel run --build_event_json_file="${events}/tools-run.json" "${flags[@]}" //:consumer_test -- --keep-tmpdir=never
+ci_collect "${events}/tools-run.json"
+if bazel build --build_event_json_file="${events}/expected/tools-non-executable.json" "${flags[@]}" //:invalid_tool_test >invalid.log 2>&1; then
     echo >&2 'ERROR: non-executable tool was accepted'
     exit 1
 fi
+ci_collect "${events}/expected/tools-non-executable.json"
 grep -F 'tools requires an executable target' invalid.log | sed 's/^/[EXPECTED FAILURE: non-executable tool] /'
-if bazel build "${flags[@]}" //:collision_test >collision.log 2>&1; then
+if bazel build --build_event_json_file="${events}/expected/tools-collision.json" "${flags[@]}" //:collision_test >collision.log 2>&1; then
     echo >&2 'ERROR: conflicting environment variable was accepted'
     exit 1
 fi
+ci_collect "${events}/expected/tools-collision.json"
 grep -F 'tool environment variable is already defined' collision.log | sed 's/^/[EXPECTED FAILURE: environment collision] /'