diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 3e3597f..96b92b4 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -20,19 +20,16 @@ jobs:
contents: read
steps:
- uses: actions/checkout@v7
- - uses: actions/setup-python@v7
+ - uses: bazel-contrib/setup-bazel@0.19.1
with:
- python-version: '3.13'
- - run: python3 -m unittest discover -s tools -p release_site_test.py
- - run: python3 -m unittest discover -s tools -p release_notes_test.py
- - run: python3 -m unittest discover -s tools -p site_artwork_test.py
- - run: python3 -m unittest discover -s tools -p infrastructure_test.py
+ cache-save: false
+ bazelisk-cache: true
- name: Verify configured documentation and generated links
env:
GH_TOKEN: ${{ github.token }}
run: |
# This is a disposable build on the runner, never a Pages publication.
- python3 tools/release_site.py . "${RUNNER_TEMP}/release-site-check" \
+ bazel run //tools:release_site -- "${GITHUB_WORKSPACE}" "${RUNNER_TEMP}/release-site-check" \
--repository "${GITHUB_REPOSITORY}" --tag 0.0.0-verification --latest ""
pre-commit:
@@ -58,6 +55,11 @@ jobs:
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v7
+ - name: Configure report capture
+ shell: bash
+ run: |
+ mkdir -p "${RUNNER_TEMP}/test-results"
+ echo "BASHTEST_CI_REPORTS=${RUNNER_TEMP}/test-results" >> "${GITHUB_ENV}"
- uses: bazel-contrib/setup-bazel@0.19.1
with:
cache-save: ${{ github.ref == 'refs/heads/main' }}
@@ -72,8 +74,10 @@ jobs:
env:
MSYS_NO_PATHCONV: "1"
run: |
- bazel test //...
- bazel test --enable_runfiles=no //...
+ bazel test //... --build_event_json_file="${BASHTEST_CI_REPORTS}/root-default.json"
+ bazel run //tools:ci_reports -- collect --events "${BASHTEST_CI_REPORTS}/root-default.json" --output "${BASHTEST_CI_REPORTS}/root-default"
+ bazel test --enable_runfiles=no //... --build_event_json_file="${BASHTEST_CI_REPORTS}/root-manifest.json"
+ bazel run //tools:ci_reports -- collect --events "${BASHTEST_CI_REPORTS}/root-manifest.json" --output "${BASHTEST_CI_REPORTS}/root-manifest"
- name: Test external module consumer
if: runner.os != 'Windows'
env:
@@ -88,6 +92,22 @@ jobs:
shell: pwsh
run: ./bashtest/tests/windows_runtime_test.ps1
+ - name: Summarize JUnit results
+ if: always()
+ env:
+ MSYS_NO_PATHCONV: "1"
+ shell: bash
+ run: bazel run //tools:ci_reports -- finalize "${BASHTEST_CI_REPORTS}" --destination "${BASHTEST_CI_REPORTS}" --output "${GITHUB_STEP_SUMMARY}"
+
+ - name: Retain test diagnostics
+ if: always()
+ uses: actions/upload-artifact@v7
+ with:
+ name: test-results-${{ matrix.os }}
+ path: ${{ runner.temp }}/test-results
+ if-no-files-found: warn
+ retention-days: 7
+
done:
permissions:
contents: read
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 59b9b76..262f137 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -33,12 +33,6 @@ repos:
- repo: local
hooks:
- - id: infrastructure-tests
- name: Test publishing and CI policy
- language: system
- entry: python3 -m unittest discover -s tools -p '*_test.py'
- files: ^(\.pre-commit/check_version\.sh$|tools/|docs/assets/|release-site\.json$|\.github/workflows/)
- pass_filenames: false
- id: uncomment-bazelmod-includes
name: uncomment-bazelmod-includes
description: |
diff --git a/AGENTS.md b/AGENTS.md
index 989289a..df6b9b0 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -5,8 +5,7 @@ compatibility; [STYLE_SH.md](STYLE_SH.md) owns shell conventions;
[GIT_RULES.md](GIT_RULES.md) owns branch and PR operations.
[CONTRIBUTING.md](CONTRIBUTING.md) describes contributor onboarding.
-Run `bazel test //...`, `python3 -m unittest discover -s tools -p '*_test.py'`, and
-`pre-commit run --all-files` before proposing repository-wide changes. Add regression tests for
+Run `bazel test //...` and `pre-commit run --all-files` before proposing repository-wide changes. Add regression tests for
behavior changes. Preserve the supported operating systems in CI and keep direct Bazel dependencies
explicit. Keep developer-only dependencies separate from the published module where supported.
diff --git a/BUILD.bazel b/BUILD.bazel
index 4f4d823..24eb373 100644
--- a/BUILD.bazel
+++ b/BUILD.bazel
@@ -14,3 +14,18 @@
# limitations under the License.
package(default_visibility = ["//visibility:private"])
+
+filegroup(
+ name = "tool_test_data",
+ srcs = glob([
+ "*.md",
+ ".github/**",
+ ".pre-commit/**",
+ "docs/**",
+ ]) + [
+ ".bazelversion",
+ "MODULE.bazel",
+ "release-site.json",
+ ],
+ visibility = ["//tools:__pkg__"],
+)
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index c1182db..184df02 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -11,6 +11,15 @@ All code must adhere to the [RULES.md](RULES.md) and mostly follows the [Google
Follow [AGENTS.md](AGENTS.md), [GIT_RULES.md](GIT_RULES.md), and [STYLE_SH.md](STYLE_SH.md).
Run `bazel test //...` and keep behavior changes covered by regression tests.
+CI summaries separate regular tests, deliberately failing consumer fixtures, and builds.
+Regular test counts use the final attempt of each run/shard; earlier retries remain in the
+downloadable `test-results` artifact with the XML, logs, and build events. Expected fixture
+diagnostics are collapsed and excluded from regular failure/error counts. Their observed nonzero
+exit alone does not establish correctness: the consumer scripts check the precise failure behavior,
+and the CI step must pass. Likewise, `bazel run` build events describe the build; its CI step checks
+the program exit. Unexpected final-attempt failures/errors, command outcomes, and reporting errors
+fail summary generation.
+
# Run pre-commit
All changes will be verified by the pre-commit rules. In order to check these before committing changes install the tool:
diff --git a/MODULE.bazel b/MODULE.bazel
index ebfea69..5a03b6f 100644
--- a/MODULE.bazel
+++ b/MODULE.bazel
@@ -30,3 +30,13 @@ windows_runtime = use_repo_rule("//bashtest/runtime:repositories.bzl", "windows_
windows_runtime(name = "bashtest_windows_runtime")
register_toolchains("//bashtest/runtime:windows_toolchain")
+
+# Repository tooling and its tests do not add a runtime dependency for consumers.
+bazel_dep(name = "rules_python", version = "2.4.5", dev_dependency = True)
+
+python = use_extension("@rules_python//python/extensions:python.bzl", "python", dev_dependency = True)
+python.defaults(python_version = "3.13")
+python.toolchain(python_version = "3.13")
+
+python_config = use_extension("@rules_python//python/extensions:config.bzl", "config", dev_dependency = True)
+python_config.explicit_init_py(default = True)
diff --git a/bashtest/tests/BUILD.bazel b/bashtest/tests/BUILD.bazel
index b859554..c77a60f 100644
--- a/bashtest/tests/BUILD.bazel
+++ b/bashtest/tests/BUILD.bazel
@@ -129,3 +129,15 @@ sh_test(
"//conditions:default": [],
}),
)
+
+filegroup(
+ name = "consumer_scripts",
+ srcs = [
+ "ci_support.sh",
+ "external_consumer_test.sh",
+ "tool_fixture.bzl",
+ "tools_consumer_test.sh",
+ "windows_runtime_test.ps1",
+ ],
+ visibility = ["//tools:__pkg__"],
+)
diff --git a/bashtest/tests/ci_support.sh b/bashtest/tests/ci_support.sh
new file mode 100644
index 0000000..b755dd9
--- /dev/null
+++ b/bashtest/tests/ci_support.sh
@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+
+# SPDX-FileCopyrightText: Copyright (c) M. Boerger, the MBO Works authors
+# SPDX-License-Identifier: Apache-2.0
+
+# Collection only: callers execute Bazel commands directly.
+ci_collect() {
+ [[ -n "${BASHTEST_CI_REPORTS:-}" ]] || return 0
+ local events="${1}" repo name
+ repo="$(cd "${BASH_SOURCE[0]%/*}/../.." && pwd)" || return 1
+ name="${events##*/}"
+ (cd "${repo}" && bazel run //tools:ci_reports -- collect \
+ --events "${events}" --output "${BASHTEST_CI_REPORTS}/${name%.json}")
+}
+
+# Capture a failed command's results before the consumer workspace is deleted.
+ci_collect_pending() {
+ [[ -n "${BASHTEST_CI_REPORTS:-}" ]] || return 0
+ local events="${1}" logs="${2}" suite="${3}" repo log status=0
+ repo="$(cd "${BASH_SOURCE[0]%/*}/../.." && pwd)" || return 1
+ (cd "${repo}" && bazel run //tools:ci_reports -- finalize \
+ "${events}" --destination "${BASHTEST_CI_REPORTS}") || status=1
+ mkdir -p "${BASHTEST_CI_REPORTS}/consumer-logs/${suite}" || return 1
+ for log in "${logs}"/*.log; do
+ [[ -f "${log}" ]] || continue
+ cp "${log}" "${BASHTEST_CI_REPORTS}/consumer-logs/${suite}/" || status=1
+ done
+ return "${status}"
+}
+
+ci_failure_logs() {
+ local status="${1}" directory="${2}" log
+ [[ "${status}" != 0 ]] || return 0
+ for log in "${directory}"/*.log; do
+ [[ -f "${log}" ]] || continue
+ printf >&2 '\n[UNEXPECTED CONSUMER FAILURE: %s]\n' "${log##*/}"
+ cat "${log}" >&2
+ done
+}
diff --git a/bashtest/tests/external_consumer_test.sh b/bashtest/tests/external_consumer_test.sh
index 96ea552..8a2a675 100755
--- a/bashtest/tests/external_consumer_test.sh
+++ b/bashtest/tests/external_consumer_test.sh
@@ -18,14 +18,27 @@
# Run outside a Bazel test: this creates and builds a separate consumer module.
set -euo pipefail
repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+# shellcheck source=bashtest/tests/ci_support.sh
+source "${repo}/bashtest/tests/ci_support.sh"
# The override consumes this checkout, including unreleased development versions.
version="$(sed -nE 's/.*version = "([^"]+)".*/\1/p' "${repo}/MODULE.bazel" | head -n1)"
root="$(mktemp -d)"
+events="${root}/events"
+mkdir -p "${events}/expected"
+case "${OSTYPE}" in
+ msys*|cygwin*) events="$(cygpath -am "${events}")" ;;
+esac
cleanup() {
+ local status="${?}"
+ ci_failure_logs "${status}" "${PWD}"
+ if ! ci_collect_pending "${events}" "${PWD}" "external"; then
+ if [[ "${status}" == 0 ]]; then status=1; fi
+ fi
# Windows cannot remove a directory while it is a process working directory.
bazel shutdown >/dev/null 2>&1 || true
cd "${repo}"
rm -rf "${root}"
+ exit "${status}"
}
trap cleanup EXIT
case "${OSTYPE}" in
@@ -65,11 +78,14 @@ test_runner
EOF
# Also run the framework's own suite as an external dependency, as BCR does.
# A standalone consumer test alone cannot catch root-workspace assumptions in it.
-bazel test --enable_runfiles=no --test_output=errors //:consumer_test @framework//bashtest/tests/...
+bazel test --build_event_json_file="${events}/external-full.json" --enable_runfiles=no --test_output=errors //:consumer_test @framework//bashtest/tests/...
+ci_collect "${events}/external-full.json"
# A filtered/sharded meta-test must not leak its settings into verifier fixtures.
-bazel test --enable_runfiles=no --test_filter='^junit_cases_and_failures$' \
+bazel test --build_event_json_file="${events}/external-meta-shards.json" --enable_runfiles=no --test_filter='^junit_cases_and_failures$' \
--test_sharding_strategy=forced=5 @framework//bashtest/tests:junit_test
-bazel run --enable_runfiles=no //:consumer_test -- --keep-tmpdir=never
+ci_collect "${events}/external-meta-shards.json"
+bazel run --build_event_json_file="${events}/external-run.json" --enable_runfiles=no //:consumer_test -- --keep-tmpdir=never
+ci_collect "${events}/external-run.json"
# Exercise both documented update spellings with a workspace path containing spaces.
# Separate MSYS2 installations have different /tmp mounts. Cross the native
@@ -79,10 +95,12 @@ case "${OSTYPE}" in
msys*|cygwin*) workspace="$(cygpath -am "${workspace}")" ;;
esac
printf 'stale\n' > golden.txt
-bazel run --enable_runfiles=no //:consumer_test -- --update-golden="${workspace}" --keep-tmpdir=never
+bazel run --build_event_json_file="${events}/external-update-long.json" --enable_runfiles=no //:consumer_test -- --update-golden="${workspace}" --keep-tmpdir=never
+ci_collect "${events}/external-update-long.json"
[[ "$(cat golden.txt)" == 'hello world' ]]
printf 'stale again\n' > golden.txt
-bazel run --enable_runfiles=no //:consumer_test -- -u="${workspace}" --keep-tmpdir=never
+bazel run --build_event_json_file="${events}/external-update-short.json" --enable_runfiles=no //:consumer_test -- -u="${workspace}" --keep-tmpdir=never
+ci_collect "${events}/external-update-short.json"
[[ "$(cat golden.txt)" == 'hello world' ]]
# Verify that Bazel collects per-function failure XML, rather than its fallback
@@ -127,10 +145,11 @@ EOF
junit_targets+=(//:junit_sh_failure_test)
;;
esac
-if bazel test --enable_runfiles=no "${junit_targets[@]}" > junit_failure.log 2>&1; then
+if bazel test --build_event_json_file="${events}/expected/external-junit-failure.json" --enable_runfiles=no "${junit_targets[@]}" > junit_failure.log 2>&1; then
echo >&2 'ERROR: intentionally failing JUnit fixtures passed'
exit 1
fi
+ci_collect "${events}/expected/external-junit-failure.json"
junit_logs="$(bazel info bazel-testlogs)"
case "${OSTYPE}" in
msys*|cygwin*) junit_logs="$(cygpath -u "${junit_logs}")" ;;
@@ -186,9 +205,11 @@ EOF
selection_targets+=(//:selection_sh_test)
;;
esac
-bazel test --enable_runfiles=no --test_filter='^(alpha|gamma)$' "${selection_targets[@]}"
-bazel test --enable_runfiles=no --test_filter='^(alpha|gamma)$' \
+bazel test --build_event_json_file="${events}/external-filter.json" --enable_runfiles=no --test_filter='^(alpha|gamma)$' "${selection_targets[@]}"
+ci_collect "${events}/external-filter.json"
+bazel test --build_event_json_file="${events}/external-shards.json" --enable_runfiles=no --test_filter='^(alpha|gamma)$' \
--test_sharding_strategy=forced=5 "${selection_targets[@]}"
+ci_collect "${events}/external-shards.json"
for target in "${selection_targets[@]}"; do
reports=("${junit_logs}/${target#//:}"/shard_*/test.xml)
[[ "${#reports[@]}" == 5 ]]
@@ -201,11 +222,12 @@ for target in "${selection_targets[@]}"; do
[[ "${alpha}" == 1 && "${gamma}" == 1 && "${empty}" == 2 ]]
echo "[SHARDING VERIFIED] ${target}: five reports, each selected case once, two empty reports"
done
-if bazel test --enable_runfiles=no --test_runner_fail_fast --test_env=FAIL_CASE=alpha \
+if bazel test --build_event_json_file="${events}/expected/external-fail-fast.json" --enable_runfiles=no --test_runner_fail_fast --test_env=FAIL_CASE=alpha \
"${selection_targets[@]}" > selection_failure.log 2>&1; then
echo >&2 'ERROR: intentionally failing fail-fast fixtures passed'
exit 1
fi
+ci_collect "${events}/expected/external-fail-fast.json"
for target in "${selection_targets[@]}"; do
report="${junit_logs}/${target#//:}/test.xml"
grep -F 'tests="3" failures="1" errors="0" skipped="2"' "${report}" >/dev/null
@@ -258,10 +280,11 @@ EOF
interrupted_targets+=(//:interrupted_sh_test)
;;
esac
-if bazel test --enable_runfiles=no --test_timeout=10 "${interrupted_targets[@]}" > interrupted.log 2>&1; then
+if bazel test --build_event_json_file="${events}/expected/external-timeout.json" --enable_runfiles=no --test_timeout=10 "${interrupted_targets[@]}" > interrupted.log 2>&1; then
echo >&2 'ERROR: intentionally hanging fixtures passed'
exit 1
fi
+ci_collect "${events}/expected/external-timeout.json"
for target in "${interrupted_targets[@]}"; do
grep -F "${target}" interrupted.log | grep -F TIMEOUT >/dev/null
report="${junit_logs}/${target#//:}/test.xml"
diff --git a/bashtest/tests/tools_consumer_test.sh b/bashtest/tests/tools_consumer_test.sh
index 7a0b3ea..0cf8043 100755
--- a/bashtest/tests/tools_consumer_test.sh
+++ b/bashtest/tests/tools_consumer_test.sh
@@ -18,13 +18,26 @@
# Integration test: distinct default and test execution platforms, external module.
set -euo pipefail
repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+# shellcheck source=bashtest/tests/ci_support.sh
+source "${repo}/bashtest/tests/ci_support.sh"
# The override consumes this checkout, including unreleased development versions.
version="$(sed -nE 's/.*version = "([^"]+)".*/\1/p' "${repo}/MODULE.bazel" | head -n1)"
root="$(mktemp -d)"
+events="${root}/events"
+mkdir -p "${events}/expected"
+case "${OSTYPE}" in
+ msys*|cygwin*) events="$(cygpath -am "${events}")" ;;
+esac
cleanup() {
+ local status="${?}"
+ ci_failure_logs "${status}" "${PWD}"
+ if ! ci_collect_pending "${events}" "${PWD}" "tools"; then
+ if [[ "${status}" == 0 ]]; then status=1; fi
+ fi
bazel shutdown >/dev/null 2>&1 || true
cd "${repo}"
rm -rf "${root}"
+ exit "${status}"
}
trap cleanup EXIT
case "${OSTYPE}" in
@@ -98,15 +111,19 @@ test::configuration() {
test_runner
EOF
flags=("--extra_execution_platforms=//:ordinary_exec,//:test_exec" --enable_runfiles=no)
-bazel test "${flags[@]}" --test_output=errors //:consumer_test
-bazel run "${flags[@]}" //:consumer_test -- --keep-tmpdir=never
-if bazel build "${flags[@]}" //:invalid_tool_test >invalid.log 2>&1; then
+bazel test --build_event_json_file="${events}/tools-test.json" "${flags[@]}" --test_output=errors //:consumer_test
+ci_collect "${events}/tools-test.json"
+bazel run --build_event_json_file="${events}/tools-run.json" "${flags[@]}" //:consumer_test -- --keep-tmpdir=never
+ci_collect "${events}/tools-run.json"
+if bazel build --build_event_json_file="${events}/expected/tools-non-executable.json" "${flags[@]}" //:invalid_tool_test >invalid.log 2>&1; then
echo >&2 'ERROR: non-executable tool was accepted'
exit 1
fi
+ci_collect "${events}/expected/tools-non-executable.json"
grep -F 'tools requires an executable target' invalid.log | sed 's/^/[EXPECTED FAILURE: non-executable tool] /'
-if bazel build "${flags[@]}" //:collision_test >collision.log 2>&1; then
+if bazel build --build_event_json_file="${events}/expected/tools-collision.json" "${flags[@]}" //:collision_test >collision.log 2>&1; then
echo >&2 'ERROR: conflicting environment variable was accepted'
exit 1
fi
+ci_collect "${events}/expected/tools-collision.json"
grep -F 'tool environment variable is already defined' collision.log | sed 's/^/[EXPECTED FAILURE: environment collision] /'
diff --git a/bashtest/tests/windows_runtime_test.ps1 b/bashtest/tests/windows_runtime_test.ps1
index b42b017..1ffb39e 100644
--- a/bashtest/tests/windows_runtime_test.ps1
+++ b/bashtest/tests/windows_runtime_test.ps1
@@ -1,6 +1,7 @@
# Verify that tests bootstrap their own Bash and Unix utilities.
$ErrorActionPreference = 'Stop'
-$bazel = (Get-Command bazel).Source
+# Use one native launcher: PowerShell script parameter binding consumes --.
+$bazel = (Get-Command bazel -CommandType Application | Select-Object -First 1).Source
# Hosted runners install Bazelisk through npm; retain its Node launcher only.
$nodeDirectory = Split-Path (Get-Command node).Source
# Exclude System32 too: it can contain the WSL bash.exe forwarding stub.
@@ -10,18 +11,27 @@ $env:MSYS_NO_PATHCONV = '1'
foreach ($name in @('bash', 'grep', 'sed', 'diff')) {
if (Get-Command $name -CommandType Application -ErrorAction SilentlyContinue) { throw "Unexpected installed tool: $name" }
}
-& $bazel test //...
+# Build events are independent of test execution; the always-run workflow step
+# collects a failed command's outputs too.
+$reports = if ($env:BASHTEST_CI_REPORTS) { $env:BASHTEST_CI_REPORTS } else { Join-Path $env:TEMP ([guid]::NewGuid().ToString()) }
+New-Item -ItemType Directory -Path $reports -Force | Out-Null
+& $bazel test //... "--build_event_json_file=$reports/root-default.json"
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
-& $bazel test --enable_runfiles=no //...
+& $bazel run //tools:ci_reports -- collect --events "$reports/root-default.json" --output "$reports/root-default"
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
-& $bazel run --enable_runfiles=no //bashtest/tests:runtime_test
+& $bazel test --enable_runfiles=no //... "--build_event_json_file=$reports/root-manifest.json"
+if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
+& $bazel run //tools:ci_reports -- collect --events "$reports/root-manifest.json" --output "$reports/root-manifest"
+if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
+& $bazel run --enable_runfiles=no //bashtest/tests:runtime_test "--build_event_json_file=$reports/root-runtime.json"
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
# Drive the existing external-consumer tests with the downloaded interpreter.
-$execRoot = & $bazel info execution_root
+# Resolve the downloaded source, not an execroot link pruned by later builds.
+$outputBase = & $bazel info output_base
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
$bashFile = & $bazel cquery '@bashtest_windows_runtime//:usr/bin/bash.exe' --output=files
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
-$bash = Join-Path $execRoot $bashFile
+$bash = Join-Path $outputBase $bashFile
$env:PATH = "$(Split-Path $bash);$(Split-Path $bazel);$env:PATH"
& $bash bashtest/tests/external_consumer_test.sh
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
diff --git a/docs/infrastructure.md b/docs/infrastructure.md
index 82d4986..5397318 100644
--- a/docs/infrastructure.md
+++ b/docs/infrastructure.md
@@ -25,6 +25,64 @@ Use these measurements before introducing stricter budgets or custom eviction. A
inventory is reported without replacing test failures. This inventory includes all refs and
may briefly lag post-job cache uploads; verify later runs before claiming savings.
+### Test diagnostics
+
+Every test-matrix job retains a `test-results-OS` Actions artifact for seven days,
+including on failure. The workflow and consumer scripts execute ordinary Bazel
+commands with `--build_event_json_file`. Collection runs separately through the
+Bazel-built `//tools:ci_reports` tool, which copies the `test.log` and `test.xml`
+files named by that
+invocation's test-result events. Root runfiles modes, external consumers, shards,
+and retry attempts remain separate. Results are copied before another command can
+overwrite them and before consumer cleanup removes its temporary workspace. Stale
+results in `bazel-testlogs` are not collected. Missing local outputs and malformed
+build events are reported as collection errors; remote-only output URIs are not
+downloaded by this local-CI collector.
+
+An always-run step writes per-invocation JUnit counts and limited failure/error
+excerpts to the job summary. It first collects pending root results, including
+commands that failed before an explicit collection call. Consumer cleanup does the
+same before deleting its workspace and retains negative-analysis command logs.
+Build/test status comes from the build events' finished event;
+Bazel's ordinary exit status still determines the test step's outcome. For
+`bazel run`, build events describe the build, while the native CI step reports
+the program's exit status. Event files
+under an `expected/` directory identify intentional failure commands. Console
+output remains in the native Actions step logs. It counts actual cases rather than trusting aggregate
+XML attributes, labels intentionally failing fixture commands, and fails on
+unexpected command outcomes, collection errors, or malformed JUnit XML. The
+consumer scripts still verify each expected failure's precise behavior and print
+captured negative-test logs if those checks fail. A missing report is shown as
+such, leaving the original Bazel command status authoritative.
+
+Summary names and excerpts are HTML-escaped, diagnostics are placed in literal
+`pre` blocks, and only 20 bounded excerpts and 200 invocation rows are displayed.
+XML files above 8 MiB remain in the artifact but are not parsed for the summary;
+DTD/entity declarations are rejected. Complete reports and logs are retained.
+Summary generation uses Python's standard library and `GITHUB_STEP_SUMMARY`, with
+no PR comments, check-writing token, or privileged follow-up workflow. Python tests
+are `py_test` targets under `//tools/...` and run with `bazel test //...`; the
+collector and documentation verifier are `py_binary` targets. `rules_python` and
+its Python 3.13 toolchain are dev dependencies, so consumers acquire no Python
+runtime requirement. No separate Python installation or unittest CI job is needed.
+Tests that exercise Unix process/signal or Git/shell release behavior run on Linux
+and macOS. Portable Python suites also run in the stripped-PATH Windows job.
+
+To reproduce capture locally, use a fresh output directory for each invocation:
+
+```sh
+export BASHTEST_CI_REPORTS=/tmp/bashtest-reports
+mkdir -p "$BASHTEST_CI_REPORTS"
+bazel test //... --build_event_json_file="$BASHTEST_CI_REPORTS/root.json"
+bazel run //tools:ci_reports -- collect --events "$BASHTEST_CI_REPORTS/root.json" --output "$BASHTEST_CI_REPORTS/root"
+bash bashtest/tests/external_consumer_test.sh
+bazel run //tools:ci_reports -- finalize "$BASHTEST_CI_REPORTS" --destination "$BASHTEST_CI_REPORTS" --output /tmp/bashtest-summary.md
+```
+
+Without `BASHTEST_CI_REPORTS`, the standalone consumer scripts run Bazel directly.
+Artifact retention after forced runner termination is limited to files already
+written; ordinary Bazel test timeouts are collected and verified explicitly.
+
## Publishing
The release publisher uses trusted main tooling and the tagged documentation snapshot.
@@ -59,8 +117,7 @@ The change follows [mbo PR #549](https://github.com/mboworks/mbo/pull/549).
retains this repository's layout and API promises. Beautysh remains the shell formatter; no C++
style guide or competing shell formatter is introduced.
-Run `bazel test //...`, `python3 -m unittest discover -s tools -p '*_test.py'`, and
-`pre-commit run --all-files`. CI also builds the configured release documentation to validate
+Run `bazel test //...` and `pre-commit run --all-files`. CI also builds the configured release documentation to validate
links and mappings before publication.
## Version consistency
diff --git a/tools/BUILD.bazel b/tools/BUILD.bazel
new file mode 100644
index 0000000..cc4c9d7
--- /dev/null
+++ b/tools/BUILD.bazel
@@ -0,0 +1,68 @@
+# SPDX-FileCopyrightText: Copyright (c) M. Boerger, the MBO Works authors
+# SPDX-License-Identifier: Apache-2.0
+
+load("@rules_python//python:py_binary.bzl", "py_binary")
+load("@rules_python//python:py_library.bzl", "py_library")
+load("@rules_python//python:py_test.bzl", "py_test")
+
+package(default_visibility = ["//visibility:private"])
+
+py_library(
+ name = "tool_lib",
+ srcs = glob(
+ ["*.py"],
+ exclude = ["*_test.py"],
+ ),
+ imports = ["."],
+)
+
+py_binary(
+ name = "ci_reports",
+ srcs = ["ci_reports.py"],
+)
+
+py_binary(
+ name = "release_site",
+ srcs = ["release_site.py"],
+)
+
+filegroup(
+ name = "test_data",
+ srcs = glob([
+ "*.sh",
+ "*.txt",
+ ]) + [
+ "//:tool_test_data",
+ "//bashtest:bashtest_sh",
+ "//bashtest/tests:consumer_scripts",
+ ],
+)
+
+[
+ py_test(
+ name = name,
+ size = "medium",
+ srcs = [name + ".py"],
+ data = [":test_data"],
+ target_compatible_with = select({
+ "//bashtest:windows": ["@platforms//:incompatible"] if name in [
+ "junit_test",
+ "release_notes_test",
+ "release_site_test",
+ "version_test",
+ ] else [],
+ "//conditions:default": [],
+ }),
+ deps = [":tool_lib"],
+ )
+ for name in [
+ "ci_reports_test",
+ "infrastructure_test",
+ "junit_test",
+ "release_notes_test",
+ "release_site_test",
+ "site_artwork_test",
+ "site_storage_test",
+ "version_test",
+ ]
+]
diff --git a/tools/ci_reports.py b/tools/ci_reports.py
new file mode 100644
index 0000000..fe786df
--- /dev/null
+++ b/tools/ci_reports.py
@@ -0,0 +1,245 @@
+# SPDX-FileCopyrightText: Copyright (c) M. Boerger, the MBO Works authors
+# SPDX-License-Identifier: Apache-2.0
+"""Retain exact Bazel invocation results and summarize local JUnit XML."""
+
+import argparse
+import html
+import json
+from pathlib import Path
+import shutil
+import sys
+from urllib.parse import urlsplit
+from urllib.request import url2pathname
+import xml.etree.ElementTree as ET
+
+
+MAX_XML_BYTES = 8 * 1024 * 1024
+
+
+def collect(events, destination):
+ """Copy only this invocation's BEP test outputs, including shards and retries."""
+ results, errors = [], []
+ try:
+ with events.open(encoding='utf-8') as stream:
+ for line in stream:
+ event = json.loads(line)
+ if 'testResult' not in event:
+ continue
+ identity = event['id']['testResult']
+ result = {'label': identity['label'], 'run': identity.get('run', 1),
+ 'shard': identity.get('shard', 1), 'attempt': identity.get('attempt', 1),
+ 'status': event['testResult'].get('status', 'UNKNOWN'), 'files': []}
+ folder = destination / f'result_{len(results) + 1:04d}'
+ results.append(result)
+ for output in event['testResult'].get('testActionOutput', []):
+ if output.get('name') not in ('test.log', 'test.xml'):
+ continue
+ uri = urlsplit(output.get('uri', ''))
+ if uri.scheme != 'file' or uri.netloc not in ('', 'localhost'):
+ errors.append(f"Unavailable local output for {identity['label']}: {output['name']}")
+ continue
+ folder.mkdir(parents=True, exist_ok=True)
+ target = folder / output['name']
+ try:
+ shutil.copyfile(Path(url2pathname(uri.path)), target)
+ result['files'].append(target.relative_to(destination).as_posix())
+ except OSError as error:
+ errors.append(str(error))
+ except (OSError, ValueError, KeyError) as error:
+ errors.append(f'Cannot read build events: {error}')
+ return results, errors
+
+
+def retain(events, output, expected=None):
+ output.mkdir(parents=True, exist_ok=True)
+ results, errors = collect(events, output)
+ status = None
+ command = None
+ try:
+ for line in events.read_text(encoding='utf-8').splitlines():
+ event = json.loads(line)
+ command = event.get('started', {}).get('command', command)
+ finished = event.get('finished', {})
+ if 'exitCode' in finished:
+ status = finished['exitCode'].get('code', 0)
+ if events.resolve() != (output / 'events.json').resolve():
+ shutil.copyfile(events, output / 'events.json')
+ except (OSError, ValueError) as error:
+ errors.append(str(error))
+ if status is None:
+ errors.append('Build events have no confirmed command exit status')
+ status = 1
+ expected = expected or ('failure' if events.parent.name == 'expected' else 'success')
+ (output / 'invocation.json').write_text(json.dumps({
+ 'exit_code': status, 'expected': expected, 'command': command, 'results': results, 'errors': errors,
+ }, indent=2) + '\n', encoding='utf-8')
+ for error in errors:
+ print(f'CI result collection: {error}', file=sys.stderr)
+ return int(bool(errors))
+
+
+def finalize(events_directory, destination):
+ failed = False
+ events_files = sorted(events_directory.glob('*.json')) + sorted((events_directory / 'expected').glob('*.json'))
+ for events in events_files:
+ output = destination / events.stem
+ # Earlier snapshots must survive later commands overwriting their source.
+ if (output / 'invocation.json').exists():
+ continue
+ failed = bool(retain(events, output)) or failed
+ return failed
+
+
+class SummaryTreeBuilder(ET.TreeBuilder):
+ def doctype(self, name, pubid, system):
+ raise ValueError('DTD/entity declarations are not supported')
+
+
+def counts(path):
+ if path.stat().st_size > MAX_XML_BYTES:
+ return None, ['XML exceeds the summary size limit; see the artifact.']
+ data = path.read_bytes()
+ root = ET.fromstring(data, parser=ET.XMLParser(target=SummaryTreeBuilder()))
+ if root.tag not in ('testsuites', 'testsuite'):
+ raise ValueError('Expected a JUnit testsuite or testsuites root')
+ cases = list(root.iter('testcase'))
+ totals = [len(cases), 0, 0, 0]
+ details = []
+ for case in cases:
+ for index, kind in enumerate(('failure', 'error', 'skipped'), 1):
+ node = case.find(kind)
+ if node is not None:
+ totals[index] += 1
+ if kind != 'skipped' and len(details) < 3:
+ details.append(f"{case.get('name', '?')[:200]}: "
+ f"{node.get('message', '')[:400]} {(node.text or '')[:800]}")
+ return totals, details
+
+
+def summarize(directory):
+ sections = {'Tests': [], 'Expected-failure checks': [], 'Builds': []}
+ details, expected_details, problems = [], [], []
+ manifests = sorted(directory.glob('*/invocation.json'))
+ for index, manifest in enumerate(manifests):
+ invocation = json.loads(manifest.read_text(encoding='utf-8'))
+ negative = invocation['expected'] == 'failure'
+ code = invocation['exit_code']
+ matched = (code != 0) if negative else (code == 0)
+ invocation_problems = list(invocation['errors'])
+ if not matched:
+ invocation_problems.append(f'unexpected command exit {code}')
+ totals = [0, 0, 0, 0]
+ reports = 0
+ messages = []
+ # Earlier failed retries remain in the artifact; only the final attempt
+ # for each run/shard contributes to the displayed outcome and case counts.
+ attempts = {}
+ for result in invocation['results']:
+ key = (result['label'], result['run'], result['shard'])
+ if key not in attempts or result['attempt'] > attempts[key]['attempt']:
+ attempts[key] = result
+ for result in attempts.values():
+ if not negative and result.get('status', 'PASSED') not in ('PASSED', 'FLAKY'):
+ invocation_problems.append(f"{result['label']}: Bazel test status {result['status']}")
+ for name in result['files']:
+ if not name.endswith('/test.xml'):
+ continue
+ try:
+ values, excerpts = counts(manifest.parent / name)
+ if values is not None:
+ totals = [a + b for a, b in zip(totals, values)]
+ reports += 1
+ if not negative and (values[1] or values[2]):
+ invocation_problems.append(f'{name}: unexpected JUnit failures/errors')
+ for message in excerpts:
+ if len(messages) < 20:
+ messages.append(f"{manifest.parent.name} / {result['label'][:200]} "
+ f"(run {result['run']}, shard {result['shard']}, attempt {result['attempt']}): {message}")
+ except (OSError, ValueError, ET.ParseError) as error:
+ invocation_problems.append(f'{name}: invalid JUnit: {error}')
+ problems.extend(f'{manifest.parent.name}: {problem}' for problem in invocation_problems)
+ if negative and matched and not invocation_problems:
+ expected_details.extend(messages)
+ else:
+ details.extend(messages)
+ outcome = 'FAILED' if invocation_problems else 'Passed'
+ if negative:
+ section = 'Expected-failure checks'
+ outcome = 'FAILED' if invocation_problems else f'Expected nonzero exit observed ({code})'
+ cells = (manifest.parent.name[:200], outcome)
+ elif invocation.get('command') in ('run', 'build'):
+ section = 'Builds'
+ if invocation.get('command') == 'run':
+ outcome = 'build ' + outcome.lower() + '; program exit reported by CI'
+ cells = (manifest.parent.name[:200], outcome)
+ else:
+ section = 'Tests'
+ cells = (manifest.parent.name[:200], outcome, *map(str, totals)) if reports else (
+ manifest.parent.name[:200], outcome, 'Unavailable', '—', '—', '—')
+ if index < 200:
+ sections[section].append('' + ''.join(f'| {html.escape(value)} | ' for value in cells) + '
')
+ lines = ['## Bazel test results', '',
+ 'Unexpected results or reporting problems detected.' if problems else
+ 'No unexpected results in the recorded build/test reports.', '',
+ 'Case counts use the final attempt of each run/shard. All attempts remain in the artifact. '
+ 'CI steps determine overall success, including consumer assertions and bazel run program exits.']
+ for section, rows in sections.items():
+ if not rows:
+ continue
+ lines.extend(['', f'### {section}', ''])
+ if section == 'Expected-failure checks':
+ lines.extend(['These fixtures deliberately fail to exercise failure XML, fail-fast, timeouts, '
+ 'and invalid tool declarations. Their failures/errors are expected fixture output, '
+ 'excluded from the test table. Consumer scripts check the precise behavior; '
+ 'the rows below report the observed command exit.', ''])
+ columns = ('Invocation', 'Outcome', 'Cases', 'Failures', 'Errors', 'Skipped') if section == 'Tests' else ('Invocation', 'Outcome')
+ lines.append('' + ''.join(f'| {column} | ' for column in columns) + '
')
+ lines.extend(rows)
+ lines.append('
')
+ lines.extend(['', f'{len(manifests)} invocations retained. Download the test-results artifact for full logs, XML, and build events.'])
+ if details:
+ lines.extend(['', 'Test diagnostics (limited to 20 excerpts):', '',
+ '' + html.escape('\n\n'.join(details[:20])) + ''])
+ if expected_details:
+ lines.extend(['', 'Expected fixture diagnostics (intentional failures; limited to 20 excerpts)
', '',
+ '' + html.escape('\n\n'.join(expected_details[:20])) + '', '', '' + html.escape('\n'.join(problems)[:8000]) + ''])
+ if not manifests:
+ lines.extend(['', 'No recorded invocations; a prerequisite may have failed.'])
+ return '\n'.join(lines) + '\n', bool(problems)
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__)
+ commands = parser.add_subparsers(dest='action', required=True)
+ collector = commands.add_parser('collect')
+ collector.add_argument('--events', type=Path, required=True)
+ collector.add_argument('--output', type=Path, required=True)
+ collector.add_argument('--expect', choices=('success', 'failure'))
+ finalizer = commands.add_parser('finalize')
+ finalizer.add_argument('directory', type=Path)
+ finalizer.add_argument('--destination', type=Path, required=True)
+ finalizer.add_argument('--output', type=Path)
+ summary = commands.add_parser('summary')
+ summary.add_argument('directory', type=Path)
+ summary.add_argument('--output', type=Path, required=True)
+ args = parser.parse_args()
+ if args.action == 'collect':
+ return retain(args.events, args.output, args.expect)
+ failed = False
+ if args.action == 'finalize':
+ failed = finalize(args.directory, args.destination)
+ if args.output is None:
+ return int(failed)
+ directory = args.destination if args.action == 'finalize' else args.directory
+ text, summary_failed = summarize(directory)
+ failed = summary_failed or failed
+ with args.output.open('a', encoding='utf-8') as stream:
+ stream.write(text)
+ return int(failed)
+
+
+if __name__ == '__main__':
+ sys.exit(main())
diff --git a/tools/ci_reports_test.py b/tools/ci_reports_test.py
new file mode 100644
index 0000000..e5f7028
--- /dev/null
+++ b/tools/ci_reports_test.py
@@ -0,0 +1,238 @@
+# SPDX-FileCopyrightText: Copyright (c) M. Boerger, the MBO Works authors
+# SPDX-License-Identifier: Apache-2.0
+"""Observable CI report collection, exit status, and safe rendering contracts."""
+
+import json
+import os
+from pathlib import Path
+import subprocess
+import tempfile
+import unittest
+from unittest import mock
+
+import ci_reports
+
+
+ROOT = Path(__file__).parents[1]
+
+
+class CiReportsTest(unittest.TestCase):
+ def setUp(self):
+ temporary = tempfile.TemporaryDirectory(prefix='ci reports ')
+ self.addCleanup(temporary.cleanup)
+ self.root = Path(temporary.name)
+
+ def events(self, *outputs, code=0):
+ events = self.root / 'events.json'
+ events.write_text(''.join(json.dumps({'id': {'testResult': {'label': '//:fixture', 'shard': i + 1}},
+ 'testResult': {'status': 'FAILED', 'testActionOutput': output}}) + '\n'
+ for i, output in enumerate(outputs)) +
+ json.dumps({'finished': {'exitCode': {'code': code}}}) + '\n')
+ return events
+
+ def manifest(self, name, expected='success', code=0, results=None, errors=None):
+ folder = self.root / name
+ folder.mkdir(exist_ok=True)
+ (folder / 'invocation.json').write_text(json.dumps({'expected': expected, 'exit_code': code,
+ 'results': results or [], 'errors': errors or []}))
+ return folder
+
+ def test_copies_only_reported_outputs_before_consumer_cleanup(self):
+ source = self.root / 'consumer with spaces'
+ source.mkdir()
+ (source / 'test.xml').write_text('')
+ (source / 'test.log').write_text('diagnostic')
+ (source / 'unrelated.xml').write_text('stale shard')
+ outputs = [{'name': name, 'uri': (source / name).as_uri()} for name in ('test.xml', 'test.log')]
+ destination = self.manifest('collected')
+ results, errors = ci_reports.collect(self.events(outputs, outputs), destination)
+ self.assertEqual(errors, [])
+ self.assertEqual(len(results), 2)
+ self.assertEqual([r['shard'] for r in results], [1, 2])
+ for item in source.iterdir():
+ item.unlink()
+ source.rmdir()
+ self.assertEqual((destination / 'result_0001/test.log').read_text(), 'diagnostic')
+ self.assertEqual(len(list(destination.rglob('test.xml'))), 2)
+ self.assertFalse(list(destination.rglob('unrelated.xml')))
+
+ def test_missing_remote_and_partial_outputs_are_reported(self):
+ outputs = [{'name': 'test.xml', 'uri': 'bytestream://remote/test.xml'},
+ {'name': 'test.log', 'uri': (self.root / 'missing').as_uri()}]
+ results, errors = ci_reports.collect(self.events(outputs), self.root / 'copy')
+ self.assertEqual(len(results), 1)
+ self.assertEqual(len(errors), 2)
+ events = self.events()
+ events.write_text('{"unfinished":')
+ self.assertTrue(ci_reports.collect(events, self.root / 'partial')[1])
+
+ def test_summary_counts_actual_cases_and_escapes_diagnostics(self):
+ folder = self.manifest('expected', expected='failure', code=3, results=[{
+ 'label': '//:', 'run': 1, 'shard': 2, 'attempt': 3,
+ 'files': ['result_0001/test.xml']}])
+ (folder / 'result_0001').mkdir()
+ (folder / 'result_0001/test.xml').write_text('''
+
+<script>bad</script> [link](https://example.com)
+
+''')
+ text, failed = ci_reports.summarize(self.root)
+ self.assertFalse(failed)
+ self.assertNotIn('Failures | ', text)
+ self.assertIn('Expected nonzero exit observed (3)', text)
+ self.assertIn('Expected fixture diagnostics', text)
+ self.assertNotIn('Test diagnostics (', text)
+ self.assertNotIn('