From 0ed1c6226e2b581b0aab494888c72910186e5a7d Mon Sep 17 00:00:00 2001 From: helly25 <6420169+helly25@users.noreply.github.com> Date: Sun, 25 Oct 2026 22:42:18 +0000 Subject: [PATCH 1/7] Retain per-invocation CI diagnostics and summarize JUnit results --- .github/workflows/main.yml | 24 ++- bashtest/tests/ci_support.sh | 30 ++++ bashtest/tests/external_consumer_test.sh | 25 ++-- bashtest/tests/tools_consumer_test.sh | 13 +- bashtest/tests/windows_runtime_test.ps1 | 24 ++- docs/infrastructure.md | 41 +++++ tools/ci_reports.py | 182 +++++++++++++++++++++++ tools/ci_reports_test.py | 159 ++++++++++++++++++++ tools/infrastructure_test.py | 11 ++ 9 files changed, 487 insertions(+), 22 deletions(-) create mode 100644 bashtest/tests/ci_support.sh create mode 100644 tools/ci_reports.py create mode 100644 tools/ci_reports_test.py diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3e3597f..763733b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -58,6 +58,12 @@ jobs: runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 + with: + python-version: '3.13' + - name: Configure report capture + shell: bash + run: echo "BASHTEST_CI_REPORTS=${RUNNER_TEMP}/test-results" >> "${GITHUB_ENV}" - uses: bazel-contrib/setup-bazel@0.19.1 with: cache-save: ${{ github.ref == 'refs/heads/main' }} @@ -72,8 +78,8 @@ jobs: env: MSYS_NO_PATHCONV: "1" run: | - bazel test //... - bazel test --enable_runfiles=no //... + python3 tools/ci_reports.py run --output "${BASHTEST_CI_REPORTS}/root-default" -- bazel test //... + python3 tools/ci_reports.py run --output "${BASHTEST_CI_REPORTS}/root-manifest" -- bazel test --enable_runfiles=no //... - name: Test external module consumer if: runner.os != 'Windows' env: @@ -88,6 +94,20 @@ jobs: shell: pwsh run: ./bashtest/tests/windows_runtime_test.ps1 + - name: Summarize JUnit results + if: always() + shell: bash + run: python tools/ci_reports.py summary "${BASHTEST_CI_REPORTS}" --output "${GITHUB_STEP_SUMMARY}" + + - name: Retain test diagnostics + if: always() + uses: actions/upload-artifact@v7 + with: + name: test-results-${{ matrix.os }} + path: ${{ runner.temp }}/test-results + if-no-files-found: warn + retention-days: 7 + done: permissions: contents: read diff --git a/bashtest/tests/ci_support.sh b/bashtest/tests/ci_support.sh new file mode 100644 index 0000000..a8e1ca9 --- /dev/null +++ b/bashtest/tests/ci_support.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +# SPDX-FileCopyrightText: Copyright (c) M. Boerger, the MBO Works authors +# SPDX-License-Identifier: Apache-2.0 + +# Private CI helpers shared by the standalone consumer scripts. +ci_bazel() { + local name="${1}" expected="${2}" reporter + shift 2 + if [[ -z "${BASHTEST_CI_REPORTS:-}" ]]; then + bazel "${@}" + return + fi + reporter="${BASH_SOURCE[0]%/*}/../../tools/ci_reports.py" + case "${OSTYPE}" in + msys*|cygwin*) reporter="$(cygpath -am "${reporter}")" ;; + esac + "${BASHTEST_CI_PYTHON:-python3}" "${reporter}" run \ + --output "${BASHTEST_CI_REPORTS}/${name}" --expect "${expected}" -- bazel "${@}" +} + +ci_failure_logs() { + local status="${1}" directory="${2}" log + [[ "${status}" != 0 ]] || return 0 + for log in "${directory}"/*.log; do + [[ -f "${log}" ]] || continue + printf >&2 '\n[UNEXPECTED CONSUMER FAILURE: %s]\n' "${log##*/}" + cat "${log}" >&2 + done +} diff --git a/bashtest/tests/external_consumer_test.sh b/bashtest/tests/external_consumer_test.sh index 96ea552..38b629e 100755 --- a/bashtest/tests/external_consumer_test.sh +++ b/bashtest/tests/external_consumer_test.sh @@ -18,14 +18,19 @@ # Run outside a Bazel test: this creates and builds a separate consumer module. set -euo pipefail repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +# shellcheck source=bashtest/tests/ci_support.sh +source "${repo}/bashtest/tests/ci_support.sh" # The override consumes this checkout, including unreleased development versions. version="$(sed -nE 's/.*version = "([^"]+)".*/\1/p' "${repo}/MODULE.bazel" | head -n1)" root="$(mktemp -d)" cleanup() { + local status="${?}" + ci_failure_logs "${status}" "${PWD}" # Windows cannot remove a directory while it is a process working directory. bazel shutdown >/dev/null 2>&1 || true cd "${repo}" rm -rf "${root}" + exit "${status}" } trap cleanup EXIT case "${OSTYPE}" in @@ -65,11 +70,11 @@ test_runner EOF # Also run the framework's own suite as an external dependency, as BCR does. # A standalone consumer test alone cannot catch root-workspace assumptions in it. -bazel test --enable_runfiles=no --test_output=errors //:consumer_test @framework//bashtest/tests/... +ci_bazel external-full success test --enable_runfiles=no --test_output=errors //:consumer_test @framework//bashtest/tests/... # A filtered/sharded meta-test must not leak its settings into verifier fixtures. -bazel test --enable_runfiles=no --test_filter='^junit_cases_and_failures$' \ +ci_bazel external-meta-shards success test --enable_runfiles=no --test_filter='^junit_cases_and_failures$' \ --test_sharding_strategy=forced=5 @framework//bashtest/tests:junit_test -bazel run --enable_runfiles=no //:consumer_test -- --keep-tmpdir=never +ci_bazel external-run success run --enable_runfiles=no //:consumer_test -- --keep-tmpdir=never # Exercise both documented update spellings with a workspace path containing spaces. # Separate MSYS2 installations have different /tmp mounts. Cross the native @@ -79,10 +84,10 @@ case "${OSTYPE}" in msys*|cygwin*) workspace="$(cygpath -am "${workspace}")" ;; esac printf 'stale\n' > golden.txt -bazel run --enable_runfiles=no //:consumer_test -- --update-golden="${workspace}" --keep-tmpdir=never +ci_bazel external-update-long success run --enable_runfiles=no //:consumer_test -- --update-golden="${workspace}" --keep-tmpdir=never [[ "$(cat golden.txt)" == 'hello world' ]] printf 'stale again\n' > golden.txt -bazel run --enable_runfiles=no //:consumer_test -- -u="${workspace}" --keep-tmpdir=never +ci_bazel external-update-short success run --enable_runfiles=no //:consumer_test -- -u="${workspace}" --keep-tmpdir=never [[ "$(cat golden.txt)" == 'hello world' ]] # Verify that Bazel collects per-function failure XML, rather than its fallback @@ -127,7 +132,7 @@ EOF junit_targets+=(//:junit_sh_failure_test) ;; esac -if bazel test --enable_runfiles=no "${junit_targets[@]}" > junit_failure.log 2>&1; then +if ci_bazel external-junit-failure failure test --enable_runfiles=no "${junit_targets[@]}" > junit_failure.log 2>&1; then echo >&2 'ERROR: intentionally failing JUnit fixtures passed' exit 1 fi @@ -186,8 +191,8 @@ EOF selection_targets+=(//:selection_sh_test) ;; esac -bazel test --enable_runfiles=no --test_filter='^(alpha|gamma)$' "${selection_targets[@]}" -bazel test --enable_runfiles=no --test_filter='^(alpha|gamma)$' \ +ci_bazel external-filter success test --enable_runfiles=no --test_filter='^(alpha|gamma)$' "${selection_targets[@]}" +ci_bazel external-shards success test --enable_runfiles=no --test_filter='^(alpha|gamma)$' \ --test_sharding_strategy=forced=5 "${selection_targets[@]}" for target in "${selection_targets[@]}"; do reports=("${junit_logs}/${target#//:}"/shard_*/test.xml) @@ -201,7 +206,7 @@ for target in "${selection_targets[@]}"; do [[ "${alpha}" == 1 && "${gamma}" == 1 && "${empty}" == 2 ]] echo "[SHARDING VERIFIED] ${target}: five reports, each selected case once, two empty reports" done -if bazel test --enable_runfiles=no --test_runner_fail_fast --test_env=FAIL_CASE=alpha \ +if ci_bazel external-fail-fast failure test --enable_runfiles=no --test_runner_fail_fast --test_env=FAIL_CASE=alpha \ "${selection_targets[@]}" > selection_failure.log 2>&1; then echo >&2 'ERROR: intentionally failing fail-fast fixtures passed' exit 1 @@ -258,7 +263,7 @@ EOF interrupted_targets+=(//:interrupted_sh_test) ;; esac -if bazel test --enable_runfiles=no --test_timeout=10 "${interrupted_targets[@]}" > interrupted.log 2>&1; then +if ci_bazel external-timeout failure test --enable_runfiles=no --test_timeout=10 "${interrupted_targets[@]}" > interrupted.log 2>&1; then echo >&2 'ERROR: intentionally hanging fixtures passed' exit 1 fi diff --git a/bashtest/tests/tools_consumer_test.sh b/bashtest/tests/tools_consumer_test.sh index 7a0b3ea..3428302 100755 --- a/bashtest/tests/tools_consumer_test.sh +++ b/bashtest/tests/tools_consumer_test.sh @@ -18,13 +18,18 @@ # Integration test: distinct default and test execution platforms, external module. set -euo pipefail repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +# shellcheck source=bashtest/tests/ci_support.sh +source "${repo}/bashtest/tests/ci_support.sh" # The override consumes this checkout, including unreleased development versions. version="$(sed -nE 's/.*version = "([^"]+)".*/\1/p' "${repo}/MODULE.bazel" | head -n1)" root="$(mktemp -d)" cleanup() { + local status="${?}" + ci_failure_logs "${status}" "${PWD}" bazel shutdown >/dev/null 2>&1 || true cd "${repo}" rm -rf "${root}" + exit "${status}" } trap cleanup EXIT case "${OSTYPE}" in @@ -98,14 +103,14 @@ test::configuration() { test_runner EOF flags=("--extra_execution_platforms=//:ordinary_exec,//:test_exec" --enable_runfiles=no) -bazel test "${flags[@]}" --test_output=errors //:consumer_test -bazel run "${flags[@]}" //:consumer_test -- --keep-tmpdir=never -if bazel build "${flags[@]}" //:invalid_tool_test >invalid.log 2>&1; then +ci_bazel tools-test success test "${flags[@]}" --test_output=errors //:consumer_test +ci_bazel tools-run success run "${flags[@]}" //:consumer_test -- --keep-tmpdir=never +if ci_bazel tools-non-executable failure build "${flags[@]}" //:invalid_tool_test >invalid.log 2>&1; then echo >&2 'ERROR: non-executable tool was accepted' exit 1 fi grep -F 'tools requires an executable target' invalid.log | sed 's/^/[EXPECTED FAILURE: non-executable tool] /' -if bazel build "${flags[@]}" //:collision_test >collision.log 2>&1; then +if ci_bazel tools-collision failure build "${flags[@]}" //:collision_test >collision.log 2>&1; then echo >&2 'ERROR: conflicting environment variable was accepted' exit 1 fi diff --git a/bashtest/tests/windows_runtime_test.ps1 b/bashtest/tests/windows_runtime_test.ps1 index b42b017..796a3ff 100644 --- a/bashtest/tests/windows_runtime_test.ps1 +++ b/bashtest/tests/windows_runtime_test.ps1 @@ -1,6 +1,21 @@ # Verify that tests bootstrap their own Bash and Unix utilities. $ErrorActionPreference = 'Stop' $bazel = (Get-Command bazel).Source +# Keep an absolute Python path for CI reporting after Unix tools and Python are +# removed from PATH. Python is a CI dependency, not a bashtest runtime dependency. +$python = $null +if ($env:BASHTEST_CI_REPORTS) { + $python = (Get-Command python).Source + $env:BASHTEST_CI_PYTHON = $python.Replace('\', '/') +} +function Invoke-TestBazel([string] $name, [string[]] $bazelArgs) { + if ($env:BASHTEST_CI_REPORTS) { + & $python tools/ci_reports.py run --output "$env:BASHTEST_CI_REPORTS/$name" -- $bazel @bazelArgs + } else { + & $bazel @bazelArgs + } + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } +} # Hosted runners install Bazelisk through npm; retain its Node launcher only. $nodeDirectory = Split-Path (Get-Command node).Source # Exclude System32 too: it can contain the WSL bash.exe forwarding stub. @@ -10,12 +25,9 @@ $env:MSYS_NO_PATHCONV = '1' foreach ($name in @('bash', 'grep', 'sed', 'diff')) { if (Get-Command $name -CommandType Application -ErrorAction SilentlyContinue) { throw "Unexpected installed tool: $name" } } -& $bazel test //... -if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } -& $bazel test --enable_runfiles=no //... -if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } -& $bazel run --enable_runfiles=no //bashtest/tests:runtime_test -if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } +Invoke-TestBazel 'root-default' @('test', '//...') +Invoke-TestBazel 'root-manifest' @('test', '--enable_runfiles=no', '//...') +Invoke-TestBazel 'root-runtime' @('run', '--enable_runfiles=no', '//bashtest/tests:runtime_test') # Drive the existing external-consumer tests with the downloaded interpreter. $execRoot = & $bazel info execution_root if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } diff --git a/docs/infrastructure.md b/docs/infrastructure.md index 82d4986..686f97a 100644 --- a/docs/infrastructure.md +++ b/docs/infrastructure.md @@ -25,6 +25,47 @@ Use these measurements before introducing stricter budgets or custom eviction. A inventory is reported without replacing test failures. This inventory includes all refs and may briefly lag post-job cache uploads; verify later runs before claiming savings. +### Test diagnostics + +Every test-matrix job retains a `test-results-OS` Actions artifact for seven days, +including on failure. `tools/ci_reports.py run` saves each Bazel command's console +log and build events, then copies the `test.log` and `test.xml` files named by that +invocation's test-result events. Root runfiles modes, external consumers, shards, +and retry attempts remain separate. Results are copied before another command can +overwrite them and before consumer cleanup removes its temporary workspace. Stale +results in `bazel-testlogs` are not collected. Missing local outputs and malformed +build events are reported as collection errors; remote-only output URIs are not +downloaded by this local-CI collector. + +An always-run step writes per-invocation JUnit counts and limited failure/error +excerpts to the job summary. It counts actual cases rather than trusting aggregate +XML attributes, labels intentionally failing fixture commands, and fails on +unexpected command outcomes, collection errors, or malformed JUnit XML. The +consumer scripts still verify each expected failure's precise behavior and print +captured negative-test logs if those checks fail. A missing report is shown as +such, leaving the original Bazel command status authoritative. + +Summary names and excerpts are HTML-escaped, diagnostics are placed in literal +`pre` blocks, and only 20 bounded excerpts and 200 invocation rows are displayed. +XML files above 8 MiB remain in the artifact but are not parsed for the summary; +DTD/entity declarations are rejected. Complete reports and logs are retained. +Summary generation uses Python's standard library and `GITHUB_STEP_SUMMARY`, with +no PR comments, check-writing token, or privileged follow-up workflow. Python is +a CI reporting dependency, including an absolute interpreter path when Windows +validation strips PATH; it is not a requirement for running bashtest. + +To reproduce capture locally, use a fresh output directory for each invocation: + +```sh +python3 tools/ci_reports.py run --output /tmp/bashtest-reports/root -- bazel test //... +BASHTEST_CI_REPORTS=/tmp/bashtest-reports bash bashtest/tests/external_consumer_test.sh +python3 tools/ci_reports.py summary /tmp/bashtest-reports --output /tmp/bashtest-summary.md +``` + +Without `BASHTEST_CI_REPORTS`, the standalone consumer scripts run Bazel directly. +Artifact retention after forced runner termination is limited to files already +written; ordinary Bazel test timeouts are collected and verified explicitly. + ## Publishing The release publisher uses trusted main tooling and the tagged documentation snapshot. diff --git a/tools/ci_reports.py b/tools/ci_reports.py new file mode 100644 index 0000000..a6d3739 --- /dev/null +++ b/tools/ci_reports.py @@ -0,0 +1,182 @@ +# SPDX-FileCopyrightText: Copyright (c) M. Boerger, the MBO Works authors +# SPDX-License-Identifier: Apache-2.0 +"""Run CI Bazel commands, retain their exact results, and summarize local JUnit XML.""" + +import argparse +import html +import json +from pathlib import Path +import shutil +import subprocess +import sys +from urllib.parse import urlsplit +from urllib.request import url2pathname +import xml.etree.ElementTree as ET + + +MAX_XML_BYTES = 8 * 1024 * 1024 + + +def collect(events, destination): + """Copy only this invocation's BEP test outputs, including shards and retries.""" + results, errors = [], [] + try: + with events.open(encoding='utf-8') as stream: + for line in stream: + event = json.loads(line) + if 'testResult' not in event: + continue + identity = event['id']['testResult'] + result = {'label': identity['label'], 'run': identity.get('run', 1), + 'shard': identity.get('shard', 1), 'attempt': identity.get('attempt', 1), + 'status': event['testResult'].get('status', 'UNKNOWN'), 'files': []} + folder = destination / f'result_{len(results) + 1:04d}' + results.append(result) + for output in event['testResult'].get('testActionOutput', []): + if output.get('name') not in ('test.log', 'test.xml'): + continue + uri = urlsplit(output.get('uri', '')) + if uri.scheme != 'file' or uri.netloc not in ('', 'localhost'): + errors.append(f"Unavailable local output for {identity['label']}: {output['name']}") + continue + folder.mkdir(parents=True, exist_ok=True) + target = folder / output['name'] + try: + shutil.copyfile(Path(url2pathname(uri.path)), target) + result['files'].append(target.relative_to(destination).as_posix()) + except OSError as error: + errors.append(str(error)) + except (OSError, ValueError, KeyError) as error: + errors.append(f'Cannot read build events: {error}') + return results, errors + + +def run(output, expected, command): + output.mkdir(parents=True, exist_ok=False) + events = output.resolve() / 'events.json' + executable = shutil.which(command[0]) or command[0] + command = [executable, command[1], f'--build_event_json_file={events}', *command[2:]] + status = 1 + errors = [] + try: + with (output / 'command.log').open('wb') as log: + process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + with process.stdout: + while chunk := process.stdout.read1(65536): + log.write(chunk) + log.flush() + sys.stdout.buffer.write(chunk) + sys.stdout.buffer.flush() + status = process.wait() + except OSError as error: + errors.append(str(error)) + results, collection_errors = collect(events, output) + errors.extend(collection_errors) + (output / 'invocation.json').write_text(json.dumps({ + 'command': command, 'exit_code': status, 'expected': expected, + 'results': results, 'errors': errors, + }, indent=2) + '\n', encoding='utf-8') + for error in errors: + print(f'CI result collection: {error}', file=sys.stderr) + return status if status else int(bool(errors)) + + +class SummaryTreeBuilder(ET.TreeBuilder): + def doctype(self, name, pubid, system): + raise ValueError('DTD/entity declarations are not supported') + + +def counts(path): + if path.stat().st_size > MAX_XML_BYTES: + return None, ['XML exceeds the summary size limit; see the artifact.'] + data = path.read_bytes() + root = ET.fromstring(data, parser=ET.XMLParser(target=SummaryTreeBuilder())) + if root.tag not in ('testsuites', 'testsuite'): + raise ValueError('Expected a JUnit testsuite or testsuites root') + cases = list(root.iter('testcase')) + totals = [len(cases), 0, 0, 0] + details = [] + for case in cases: + for index, kind in enumerate(('failure', 'error', 'skipped'), 1): + node = case.find(kind) + if node is not None: + totals[index] += 1 + if kind != 'skipped' and len(details) < 3: + details.append(f"{case.get('name', '?')[:200]}: " + f"{node.get('message', '')[:400]} {(node.text or '')[:800]}") + return totals, details + + +def summarize(directory): + lines = ['## Bazel test results', '', + 'Each row is one invocation. Shards and retry attempts are retained separately; ' + 'counts include every reported attempt. Expected failures are checked by the consumer scripts.', '', + ''] + details, problems = [], [] + manifests = sorted(directory.glob('*/invocation.json')) + for index, manifest in enumerate(manifests): + invocation = json.loads(manifest.read_text(encoding='utf-8')) + expected = invocation['expected'] + code = invocation['exit_code'] + matched = (code == 0) == (expected == 'success') + outcome = ('passed' if code == 0 else 'expected failure') if matched else ('UNEXPECTED PASS' if code == 0 else 'FAILED') + if not matched: + problems.append(f'{manifest.parent.name}: unexpected command exit {code}') + problems.extend(invocation['errors']) + totals = [0, 0, 0, 0] + reports = 0 + for result in invocation['results']: + for name in result['files']: + if not name.endswith('/test.xml'): + continue + try: + values, messages = counts(manifest.parent / name) + if values is not None: + totals = [a + b for a, b in zip(totals, values)] + reports += 1 + for message in messages: + if len(details) < 20: + details.append(f"{manifest.parent.name} / {result['label'][:200]} " + f"(run {result['run']}, shard {result['shard']}, attempt {result['attempt']}): {message}") + except (OSError, ValueError, ET.ParseError) as error: + problems.append(f'{manifest.parent.name}/{name}: invalid JUnit: {error}') + count_text = ' / '.join(map(str, totals)) if reports else 'No summarized XML' + if index < 200: + lines.append('' + ''.join(f'' for value in + (manifest.parent.name[:200], f'{outcome} (exit {code}, expected {expected})', count_text)) + '') + lines.extend(['
InvocationCommandTests / failures / errors / skipped
{html.escape(value)}
', '', f'{len(manifests)} invocations retained. Download the test-results artifact for full logs, XML, and build events.']) + if details: + lines.extend(['', 'Failure/error diagnostics (limited to 20 excerpts):', '', + '
' + html.escape('\n\n'.join(details)) + '
']) + if problems: + lines.extend(['', 'Reporting/command problems:', '', + '
' + html.escape('\n'.join(problems)[:8000]) + '
']) + if not manifests: + lines.extend(['', 'No recorded invocations; a prerequisite may have failed.']) + return '\n'.join(lines) + '\n', bool(problems) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest='action', required=True) + runner = commands.add_parser('run') + runner.add_argument('--output', type=Path, required=True) + runner.add_argument('--expect', choices=('success', 'failure'), default='success') + runner.add_argument('command', nargs=argparse.REMAINDER) + summary = commands.add_parser('summary') + summary.add_argument('directory', type=Path) + summary.add_argument('--output', type=Path, required=True) + args = parser.parse_args() + if args.action == 'run': + command = args.command[1:] if args.command[:1] == ['--'] else args.command + if len(command) < 2 or command[1] not in ('test', 'build', 'run'): + parser.error('run requires a Bazel test, build, or run command after --') + return run(args.output, args.expect, command) + text, failed = summarize(args.directory) + with args.output.open('a', encoding='utf-8') as stream: + stream.write(text) + return int(failed) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tools/ci_reports_test.py b/tools/ci_reports_test.py new file mode 100644 index 0000000..9d28b6c --- /dev/null +++ b/tools/ci_reports_test.py @@ -0,0 +1,159 @@ +# SPDX-FileCopyrightText: Copyright (c) M. Boerger, the MBO Works authors +# SPDX-License-Identifier: Apache-2.0 +"""Observable CI report collection, exit status, and safe rendering contracts.""" + +import io +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest +from unittest import mock + +import ci_reports + + +ROOT = Path(__file__).resolve().parents[1] + + +class CiReportsTest(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory(prefix='ci reports ') + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) + + def events(self, *outputs): + events = self.root / 'events.json' + events.write_text(''.join(json.dumps({'id': {'testResult': {'label': '//:fixture', 'shard': i + 1}}, + 'testResult': {'status': 'FAILED', 'testActionOutput': output}}) + '\n' + for i, output in enumerate(outputs))) + return events + + def manifest(self, name, expected='success', code=0, results=None, errors=None): + folder = self.root / name + folder.mkdir(exist_ok=True) + (folder / 'invocation.json').write_text(json.dumps({'expected': expected, 'exit_code': code, + 'results': results or [], 'errors': errors or []})) + return folder + + def test_copies_only_reported_outputs_before_consumer_cleanup(self): + source = self.root / 'consumer with spaces' + source.mkdir() + (source / 'test.xml').write_text('') + (source / 'test.log').write_text('diagnostic') + (source / 'unrelated.xml').write_text('stale shard') + outputs = [{'name': name, 'uri': (source / name).as_uri()} for name in ('test.xml', 'test.log')] + destination = self.manifest('collected') + results, errors = ci_reports.collect(self.events(outputs, outputs), destination) + self.assertEqual(errors, []) + self.assertEqual(len(results), 2) + self.assertEqual([r['shard'] for r in results], [1, 2]) + for item in source.iterdir(): + item.unlink() + source.rmdir() + self.assertEqual((destination / 'result_0001/test.log').read_text(), 'diagnostic') + self.assertEqual(len(list(destination.rglob('test.xml'))), 2) + self.assertFalse(list(destination.rglob('unrelated.xml'))) + + def test_missing_remote_and_partial_outputs_are_reported(self): + outputs = [{'name': 'test.xml', 'uri': 'bytestream://remote/test.xml'}, + {'name': 'test.log', 'uri': (self.root / 'missing').as_uri()}] + results, errors = ci_reports.collect(self.events(outputs), self.root / 'copy') + self.assertEqual(len(results), 1) + self.assertEqual(len(errors), 2) + events = self.events() + events.write_text('{"unfinished":') + self.assertTrue(ci_reports.collect(events, self.root / 'partial')[1]) + + def test_summary_counts_actual_cases_and_escapes_diagnostics(self): + folder = self.manifest('expected', expected='failure', code=3, results=[{ + 'label': '//:', 'run': 1, 'shard': 2, 'attempt': 3, + 'files': ['result_0001/test.xml']}]) + (folder / 'result_0001').mkdir() + (folder / 'result_0001/test.xml').write_text(''' + +<script>bad</script> [link](https://example.com) + +''') + text, failed = ci_reports.summarize(self.root) + self.assertFalse(failed) + self.assertIn('4 / 1 / 1 / 1', text) + self.assertIn('expected failure', text) + self.assertNotIn('