From 06cd3c09735040b0ae67d04f86acb3686b9a5116 Mon Sep 17 00:00:00 2001 From: overtrue Date: Sat, 5 Sep 2026 18:54:47 +0800 Subject: [PATCH 1/2] fix(performance): preserve failed and incomplete benchmark rounds --- .../workflows/performance-script-tests.yml | 30 ++++ rustfs_performance_testing.sh | 65 +++++-- test_rustfs_performance_testing.py | 170 ++++++++++++++++++ 3 files changed, 248 insertions(+), 17 deletions(-) create mode 100644 .github/workflows/performance-script-tests.yml create mode 100644 test_rustfs_performance_testing.py diff --git a/.github/workflows/performance-script-tests.yml b/.github/workflows/performance-script-tests.yml new file mode 100644 index 00000000..35cf6cf7 --- /dev/null +++ b/.github/workflows/performance-script-tests.yml @@ -0,0 +1,30 @@ +name: Performance Script Tests + +on: + pull_request: + paths: + - rustfs_performance_testing.sh + - test_rustfs_performance_testing.py + - .github/workflows/performance-script-tests.yml + push: + branches: [main] + paths: + - rustfs_performance_testing.sh + - test_rustfs_performance_testing.py + - .github/workflows/performance-script-tests.yml + +permissions: + contents: read + +jobs: + performance-script-tests: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + with: + persist-credentials: false + - name: Validate shell syntax + run: bash -n rustfs_performance_testing.sh + - name: Test benchmark result contracts without a server + run: python3 test_rustfs_performance_testing.py diff --git a/rustfs_performance_testing.sh b/rustfs_performance_testing.sh index 44dd4f54..c6868008 100755 --- a/rustfs_performance_testing.sh +++ b/rustfs_performance_testing.sh @@ -45,8 +45,11 @@ else fi TOTAL_ROUNDS=$(( ${#METHODS[@]} * ${#SIZES[@]} )) ROUND=0 +PARSED_ROUNDS=0 +FAILED_ROUNDS=0 # --parse-only : parse an existing result dir (${method}_${size}.txt) without running +# Only round files with a successful terminal execution status can be certified. if [[ "${1:-}" == "--parse-only" && -n "${2:-}" ]]; then RESULT_DIR="$2" fi @@ -100,6 +103,10 @@ field() { # Parse one round's output and append a row to summary.tsv parse_round() { local method="$1" size="$2" file="$3" + if [[ "$(tail -n 1 "$file")" != "RustFS round status: warp=0 capture=0" ]]; then + log "PARSE_FAILED method=${method} size=${size} reason=missing_or_failed_round_status file=${file}" + return 1 + fi local line if [[ "${method}" == "mixed" ]]; then local total get put @@ -129,6 +136,14 @@ parse_round() { "$(field "${stats}" ttfb_p99)" \ "$(field "${stats}" ttfb_worst)") fi + # TTFB is optional in warp; throughput and request statistics define a result. + if ! printf '%s\n' "${line}" | awk -F '\t' ' + { last = $1 == "mixed" ? 6 : 8 + for (i = 3; i <= last; i++) if ($i !~ /[^[:space:]]/) exit 1 } + '; then + log "PARSE_FAILED method=${method} size=${size} reason=missing_required_metrics file=${file}" + return 1 + fi printf '%s\n' "${line}" >> "${SUMMARY_TSV}" } @@ -140,7 +155,8 @@ gen_summary_md() { echo "- Date: $(date -u '+%Y-%m-%d %H:%M:%S UTC')" echo "- Target: ${HOST}" echo "- Concurrency: ${CONCURRENCY}; duration: ${DURATION}; sleep: ${SLEEP_BETWEEN_ROUNDS}s; GET objects: ${GET_OBJECTS}" - echo "- Methods: GET / PUT / MIXED (warp default mix); sizes: ${SIZES[*]}" + echo "- Methods: GET / PUT / MIXED (warp default mix); sizes: ${SIZES[*]-}" + echo "- Rounds: selected=${TOTAL_ROUNDS}; parsed=${PARSED_ROUNDS}; failed=${FAILED_ROUNDS}" echo "" } > "${SUMMARY_MD}" @@ -173,24 +189,34 @@ gen_summary_md() { # ---- Main flow ---- -mkdir -p "${RESULT_DIR}" -log "CONFIG host=${HOST} bucket=${BUCKET} concurrency=${CONCURRENCY} duration=${DURATION} get_objects=${GET_OBJECTS} sleep_between_rounds=${SLEEP_BETWEEN_ROUNDS}s" -printf 'method\tsize\tthroughput\tobj_per_s\treq_avg\treq_p50\treq_p90\treq_p99\tttfb_avg\tttfb_p99\tttfb_worst\n' > "${SUMMARY_TSV}" +mkdir -p "${RESULT_DIR}" || exit 1 +log "CONFIG host=${HOST} bucket=${BUCKET} concurrency=${CONCURRENCY} duration=${DURATION} get_objects=${GET_OBJECTS} sleep_between_rounds=${SLEEP_BETWEEN_ROUNDS}s" || exit 1 +printf 'method\tsize\tthroughput\tobj_per_s\treq_avg\treq_p50\treq_p90\treq_p99\tttfb_avg\tttfb_p99\tttfb_worst\n' > "${SUMMARY_TSV}" || exit 1 + +if [[ ${TOTAL_ROUNDS} -eq 0 ]]; then + log "NO_ROUNDS_SELECTED" + gen_summary_md + exit 1 +fi if [[ "${1:-}" == "--parse-only" ]]; then for method in "${METHODS[@]}"; do for size in "${SIZES[@]}"; do outfile="${RESULT_DIR}/${method}_${size}.txt" - if [[ -s "${outfile}" ]]; then - parse_round "${method}" "${size}" "${outfile}" + if [[ -s "${outfile}" ]] && parse_round "${method}" "${size}" "${outfile}"; then + PARSED_ROUNDS=$((PARSED_ROUNDS + 1)) + else + FAILED_ROUNDS=$((FAILED_ROUNDS + 1)) + log "PARSE_FAILED method=${method} size=${size} file=${outfile}" fi done done - gen_summary_md + gen_summary_md || exit 1 echo "parsed from ${RESULT_DIR}" echo "" - cat "${SUMMARY_MD}" - exit 0 + cat "${SUMMARY_MD}" || exit 1 + [[ ${FAILED_ROUNDS} -eq 0 && ${PARSED_ROUNDS} -eq ${TOTAL_ROUNDS} ]] + exit $? fi for method in "${METHODS[@]}"; do @@ -214,16 +240,20 @@ for method in "${METHODS[@]}"; do --concurrent "${CONCURRENCY}" \ --duration "${DURATION}" \ --obj.size "${size}" \ - "${extra_args[@]}" \ + ${extra_args[@]+"${extra_args[@]}"} \ --no-color 2>&1 | tee "${outfile}" - rc=${PIPESTATUS[0]} + round_status=("${PIPESTATUS[@]}") + rc=${round_status[0]} + capture_rc=${round_status[1]} end_epoch=$(date +%s) + printf '\nRustFS round status: warp=%s capture=%s\n' "${rc}" "${capture_rc}" >> "${outfile}" || capture_rc=1 - if [[ ${rc} -eq 0 ]]; then - parse_round "${method}" "${size}" "${outfile}" + if [[ ${rc} -eq 0 && ${capture_rc} -eq 0 ]] && parse_round "${method}" "${size}" "${outfile}"; then + PARSED_ROUNDS=$((PARSED_ROUNDS + 1)) log "END round=${ROUND}/${TOTAL_ROUNDS} method=${method} size=${size} rc=${rc} elapsed=$((end_epoch - start_epoch))s parsed=ok" else - log "END round=${ROUND}/${TOTAL_ROUNDS} method=${method} size=${size} rc=${rc} elapsed=$((end_epoch - start_epoch))s parsed=skipped" + FAILED_ROUNDS=$((FAILED_ROUNDS + 1)) + log "END round=${ROUND}/${TOTAL_ROUNDS} method=${method} size=${size} rc=${rc} capture_rc=${capture_rc} elapsed=$((end_epoch - start_epoch))s parsed=failed" fi if [[ ${ROUND} -lt ${TOTAL_ROUNDS} ]]; then @@ -233,8 +263,9 @@ for method in "${METHODS[@]}"; do done done -gen_summary_md -log "ALL_ROUNDS_COMPLETE summary_tsv=${SUMMARY_TSV} summary_md=${SUMMARY_MD}" +gen_summary_md || exit 1 +log "ALL_ROUNDS_COMPLETE selected=${TOTAL_ROUNDS} parsed=${PARSED_ROUNDS} failed=${FAILED_ROUNDS} summary_tsv=${SUMMARY_TSV} summary_md=${SUMMARY_MD}" || exit 1 echo "" echo "==== Results summary ====" -cat "${SUMMARY_MD}" +cat "${SUMMARY_MD}" || exit 1 +[[ ${FAILED_ROUNDS} -eq 0 && ${PARSED_ROUNDS} -eq ${TOTAL_ROUNDS} ]] diff --git a/test_rustfs_performance_testing.py b/test_rustfs_performance_testing.py new file mode 100644 index 00000000..a3fba813 --- /dev/null +++ b/test_rustfs_performance_testing.py @@ -0,0 +1,170 @@ +#!/usr/bin/env python3 +"""Benchmark result contracts exercised with a local fake warp executable.""" + +import csv +import os +from pathlib import Path +import shutil +import subprocess +import tempfile +import unittest + + +class PerformanceResultTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.results = self.root / "results" + self.shims = self.root / "fake-tools.sh" + self.shims.write_text(r'''warp() { + local method="$1" size="" objects=0 mode="${FAKE_WARP_MODE:-success}" omit="${FAKE_WARP_OMIT:-}" + shift + while [[ $# -gt 0 ]]; do + case "$1" in + --obj.size) size="$2"; shift;; + --objects) objects=1; shift;; + esac + shift + done + printf '%s %s\n' "$method" "$size" >> "$FAKE_WARP_CALLS" + [[ "$method" == put && "$objects" == 0 || "$method" != put && "$objects" == 1 ]] || return 21 + printf 'diagnostic %s %s\n' "$method" "$size" >&2 + if [[ "$mode" == fail || "$mode" == mixed-failure && "$method" == put ]]; then return 42; fi + local sections=() section text + case "$method" in + get) sections=(GET);; + put) sections=(PUT);; + mixed) sections=(Total GET PUT);; + *) return 2;; + esac + for section in "${sections[@]}"; do + printf 'Report: %s. Concurrency: 64. Ran: 1s\n' "$section" + if [[ "$mode" != missing-throughput ]]; then + text=' * Average: 1.25 MiB/s, 1280.0 obj/s' + printf '%s\n' "${text//"$omit"/}" + fi + if [[ "$mode" == missing-requests || "$method" == mixed && ( "$mode" == missing-mixed-put && "$section" == PUT || "$mode" == missing-mixed-get && "$section" == GET ) ]]; then continue; fi + text=' * Reqs: Avg: 1.2ms, 50%: 1.1ms, 90%: 1.5ms, 99%: 2.1ms, Fastest: 1ms, Slowest: 3ms' + printf '%s\n' "${text//"$omit"/}" + done + [[ "$mode" != failure-with-metrics ]] || return 42 +} +''') + self.calls = self.root / "calls" + self.env = dict(os.environ, BASH_ENV=str(self.shims), FAKE_WARP_CALLS=str(self.calls), WARP_RESULT_DIR=str(self.results), WARP_HOST="127.0.0.1:1", WARP_ACCESS_KEY="test-access", WARP_SECRET_KEY="test-secret", WARP_METHODS="get,put,mixed", WARP_SIZES="1KiB,4KiB", WARP_SLEEP_BETWEEN_ROUNDS="0", WARP_DURATION="1s") + self.script = Path(__file__).with_name("rustfs_performance_testing.sh") + + def run_benchmark(self, mode="success", **overrides): + result = subprocess.run(["bash", str(self.script)], cwd=self.root, env=dict(self.env, FAKE_WARP_MODE=mode, **overrides), capture_output=True, text=True) + with (self.results / "summary.tsv").open() as stream: + rows = list(csv.DictReader(stream, delimiter="\t")) + return result, rows + + def assert_diagnostics(self, expected): + self.assertEqual(len(self.calls.read_text().splitlines()), expected) + outputs = list(self.results.glob("*.txt")) + self.assertEqual(len(outputs), expected) + for output in outputs: + self.assertIn("diagnostic", output.read_text()) + self.assertTrue((self.results / "summary.md").is_file()) + + def test_success_keeps_selected_matrix_and_optional_ttfb(self): + result, rows = self.run_benchmark() + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(len(rows), 6) + self.assert_diagnostics(6) + self.assertIn("parsed=6", (self.results / "master.log").read_text()) + + def test_all_failed_rounds_return_failure_after_collecting_every_round(self): + result, rows = self.run_benchmark("fail") + self.assertNotEqual(result.returncode, 0) + self.assertEqual(rows, []) + self.assert_diagnostics(6) + self.assertEqual((self.results / "master.log").read_text().count("rc=42"), 6) + self.assertIn("failed=6", (self.results / "summary.md").read_text()) + + def test_failure_between_successes_remains_failed(self): + result, rows = self.run_benchmark("mixed-failure") + self.assertNotEqual(result.returncode, 0) + self.assertEqual([row["method"] for row in rows], ["get", "get", "mixed", "mixed"]) + self.assert_diagnostics(6) + self.assertIn("failed=2", (self.results / "summary.md").read_text()) + + def test_failed_output_capture_is_not_a_successful_round(self): + with self.shims.open("a") as shims: + shims.write(f'\ntee() {{ "{shutil.which("tee")}" "$@" || return 1; case "$*" in *.txt) return 23;; esac; }}\n') + result, rows = self.run_benchmark() + self.assertNotEqual(result.returncode, 0) + self.assertEqual(rows, []) + self.assert_diagnostics(6) + self.assertEqual((self.results / "master.log").read_text().count("capture_rc=23"), 6) + + def test_zero_exit_with_missing_metrics_is_not_a_result(self): + for mode in ("missing-throughput", "missing-requests", "missing-mixed-get", "missing-mixed-put"): + with self.subTest(mode=mode): + self.calls.unlink(missing_ok=True) + result, rows = self.run_benchmark(mode) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(len(rows), 4 if mode.startswith("missing-mixed-") else 0) + self.assert_diagnostics(6) + + def test_each_required_single_method_metric_is_checked(self): + for value in ("1.25 MiB/s", "1280.0", "1.2ms", "1.1ms", "1.5ms", "2.1ms"): + with self.subTest(missing=value): + result, rows = self.run_benchmark(FAKE_WARP_OMIT=value, WARP_METHODS="get", WARP_SIZES="1KiB") + self.assertNotEqual(result.returncode, 0) + self.assertEqual(rows, []) + + def test_zero_selection_is_not_success(self): + for selection in ({"WARP_METHODS": ", ,"}, {"WARP_SIZES": " , "}): + with self.subTest(selection=selection): + result, rows = self.run_benchmark(**selection) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(rows, []) + self.assertFalse(self.calls.exists()) + self.assertIn("selected=0", (self.results / "summary.md").read_text()) + + def test_one_round_and_default_full_matrix_are_valid_selections(self): + result, rows = self.run_benchmark(WARP_METHODS="get", WARP_SIZES="1KiB") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(len(rows), 1) + self.calls.unlink() + result, rows = self.run_benchmark(WARP_METHODS="all", WARP_SIZES="all") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(len(rows), 30) + self.assert_diagnostics(30) + + def test_parse_only_rejects_missing_or_incomplete_selected_results(self): + result, _ = self.run_benchmark() + self.assertEqual(result.returncode, 0, result.stderr) + command = ["bash", str(self.script), "--parse-only", str(self.results)] + parsed = subprocess.run(command, cwd=self.root, env=self.env, capture_output=True, text=True) + self.assertEqual(parsed.returncode, 0, parsed.stderr) + (self.results / "get_1KiB.txt").unlink() + parsed = subprocess.run(command, cwd=self.root, env=self.env, capture_output=True, text=True) + self.assertNotEqual(parsed.returncode, 0) + self.assertIn("failed=1", (self.results / "summary.md").read_text()) + + def test_parse_only_cannot_certify_failed_or_unverified_metrics(self): + result, rows = self.run_benchmark("failure-with-metrics") + self.assertNotEqual(result.returncode, 0) + self.assertEqual(rows, []) + command = ["bash", str(self.script), "--parse-only", str(self.results)] + parsed = subprocess.run(command, cwd=self.root, env=self.env, capture_output=True, text=True) + self.assertNotEqual(parsed.returncode, 0) + self.assertIn("failed=6", (self.results / "summary.md").read_text()) + result, _ = self.run_benchmark() + self.assertEqual(result.returncode, 0, result.stderr) + output = self.results / "get_1KiB.txt" + original = output.read_text() + for terminal in ("", "RustFS round status: warp=0 capture=", "RustFS round status: warp=0 capture=0\ntruncated"): + with self.subTest(terminal=terminal): + output.write_text(original.replace("RustFS round status: warp=0 capture=0\n", terminal)) + parsed = subprocess.run(command, cwd=self.root, env=self.env, capture_output=True, text=True) + self.assertNotEqual(parsed.returncode, 0) + self.assertIn("failed=1", (self.results / "summary.md").read_text()) + + +if __name__ == "__main__": + unittest.main() From 02d4c196f9095d521a6e902d44bde349de39693f Mon Sep 17 00:00:00 2001 From: overtrue Date: Sat, 5 Sep 2026 21:32:18 +0800 Subject: [PATCH 2/2] fix(performance): report the selected benchmark methods --- rustfs_performance_testing.sh | 2 +- test_rustfs_performance_testing.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/rustfs_performance_testing.sh b/rustfs_performance_testing.sh index c6868008..0b6ec982 100755 --- a/rustfs_performance_testing.sh +++ b/rustfs_performance_testing.sh @@ -155,7 +155,7 @@ gen_summary_md() { echo "- Date: $(date -u '+%Y-%m-%d %H:%M:%S UTC')" echo "- Target: ${HOST}" echo "- Concurrency: ${CONCURRENCY}; duration: ${DURATION}; sleep: ${SLEEP_BETWEEN_ROUNDS}s; GET objects: ${GET_OBJECTS}" - echo "- Methods: GET / PUT / MIXED (warp default mix); sizes: ${SIZES[*]-}" + echo "- Methods: ${METHODS[*]-}; sizes: ${SIZES[*]-}" echo "- Rounds: selected=${TOTAL_ROUNDS}; parsed=${PARSED_ROUNDS}; failed=${FAILED_ROUNDS}" echo "" } > "${SUMMARY_MD}" diff --git a/test_rustfs_performance_testing.py b/test_rustfs_performance_testing.py index a3fba813..795e45d3 100644 --- a/test_rustfs_performance_testing.py +++ b/test_rustfs_performance_testing.py @@ -129,11 +129,13 @@ def test_one_round_and_default_full_matrix_are_valid_selections(self): result, rows = self.run_benchmark(WARP_METHODS="get", WARP_SIZES="1KiB") self.assertEqual(result.returncode, 0, result.stderr) self.assertEqual(len(rows), 1) + self.assertIn("- Methods: get; sizes: 1KiB\n", (self.results / "summary.md").read_text()) self.calls.unlink() result, rows = self.run_benchmark(WARP_METHODS="all", WARP_SIZES="all") self.assertEqual(result.returncode, 0, result.stderr) self.assertEqual(len(rows), 30) self.assert_diagnostics(30) + self.assertIn("- Methods: get put mixed; sizes: ", (self.results / "summary.md").read_text()) def test_parse_only_rejects_missing_or_incomplete_selected_results(self): result, _ = self.run_benchmark()