fix(performance): preserve failed and incomplete benchmark rounds

.github/workflows/performance-script-tests.yml

@@ -0,0 +1,30 @@
+name: Performance Script Tests
+
+on:
+  pull_request:
+    paths:
+      - rustfs_performance_testing.sh
+      - test_rustfs_performance_testing.py
+      - .github/workflows/performance-script-tests.yml
+  push:
+    branches: [main]
+    paths:
+      - rustfs_performance_testing.sh
+      - test_rustfs_performance_testing.py
+      - .github/workflows/performance-script-tests.yml
+
+permissions:
+  contents: read
+
+jobs:
+  performance-script-tests:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+      - name: Validate shell syntax
+        run: bash -n rustfs_performance_testing.sh
+      - name: Test benchmark result contracts without a server
+        run: python3 test_rustfs_performance_testing.py

rustfs_performance_testing.sh

@@ -45,8 +45,11 @@ else
 fi
 TOTAL_ROUNDS=$(( ${#METHODS[@]} * ${#SIZES[@]} ))
 ROUND=0
+PARSED_ROUNDS=0
+FAILED_ROUNDS=0
 
 # --parse-only <result-dir>: parse an existing result dir (${method}_${size}.txt) without running
+# Only round files with a successful terminal execution status can be certified.
 if [[ "${1:-}" == "--parse-only" && -n "${2:-}" ]]; then
   RESULT_DIR="$2"
 fi
@@ -100,6 +103,10 @@ field() {
 # Parse one round's output and append a row to summary.tsv
 parse_round() {
   local method="$1" size="$2" file="$3"
+  if [[ "$(tail -n 1 "$file")" != "RustFS round status: warp=0 capture=0" ]]; then
+    log "PARSE_FAILED method=${method} size=${size} reason=missing_or_failed_round_status file=${file}"
+    return 1
+  fi
   local line
   if [[ "${method}" == "mixed" ]]; then
     local total get put
@@ -129,6 +136,14 @@ parse_round() {
       "$(field "${stats}" ttfb_p99)" \
       "$(field "${stats}" ttfb_worst)")
   fi
+  # TTFB is optional in warp; throughput and request statistics define a result.
+  if ! printf '%s\n' "${line}" | awk -F '\t' '
+    { last = $1 == "mixed" ? 6 : 8
+      for (i = 3; i <= last; i++) if ($i !~ /[^[:space:]]/) exit 1 }
+  '; then
+    log "PARSE_FAILED method=${method} size=${size} reason=missing_required_metrics file=${file}"
+    return 1
+  fi
   printf '%s\n' "${line}" >> "${SUMMARY_TSV}"
 }
 
@@ -140,7 +155,8 @@ gen_summary_md() {
     echo "- Date: $(date -u '+%Y-%m-%d %H:%M:%S UTC')"
     echo "- Target: ${HOST}"
     echo "- Concurrency: ${CONCURRENCY}; duration: ${DURATION}; sleep: ${SLEEP_BETWEEN_ROUNDS}s; GET objects: ${GET_OBJECTS}"
-    echo "- Methods: GET / PUT / MIXED (warp default mix); sizes: ${SIZES[*]}"
+    echo "- Methods: ${METHODS[*]-}; sizes: ${SIZES[*]-}"
+    echo "- Rounds: selected=${TOTAL_ROUNDS}; parsed=${PARSED_ROUNDS}; failed=${FAILED_ROUNDS}"
     echo ""
   } > "${SUMMARY_MD}"
 
@@ -173,24 +189,34 @@ gen_summary_md() {
 
 # ---- Main flow ----
 
-mkdir -p "${RESULT_DIR}"
-log "CONFIG host=${HOST} bucket=${BUCKET} concurrency=${CONCURRENCY} duration=${DURATION} get_objects=${GET_OBJECTS} sleep_between_rounds=${SLEEP_BETWEEN_ROUNDS}s"
-printf 'method\tsize\tthroughput\tobj_per_s\treq_avg\treq_p50\treq_p90\treq_p99\tttfb_avg\tttfb_p99\tttfb_worst\n' > "${SUMMARY_TSV}"
+mkdir -p "${RESULT_DIR}" || exit 1
+log "CONFIG host=${HOST} bucket=${BUCKET} concurrency=${CONCURRENCY} duration=${DURATION} get_objects=${GET_OBJECTS} sleep_between_rounds=${SLEEP_BETWEEN_ROUNDS}s" || exit 1
+printf 'method\tsize\tthroughput\tobj_per_s\treq_avg\treq_p50\treq_p90\treq_p99\tttfb_avg\tttfb_p99\tttfb_worst\n' > "${SUMMARY_TSV}" || exit 1
+
+if [[ ${TOTAL_ROUNDS} -eq 0 ]]; then
+  log "NO_ROUNDS_SELECTED"
+  gen_summary_md
+  exit 1
+fi
 
 if [[ "${1:-}" == "--parse-only" ]]; then
   for method in "${METHODS[@]}"; do
     for size in "${SIZES[@]}"; do
       outfile="${RESULT_DIR}/${method}_${size}.txt"
-      if [[ -s "${outfile}" ]]; then
-        parse_round "${method}" "${size}" "${outfile}"
+      if [[ -s "${outfile}" ]] && parse_round "${method}" "${size}" "${outfile}"; then
+        PARSED_ROUNDS=$((PARSED_ROUNDS + 1))
+      else
+        FAILED_ROUNDS=$((FAILED_ROUNDS + 1))
+        log "PARSE_FAILED method=${method} size=${size} file=${outfile}"
       fi
     done
   done
-  gen_summary_md
+  gen_summary_md || exit 1
   echo "parsed from ${RESULT_DIR}"
   echo ""
-  cat "${SUMMARY_MD}"
-  exit 0
+  cat "${SUMMARY_MD}" || exit 1
+  [[ ${FAILED_ROUNDS} -eq 0 && ${PARSED_ROUNDS} -eq ${TOTAL_ROUNDS} ]]
+  exit $?
 fi
 
 for method in "${METHODS[@]}"; do
@@ -214,16 +240,20 @@ for method in "${METHODS[@]}"; do
       --concurrent "${CONCURRENCY}" \
       --duration "${DURATION}" \
       --obj.size "${size}" \
-      "${extra_args[@]}" \
+      ${extra_args[@]+"${extra_args[@]}"} \
       --no-color 2>&1 | tee "${outfile}"
-    rc=${PIPESTATUS[0]}
+    round_status=("${PIPESTATUS[@]}")
+    rc=${round_status[0]}
+    capture_rc=${round_status[1]}
     end_epoch=$(date +%s)
+    printf '\nRustFS round status: warp=%s capture=%s\n' "${rc}" "${capture_rc}" >> "${outfile}" || capture_rc=1
 
-    if [[ ${rc} -eq 0 ]]; then
-      parse_round "${method}" "${size}" "${outfile}"
+    if [[ ${rc} -eq 0 && ${capture_rc} -eq 0 ]] && parse_round "${method}" "${size}" "${outfile}"; then
+      PARSED_ROUNDS=$((PARSED_ROUNDS + 1))
       log "END round=${ROUND}/${TOTAL_ROUNDS} method=${method} size=${size} rc=${rc} elapsed=$((end_epoch - start_epoch))s parsed=ok"
     else
-      log "END round=${ROUND}/${TOTAL_ROUNDS} method=${method} size=${size} rc=${rc} elapsed=$((end_epoch - start_epoch))s parsed=skipped"
+      FAILED_ROUNDS=$((FAILED_ROUNDS + 1))
+      log "END round=${ROUND}/${TOTAL_ROUNDS} method=${method} size=${size} rc=${rc} capture_rc=${capture_rc} elapsed=$((end_epoch - start_epoch))s parsed=failed"
     fi
 
     if [[ ${ROUND} -lt ${TOTAL_ROUNDS} ]]; then
@@ -233,8 +263,9 @@ for method in "${METHODS[@]}"; do
   done
 done
 
-gen_summary_md
-log "ALL_ROUNDS_COMPLETE summary_tsv=${SUMMARY_TSV} summary_md=${SUMMARY_MD}"
+gen_summary_md || exit 1
+log "ALL_ROUNDS_COMPLETE selected=${TOTAL_ROUNDS} parsed=${PARSED_ROUNDS} failed=${FAILED_ROUNDS} summary_tsv=${SUMMARY_TSV} summary_md=${SUMMARY_MD}" || exit 1
 echo ""
 echo "==== Results summary ===="
-cat "${SUMMARY_MD}"
+cat "${SUMMARY_MD}" || exit 1
+[[ ${FAILED_ROUNDS} -eq 0 && ${PARSED_ROUNDS} -eq ${TOTAL_ROUNDS} ]]

test_rustfs_performance_testing.py

@@ -0,0 +1,172 @@
+#!/usr/bin/env python3
+"""Benchmark result contracts exercised with a local fake warp executable."""
+
+import csv
+import os
+from pathlib import Path
+import shutil
+import subprocess
+import tempfile
+import unittest
+
+
+class PerformanceResultTests(unittest.TestCase):
+    def setUp(self):
+        self.temp = tempfile.TemporaryDirectory()
+        self.addCleanup(self.temp.cleanup)
+        self.root = Path(self.temp.name)
+        self.results = self.root / "results"
+        self.shims = self.root / "fake-tools.sh"
+        self.shims.write_text(r'''warp() {
+  local method="$1" size="" objects=0 mode="${FAKE_WARP_MODE:-success}" omit="${FAKE_WARP_OMIT:-}"
+  shift
+  while [[ $# -gt 0 ]]; do
+    case "$1" in
+      --obj.size) size="$2"; shift;;
+      --objects) objects=1; shift;;
+    esac
+    shift
+  done
+  printf '%s %s\n' "$method" "$size" >> "$FAKE_WARP_CALLS"
+  [[ "$method" == put && "$objects" == 0 || "$method" != put && "$objects" == 1 ]] || return 21
+  printf 'diagnostic %s %s\n' "$method" "$size" >&2
+  if [[ "$mode" == fail || "$mode" == mixed-failure && "$method" == put ]]; then return 42; fi
+  local sections=() section text
+  case "$method" in
+    get) sections=(GET);;
+    put) sections=(PUT);;
+    mixed) sections=(Total GET PUT);;
+    *) return 2;;
+  esac
+  for section in "${sections[@]}"; do
+    printf 'Report: %s. Concurrency: 64. Ran: 1s\n' "$section"
+    if [[ "$mode" != missing-throughput ]]; then
+      text=' * Average: 1.25 MiB/s, 1280.0 obj/s'
+      printf '%s\n' "${text//"$omit"/}"
+    fi
+    if [[ "$mode" == missing-requests || "$method" == mixed && ( "$mode" == missing-mixed-put && "$section" == PUT || "$mode" == missing-mixed-get && "$section" == GET ) ]]; then continue; fi
+    text=' * Reqs: Avg: 1.2ms, 50%: 1.1ms, 90%: 1.5ms, 99%: 2.1ms, Fastest: 1ms, Slowest: 3ms'
+    printf '%s\n' "${text//"$omit"/}"
+  done
+  [[ "$mode" != failure-with-metrics ]] || return 42
+}
+''')
+        self.calls = self.root / "calls"
+        self.env = dict(os.environ, BASH_ENV=str(self.shims), FAKE_WARP_CALLS=str(self.calls), WARP_RESULT_DIR=str(self.results), WARP_HOST="127.0.0.1:1", WARP_ACCESS_KEY="test-access", WARP_SECRET_KEY="test-secret", WARP_METHODS="get,put,mixed", WARP_SIZES="1KiB,4KiB", WARP_SLEEP_BETWEEN_ROUNDS="0", WARP_DURATION="1s")
+        self.script = Path(__file__).with_name("rustfs_performance_testing.sh")
+
+    def run_benchmark(self, mode="success", **overrides):
+        result = subprocess.run(["bash", str(self.script)], cwd=self.root, env=dict(self.env, FAKE_WARP_MODE=mode, **overrides), capture_output=True, text=True)
+        with (self.results / "summary.tsv").open() as stream:
+            rows = list(csv.DictReader(stream, delimiter="\t"))
+        return result, rows
+
+    def assert_diagnostics(self, expected):
+        self.assertEqual(len(self.calls.read_text().splitlines()), expected)
+        outputs = list(self.results.glob("*.txt"))
+        self.assertEqual(len(outputs), expected)
+        for output in outputs:
+            self.assertIn("diagnostic", output.read_text())
+        self.assertTrue((self.results / "summary.md").is_file())
+
+    def test_success_keeps_selected_matrix_and_optional_ttfb(self):
+        result, rows = self.run_benchmark()
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertEqual(len(rows), 6)
+        self.assert_diagnostics(6)
+        self.assertIn("parsed=6", (self.results / "master.log").read_text())
+
+    def test_all_failed_rounds_return_failure_after_collecting_every_round(self):
+        result, rows = self.run_benchmark("fail")
+        self.assertNotEqual(result.returncode, 0)
+        self.assertEqual(rows, [])
+        self.assert_diagnostics(6)
+        self.assertEqual((self.results / "master.log").read_text().count("rc=42"), 6)
+        self.assertIn("failed=6", (self.results / "summary.md").read_text())
+
+    def test_failure_between_successes_remains_failed(self):
+        result, rows = self.run_benchmark("mixed-failure")
+        self.assertNotEqual(result.returncode, 0)
+        self.assertEqual([row["method"] for row in rows], ["get", "get", "mixed", "mixed"])
+        self.assert_diagnostics(6)
+        self.assertIn("failed=2", (self.results / "summary.md").read_text())
+
+    def test_failed_output_capture_is_not_a_successful_round(self):
+        with self.shims.open("a") as shims:
+            shims.write(f'\ntee() {{ "{shutil.which("tee")}" "$@" || return 1; case "$*" in *.txt) return 23;; esac; }}\n')
+        result, rows = self.run_benchmark()
+        self.assertNotEqual(result.returncode, 0)
+        self.assertEqual(rows, [])
+        self.assert_diagnostics(6)
+        self.assertEqual((self.results / "master.log").read_text().count("capture_rc=23"), 6)
+
+    def test_zero_exit_with_missing_metrics_is_not_a_result(self):
+        for mode in ("missing-throughput", "missing-requests", "missing-mixed-get", "missing-mixed-put"):
+            with self.subTest(mode=mode):
+                self.calls.unlink(missing_ok=True)
+                result, rows = self.run_benchmark(mode)
+                self.assertNotEqual(result.returncode, 0)
+                self.assertEqual(len(rows), 4 if mode.startswith("missing-mixed-") else 0)
+                self.assert_diagnostics(6)
+
+    def test_each_required_single_method_metric_is_checked(self):
+        for value in ("1.25 MiB/s", "1280.0", "1.2ms", "1.1ms", "1.5ms", "2.1ms"):
+            with self.subTest(missing=value):
+                result, rows = self.run_benchmark(FAKE_WARP_OMIT=value, WARP_METHODS="get", WARP_SIZES="1KiB")
+                self.assertNotEqual(result.returncode, 0)
+                self.assertEqual(rows, [])
+
+    def test_zero_selection_is_not_success(self):
+        for selection in ({"WARP_METHODS": ", ,"}, {"WARP_SIZES": " , "}):
+            with self.subTest(selection=selection):
+                result, rows = self.run_benchmark(**selection)
+                self.assertNotEqual(result.returncode, 0)
+                self.assertEqual(rows, [])
+                self.assertFalse(self.calls.exists())
+                self.assertIn("selected=0", (self.results / "summary.md").read_text())
+
+    def test_one_round_and_default_full_matrix_are_valid_selections(self):
+        result, rows = self.run_benchmark(WARP_METHODS="get", WARP_SIZES="1KiB")
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertEqual(len(rows), 1)
+        self.assertIn("- Methods: get; sizes: 1KiB\n", (self.results / "summary.md").read_text())
+        self.calls.unlink()
+        result, rows = self.run_benchmark(WARP_METHODS="all", WARP_SIZES="all")
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertEqual(len(rows), 30)
+        self.assert_diagnostics(30)
+        self.assertIn("- Methods: get put mixed; sizes: ", (self.results / "summary.md").read_text())
+
+    def test_parse_only_rejects_missing_or_incomplete_selected_results(self):
+        result, _ = self.run_benchmark()
+        self.assertEqual(result.returncode, 0, result.stderr)
+        command = ["bash", str(self.script), "--parse-only", str(self.results)]
+        parsed = subprocess.run(command, cwd=self.root, env=self.env, capture_output=True, text=True)
+        self.assertEqual(parsed.returncode, 0, parsed.stderr)
+        (self.results / "get_1KiB.txt").unlink()
+        parsed = subprocess.run(command, cwd=self.root, env=self.env, capture_output=True, text=True)
+        self.assertNotEqual(parsed.returncode, 0)
+        self.assertIn("failed=1", (self.results / "summary.md").read_text())
+
+    def test_parse_only_cannot_certify_failed_or_unverified_metrics(self):
+        result, rows = self.run_benchmark("failure-with-metrics")
+        self.assertNotEqual(result.returncode, 0)
+        self.assertEqual(rows, [])
+        command = ["bash", str(self.script), "--parse-only", str(self.results)]
+        parsed = subprocess.run(command, cwd=self.root, env=self.env, capture_output=True, text=True)
+        self.assertNotEqual(parsed.returncode, 0)
+        self.assertIn("failed=6", (self.results / "summary.md").read_text())
+        result, _ = self.run_benchmark()
+        self.assertEqual(result.returncode, 0, result.stderr)
+        output = self.results / "get_1KiB.txt"
+        original = output.read_text()
+        for terminal in ("", "RustFS round status: warp=0 capture=", "RustFS round status: warp=0 capture=0\ntruncated"):
+            with self.subTest(terminal=terminal):
+                output.write_text(original.replace("RustFS round status: warp=0 capture=0\n", terminal))
+                parsed = subprocess.run(command, cwd=self.root, env=self.env, capture_output=True, text=True)
+                self.assertNotEqual(parsed.returncode, 0)
+                self.assertIn("failed=1", (self.results / "summary.md").read_text())
+
+
+if __name__ == "__main__":
+    unittest.main()