fix(test): make VM failure reruns deterministic

rustfs-security-test.sh

@@ -39,6 +39,7 @@
 #   RUSTFS_NODES / RUSTFS_SSH_USER          shared test VMs (default vm000..002 / azureuser)
 #   RUSTFS_API_ENDPOINT                     override the topology endpoint
 #   REPORT_FILE                             markdown report path (default /tmp/rustfs-security-report.md)
+#   LOG_FILE                                append suite output to this file
 set -Eeuo pipefail
 
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
@@ -104,6 +105,7 @@ API_REQUEST_TIMEOUT=120
 ADMIN_API_CODE_FILE="${TEST_TMP}/api-code"
 
 REPORT_FILE="${REPORT_FILE:-/tmp/rustfs-security-report.md}"
+LOG_FILE="${LOG_FILE:-}"
 REPORT_SECTIONS_FILE="${TEST_TMP}/report-sections.md"
 RUSTFS_VERSION_INFO=""
 
@@ -1046,6 +1048,7 @@ parse_args() {
       --validation-artifacts-dir) [ "$#" -ge 2 ] || die "--validation-artifacts-dir requires a value"; VALIDATION_ARTIFACTS_DIR="$2"; shift ;;
       --candidate-manifest) [ "$#" -ge 2 ] || die "--candidate-manifest requires a value"; VALIDATION_CANDIDATE_MANIFEST="$2"; shift ;;
       --testing-commit) [ "$#" -ge 2 ] || die "--testing-commit requires a value"; VALIDATION_TESTING_COMMIT="$2"; shift ;;
+      --log-file) [ "$#" -ge 2 ] || die "--log-file requires a value"; LOG_FILE="$2"; shift ;;
       --keep-env) KEEP_ENV=1 ;;
       -y) ASSUME_YES=1 ;;
       --dry-run) DRY_RUN=1 ;;
@@ -1169,6 +1172,27 @@ main() {
   echo "security suite: ${PASS} passed, ${FAIL} failed, ${SKIP} skipped"
 }
 
-if [ "${RUSTFS_SECURITY_SCRIPT_SOURCE_ONLY:-0}" != 1 ]; then
+run_main() {
+  local log_file="" index
+  local -a args=("$@")
+  for ((index=0; index<${#args[@]}; index++)); do
+    if [ "${args[index]}" = "--log-file" ]; then
+      log_file="${args[index+1]:-}"
+    fi
+  done
+  if [ -n "${log_file}" ]; then
+    command -v tee >/dev/null 2>&1 || die "missing command: tee (required by --log-file)"
+    mkdir -p "$(dirname "${log_file}")"
+    set +e
+    main "$@" 2>&1 | tee -a "${log_file}"
+    local suite_rc=${PIPESTATUS[0]} tee_rc=${PIPESTATUS[1]}
+    set -e
+    [ "${suite_rc}" -eq 0 ] || exit "${suite_rc}"
+    exit "${tee_rc}"
+  fi
   main "$@"
+}
+
+if [ "${RUSTFS_SECURITY_SCRIPT_SOURCE_ONLY:-0}" != 1 ]; then
+  run_main "$@"
 fi

rustfs-table-test.sh

@@ -168,9 +168,14 @@ setup_environment() {
   local v
   for v in "${NODES[@]}"; do
     remote_root "${v}" <<EOF || die "install failed on ${v}"
-set -e
-curl -fsSL --retry 3 -o /tmp/rustfs-table.deb '${PACKAGE_URL}'
-dpkg -i --force-confold /tmp/rustfs-table.deb >/dev/null
+set -euo pipefail
+pkg_dir="/data/rustfs1"
+pkg_tmp="\${pkg_dir}/.rustfs-table-${RUN_ID}.deb"
+mkdir -p "\${pkg_dir}"
+trap 'rm -f "\${pkg_tmp}"' EXIT
+curl --fail --show-error --location --retry 3 --retry-all-errors \
+  --connect-timeout 15 --max-time 600 -o "\${pkg_tmp}" '${PACKAGE_URL}'
+dpkg -i --force-confold "\${pkg_tmp}" >/dev/null
 mkdir -p /data/rustfs{1..4}/mnmd
 chown -R rustfs:rustfs /data/rustfs{1..4}/mnmd
 EOF

rustfs-tier-test.sh

@@ -142,7 +142,6 @@ finish_log_process_bounded() {
   for ((i=0; i<ticks; i++)); do
     if ! log_capture_job_is_active "${pid}"; then
       if reap_log_process_if_done "${pid}"; then process_status=0; else process_status=$?; fi
-      [ "${process_status}" -lt 124 ] || return 124
       [ "${process_status}" -ne 0 ] || process_status="${stop_status}"
       return "${process_status}"
     fi
@@ -151,7 +150,6 @@ finish_log_process_bounded() {
 
   if ! log_capture_job_is_active "${pid}"; then
     if reap_log_process_if_done "${pid}"; then process_status=0; else process_status=$?; fi
-    [ "${process_status}" -lt 124 ] || return 124
     [ "${process_status}" -ne 0 ] || process_status="${stop_status}"
     return "${process_status}"
   fi
@@ -161,10 +159,9 @@ finish_log_process_bounded() {
     if ! log_capture_job_is_active "${pid}"; then
       # The helper exited on TERM. Harvest the real status: a graceful 0 or
       # stop_status means the log is intact and the escalation was just slow
-      # politeness, not a failure. Hardcoding 124 here turned all-green runs
-      # red (09-21 run 35560597650). Signal deaths (>=128) stay 124.
+      # politeness, not a failure. Keep signal deaths distinct from the
+      # timeout sentinel so a crashed logger cannot be hidden by the suite.
       if reap_log_process_if_done "${pid}"; then process_status=0; else process_status=$?; fi
-      [ "${process_status}" -lt 128 ] || process_status=124
       [ "${process_status}" -ne 0 ] || process_status="${stop_status}"
       return "${process_status}"
     fi
@@ -543,7 +540,22 @@ run_bounded() {
 run_hard_bounded() {
   local seconds="$1"
   shift
-  timeout --signal=KILL "${seconds}" "$@"
+  python3 - "$seconds" "$@" <<'PY'
+import os
+import signal
+import subprocess
+import sys
+
+timeout = float(sys.argv[1])
+process = subprocess.Popen(sys.argv[2:], start_new_session=True)
+try:
+    status = process.wait(timeout=timeout)
+except subprocess.TimeoutExpired:
+    os.killpg(process.pid, signal.SIGKILL)
+    process.wait()
+    raise SystemExit(137)
+raise SystemExit(status if status >= 0 else 128 - status)
+PY
 }
 
 bounded_safe_excerpt() {

rustfs_performance_test.sh

@@ -100,6 +100,8 @@ WARP_CONCURRENCY="${RUSTFS_WARP_CONCURRENCY:-64}"
 WARP_DURATION="${RUSTFS_WARP_DURATION:-5m}"
 WARP_GET_OBJECTS="${RUSTFS_WARP_GET_OBJECTS:-2500}"
 WARP_SLEEP="${RUSTFS_WARP_SLEEP:-60}"
+WARP_ROUND_TIMEOUT="${RUSTFS_WARP_ROUND_TIMEOUT:-900}"
+WARP_MAX_CONSECUTIVE_FAILURES="${RUSTFS_WARP_MAX_CONSECUTIVE_FAILURES:-0}"
 # Manual method/size selection (passed through to the benchmark runner; empty = full run)
 WARP_METHODS="${RUSTFS_WARP_METHODS:-}"
 WARP_SIZES="${RUSTFS_WARP_SIZES:-}"
@@ -476,6 +478,8 @@ step5_benchmark() {
   WARP_DURATION="${WARP_DURATION}" \
   WARP_GET_OBJECTS="${WARP_GET_OBJECTS}" \
   WARP_SLEEP_BETWEEN_ROUNDS="${WARP_SLEEP}" \
+  WARP_ROUND_TIMEOUT="${WARP_ROUND_TIMEOUT}" \
+  WARP_MAX_CONSECUTIVE_FAILURES="${WARP_MAX_CONSECUTIVE_FAILURES}" \
   WARP_METHODS="${WARP_METHODS}" \
   WARP_SIZES="${WARP_SIZES}" \
   WARP_RESULT_DIR="${RESULT_DIR}" \
@@ -552,6 +556,9 @@ Options:
   --bench-script PATH       Benchmark runner (default: Obsidian Vault rustfs-performance-testing.sh)
   --result-dir DIR          Benchmark result directory
   --warp-duration DUR       warp duration per round (default 5m)
+  --warp-round-timeout N    hard timeout per round in seconds (default 900)
+  --warp-max-consecutive-failures N
+                            abort after N consecutive failed rounds (default 0 = never)
   --warp-concurrency N      warp concurrency (default 64)
   --ssh-user USER           SSH user (default azureuser)
   --ssh-port PORT           SSH port (default 22)
@@ -617,6 +624,8 @@ main() {
       --bench-script) BENCH_SCRIPT="$1"; shift ;;
       --result-dir) RESULT_DIR="$1"; shift ;;
       --warp-duration) WARP_DURATION="$1"; shift ;;
+      --warp-round-timeout) WARP_ROUND_TIMEOUT="$1"; shift ;;
+      --warp-max-consecutive-failures) WARP_MAX_CONSECUTIVE_FAILURES="$1"; shift ;;
       --warp-concurrency) WARP_CONCURRENCY="$1"; shift ;;
       --ssh-user) SSH_USER="$1"; shift ;;
       --ssh-port) SSH_PORT="$1"; shift ;;

rustfs_performance_testing.sh

@@ -20,6 +20,7 @@
 #   WARP_CONCURRENCY WARP_DURATION WARP_GET_OBJECTS WARP_SLEEP_BETWEEN_ROUNDS
 #   WARP_RESULT_DIR
 #   WARP_METHODS WARP_SIZES   # restrict methods/sizes (comma/space separated); empty = full run
+#   WARP_ROUND_TIMEOUT WARP_MAX_CONSECUTIVE_FAILURES
 
 set -u -o pipefail
 
@@ -31,6 +32,8 @@ CONCURRENCY="${WARP_CONCURRENCY:-64}"
 DURATION="${WARP_DURATION:-5m}"
 GET_OBJECTS="${WARP_GET_OBJECTS:-2500}"
 SLEEP_BETWEEN_ROUNDS="${WARP_SLEEP_BETWEEN_ROUNDS:-60}"
+ROUND_TIMEOUT="${WARP_ROUND_TIMEOUT:-900}"
+MAX_CONSECUTIVE_FAILURES="${WARP_MAX_CONSECUTIVE_FAILURES:-0}"
 RESULT_DIR="${WARP_RESULT_DIR:-$(pwd)/warp-bench-results-$(date +%Y%m%d-%H%M%S)}"
 
 if [ -n "${WARP_SIZES:-}" ] && [ "${WARP_SIZES}" != "all" ] && [ "${WARP_SIZES}" != "ALL" ]; then
@@ -47,6 +50,7 @@ TOTAL_ROUNDS=$(( ${#METHODS[@]} * ${#SIZES[@]} ))
 ROUND=0
 PARSED_ROUNDS=0
 FAILED_ROUNDS=0
+CONSECUTIVE_FAILURES=0
 
 # --parse-only <result-dir>: parse an existing result dir (${method}_${size}.txt) without running
 # Only round files with a successful terminal execution status can be certified.
@@ -231,8 +235,7 @@ for method in "${METHODS[@]}"; do
       extra_args=(--objects "${GET_OBJECTS}")
     fi
 
-    start_epoch=$(date +%s)
-    warp "${method}" \
+    warp_args=("${method}" \
       --host "${HOST}" \
       --access-key "${ACCESS_KEY}" \
       --secret-key "${SECRET_KEY}" \
@@ -241,7 +244,14 @@ for method in "${METHODS[@]}"; do
       --duration "${DURATION}" \
       --obj.size "${size}" \
       ${extra_args[@]+"${extra_args[@]}"} \
-      --no-color 2>&1 | tee "${outfile}"
+      --no-color)
+    start_epoch=$(date +%s)
+    if [[ "${ROUND_TIMEOUT}" =~ ^[1-9][0-9]*$ ]]; then
+      timeout --signal=TERM --kill-after=30 "${ROUND_TIMEOUT}" \
+        bash -c 'warp "$@"' bash "${warp_args[@]}" 2>&1 | tee "${outfile}"
+    else
+      warp "${warp_args[@]}" 2>&1 | tee "${outfile}"
+    fi
     round_status=("${PIPESTATUS[@]}")
     rc=${round_status[0]}
     capture_rc=${round_status[1]}
@@ -250,10 +260,17 @@ for method in "${METHODS[@]}"; do
 
     if [[ ${rc} -eq 0 && ${capture_rc} -eq 0 ]] && parse_round "${method}" "${size}" "${outfile}"; then
       PARSED_ROUNDS=$((PARSED_ROUNDS + 1))
+      CONSECUTIVE_FAILURES=0
       log "END round=${ROUND}/${TOTAL_ROUNDS} method=${method} size=${size} rc=${rc} elapsed=$((end_epoch - start_epoch))s parsed=ok"
     else
       FAILED_ROUNDS=$((FAILED_ROUNDS + 1))
+      CONSECUTIVE_FAILURES=$((CONSECUTIVE_FAILURES + 1))
       log "END round=${ROUND}/${TOTAL_ROUNDS} method=${method} size=${size} rc=${rc} capture_rc=${capture_rc} elapsed=$((end_epoch - start_epoch))s parsed=failed"
+      if [[ "${MAX_CONSECUTIVE_FAILURES}" =~ ^[1-9][0-9]*$ ]] \
+        && (( CONSECUTIVE_FAILURES >= MAX_CONSECUTIVE_FAILURES )); then
+        log "ABORT consecutive_failures=${CONSECUTIVE_FAILURES} limit=${MAX_CONSECUTIVE_FAILURES}"
+        break 2
+      fi
     fi
 
     if [[ ${ROUND} -lt ${TOTAL_ROUNDS} ]]; then