From 3b825f34b2d636a3e9ee9f7d5cc6c5d54c227bbb Mon Sep 17 00:00:00 2001 From: cxymds Date: Wed, 4 Nov 2026 16:27:53 +0800 Subject: [PATCH 1/3] fix(test): make VM failure reruns deterministic --- rustfs-security-test.sh | 8 ++ rustfs-table-test.sh | 11 ++- rustfs-tier-test.sh | 7 +- rustfs_performance_test.sh | 9 ++ rustfs_performance_testing.sh | 23 ++++- rustfs_pool_expand.sh | 113 ++++++++++++++---------- scripts/test_functional_dispatch.py | 33 +++++++ scripts/validation_pool/run_case.sh | 30 +++++-- scripts/validation_registry.py | 5 +- test_rustfs_performance_testing.py | 12 +++ test_rustfs_pool_expand.sh | 54 +++++++++++ test_rustfs_security.py | 6 +- test_rustfs_table.py | 7 ++ test_rustfs_tier.sh | 8 +- test_validation_heal_persistence_mrf.py | 3 +- test_validation_pool.py | 6 ++ validation-migration-review.json | 6 +- 17 files changed, 271 insertions(+), 70 deletions(-) diff --git a/rustfs-security-test.sh b/rustfs-security-test.sh index 217dc89e..d6ba60d8 100755 --- a/rustfs-security-test.sh +++ b/rustfs-security-test.sh @@ -39,6 +39,7 @@ # RUSTFS_NODES / RUSTFS_SSH_USER shared test VMs (default vm000..002 / azureuser) # RUSTFS_API_ENDPOINT override the topology endpoint # REPORT_FILE markdown report path (default /tmp/rustfs-security-report.md) +# LOG_FILE append suite output to this file set -Eeuo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -104,6 +105,7 @@ API_REQUEST_TIMEOUT=120 ADMIN_API_CODE_FILE="${TEST_TMP}/api-code" REPORT_FILE="${REPORT_FILE:-/tmp/rustfs-security-report.md}" +LOG_FILE="${LOG_FILE:-}" REPORT_SECTIONS_FILE="${TEST_TMP}/report-sections.md" RUSTFS_VERSION_INFO="" @@ -1046,6 +1048,7 @@ parse_args() { --validation-artifacts-dir) [ "$#" -ge 2 ] || die "--validation-artifacts-dir requires a value"; VALIDATION_ARTIFACTS_DIR="$2"; shift ;; --candidate-manifest) [ "$#" -ge 2 ] || die "--candidate-manifest requires a value"; VALIDATION_CANDIDATE_MANIFEST="$2"; shift ;; --testing-commit) [ "$#" -ge 2 ] || die "--testing-commit requires a value"; VALIDATION_TESTING_COMMIT="$2"; shift ;; + --log-file) [ "$#" -ge 2 ] || die "--log-file requires a value"; LOG_FILE="$2"; shift ;; --keep-env) KEEP_ENV=1 ;; -y) ASSUME_YES=1 ;; --dry-run) DRY_RUN=1 ;; @@ -1106,6 +1109,11 @@ run_security_cases() { main() { parse_args "$@" + if [ -n "${LOG_FILE}" ]; then + command -v tee >/dev/null 2>&1 || die "missing command: tee (required by --log-file)" + mkdir -p "$(dirname "${LOG_FILE}")" + exec > >(tee -a "${LOG_FILE}") 2>&1 + fi need_cmd ssh "openssh client" need_cmd curl "curl" need_cmd openssl "openssl" diff --git a/rustfs-table-test.sh b/rustfs-table-test.sh index fb98037b..0bf29f31 100755 --- a/rustfs-table-test.sh +++ b/rustfs-table-test.sh @@ -168,9 +168,14 @@ setup_environment() { local v for v in "${NODES[@]}"; do remote_root "${v}" </dev/null +set -euo pipefail +pkg_dir="/data/rustfs1" +pkg_tmp="\${pkg_dir}/.rustfs-table-\${RUN_ID}.deb" +mkdir -p "\${pkg_dir}" +trap 'rm -f "\${pkg_tmp}"' EXIT +curl --fail --show-error --location --retry 3 --retry-all-errors \ + --connect-timeout 15 --max-time 600 -o "\${pkg_tmp}" '${PACKAGE_URL}' +dpkg -i --force-confold "\${pkg_tmp}" >/dev/null mkdir -p /data/rustfs{1..4}/mnmd chown -R rustfs:rustfs /data/rustfs{1..4}/mnmd EOF diff --git a/rustfs-tier-test.sh b/rustfs-tier-test.sh index c91e6dcd..7e220fde 100755 --- a/rustfs-tier-test.sh +++ b/rustfs-tier-test.sh @@ -142,7 +142,6 @@ finish_log_process_bounded() { for ((i=0; i=128) stay 124. + # politeness, not a failure. Keep signal deaths distinct from the + # timeout sentinel so a crashed logger cannot be hidden by the suite. if reap_log_process_if_done "${pid}"; then process_status=0; else process_status=$?; fi - [ "${process_status}" -lt 128 ] || process_status=124 [ "${process_status}" -ne 0 ] || process_status="${stop_status}" return "${process_status}" fi diff --git a/rustfs_performance_test.sh b/rustfs_performance_test.sh index 106ce857..3f1931d9 100755 --- a/rustfs_performance_test.sh +++ b/rustfs_performance_test.sh @@ -100,6 +100,8 @@ WARP_CONCURRENCY="${RUSTFS_WARP_CONCURRENCY:-64}" WARP_DURATION="${RUSTFS_WARP_DURATION:-5m}" WARP_GET_OBJECTS="${RUSTFS_WARP_GET_OBJECTS:-2500}" WARP_SLEEP="${RUSTFS_WARP_SLEEP:-60}" +WARP_ROUND_TIMEOUT="${RUSTFS_WARP_ROUND_TIMEOUT:-900}" +WARP_MAX_CONSECUTIVE_FAILURES="${RUSTFS_WARP_MAX_CONSECUTIVE_FAILURES:-0}" # Manual method/size selection (passed through to the benchmark runner; empty = full run) WARP_METHODS="${RUSTFS_WARP_METHODS:-}" WARP_SIZES="${RUSTFS_WARP_SIZES:-}" @@ -476,6 +478,8 @@ step5_benchmark() { WARP_DURATION="${WARP_DURATION}" \ WARP_GET_OBJECTS="${WARP_GET_OBJECTS}" \ WARP_SLEEP_BETWEEN_ROUNDS="${WARP_SLEEP}" \ + WARP_ROUND_TIMEOUT="${WARP_ROUND_TIMEOUT}" \ + WARP_MAX_CONSECUTIVE_FAILURES="${WARP_MAX_CONSECUTIVE_FAILURES}" \ WARP_METHODS="${WARP_METHODS}" \ WARP_SIZES="${WARP_SIZES}" \ WARP_RESULT_DIR="${RESULT_DIR}" \ @@ -552,6 +556,9 @@ Options: --bench-script PATH Benchmark runner (default: Obsidian Vault rustfs-performance-testing.sh) --result-dir DIR Benchmark result directory --warp-duration DUR warp duration per round (default 5m) + --warp-round-timeout N hard timeout per round in seconds (default 900) + --warp-max-consecutive-failures N + abort after N consecutive failed rounds (default 0 = never) --warp-concurrency N warp concurrency (default 64) --ssh-user USER SSH user (default azureuser) --ssh-port PORT SSH port (default 22) @@ -617,6 +624,8 @@ main() { --bench-script) BENCH_SCRIPT="$1"; shift ;; --result-dir) RESULT_DIR="$1"; shift ;; --warp-duration) WARP_DURATION="$1"; shift ;; + --warp-round-timeout) WARP_ROUND_TIMEOUT="$1"; shift ;; + --warp-max-consecutive-failures) WARP_MAX_CONSECUTIVE_FAILURES="$1"; shift ;; --warp-concurrency) WARP_CONCURRENCY="$1"; shift ;; --ssh-user) SSH_USER="$1"; shift ;; --ssh-port) SSH_PORT="$1"; shift ;; diff --git a/rustfs_performance_testing.sh b/rustfs_performance_testing.sh index 0b6ec982..436a0a28 100755 --- a/rustfs_performance_testing.sh +++ b/rustfs_performance_testing.sh @@ -20,6 +20,7 @@ # WARP_CONCURRENCY WARP_DURATION WARP_GET_OBJECTS WARP_SLEEP_BETWEEN_ROUNDS # WARP_RESULT_DIR # WARP_METHODS WARP_SIZES # restrict methods/sizes (comma/space separated); empty = full run +# WARP_ROUND_TIMEOUT WARP_MAX_CONSECUTIVE_FAILURES set -u -o pipefail @@ -31,6 +32,8 @@ CONCURRENCY="${WARP_CONCURRENCY:-64}" DURATION="${WARP_DURATION:-5m}" GET_OBJECTS="${WARP_GET_OBJECTS:-2500}" SLEEP_BETWEEN_ROUNDS="${WARP_SLEEP_BETWEEN_ROUNDS:-60}" +ROUND_TIMEOUT="${WARP_ROUND_TIMEOUT:-900}" +MAX_CONSECUTIVE_FAILURES="${WARP_MAX_CONSECUTIVE_FAILURES:-0}" RESULT_DIR="${WARP_RESULT_DIR:-$(pwd)/warp-bench-results-$(date +%Y%m%d-%H%M%S)}" if [ -n "${WARP_SIZES:-}" ] && [ "${WARP_SIZES}" != "all" ] && [ "${WARP_SIZES}" != "ALL" ]; then @@ -47,6 +50,7 @@ TOTAL_ROUNDS=$(( ${#METHODS[@]} * ${#SIZES[@]} )) ROUND=0 PARSED_ROUNDS=0 FAILED_ROUNDS=0 +CONSECUTIVE_FAILURES=0 # --parse-only : parse an existing result dir (${method}_${size}.txt) without running # Only round files with a successful terminal execution status can be certified. @@ -231,8 +235,7 @@ for method in "${METHODS[@]}"; do extra_args=(--objects "${GET_OBJECTS}") fi - start_epoch=$(date +%s) - warp "${method}" \ + warp_args=("${method}" \ --host "${HOST}" \ --access-key "${ACCESS_KEY}" \ --secret-key "${SECRET_KEY}" \ @@ -241,7 +244,14 @@ for method in "${METHODS[@]}"; do --duration "${DURATION}" \ --obj.size "${size}" \ ${extra_args[@]+"${extra_args[@]}"} \ - --no-color 2>&1 | tee "${outfile}" + --no-color) + start_epoch=$(date +%s) + if [[ "${ROUND_TIMEOUT}" =~ ^[1-9][0-9]*$ ]]; then + timeout --signal=TERM --kill-after=30 "${ROUND_TIMEOUT}" \ + bash -c 'warp "$@"' bash "${warp_args[@]}" 2>&1 | tee "${outfile}" + else + warp "${warp_args[@]}" 2>&1 | tee "${outfile}" + fi round_status=("${PIPESTATUS[@]}") rc=${round_status[0]} capture_rc=${round_status[1]} @@ -250,10 +260,17 @@ for method in "${METHODS[@]}"; do if [[ ${rc} -eq 0 && ${capture_rc} -eq 0 ]] && parse_round "${method}" "${size}" "${outfile}"; then PARSED_ROUNDS=$((PARSED_ROUNDS + 1)) + CONSECUTIVE_FAILURES=0 log "END round=${ROUND}/${TOTAL_ROUNDS} method=${method} size=${size} rc=${rc} elapsed=$((end_epoch - start_epoch))s parsed=ok" else FAILED_ROUNDS=$((FAILED_ROUNDS + 1)) + CONSECUTIVE_FAILURES=$((CONSECUTIVE_FAILURES + 1)) log "END round=${ROUND}/${TOTAL_ROUNDS} method=${method} size=${size} rc=${rc} capture_rc=${capture_rc} elapsed=$((end_epoch - start_epoch))s parsed=failed" + if [[ "${MAX_CONSECUTIVE_FAILURES}" =~ ^[1-9][0-9]*$ ]] \ + && (( CONSECUTIVE_FAILURES >= MAX_CONSECUTIVE_FAILURES )); then + log "ABORT consecutive_failures=${CONSECUTIVE_FAILURES} limit=${MAX_CONSECUTIVE_FAILURES}" + break 2 + fi fi if [[ ${ROUND} -lt ${TOTAL_ROUNDS} ]]; then diff --git a/rustfs_pool_expand.sh b/rustfs_pool_expand.sh index cd765b28..0f51e258 100755 --- a/rustfs_pool_expand.sh +++ b/rustfs_pool_expand.sh @@ -110,12 +110,13 @@ WARP_BUCKET="test-10mb" WARP_OBJ_SIZE="100MiB" WARP_CONCURRENT="${RUSTFS_WARP_CONCURRENT:-32}" WARP_DURATION="5m" +WARP_ROUNDS="${RUSTFS_WARP_ROUNDS:-1}" # Warp log path; empty = auto-created unique temp file (the runner user may # not be able to write a shared /tmp path owned by another user). WARP_LOG_FILE="${RUSTFS_WARP_LOG_FILE:-}" STORAGE_THRESHOLD=85 # stop writing when usage reaches N% (note suggests 80-85) POLL_INTERVAL=30 # status polling interval (seconds) -WARP_TRANSPORT_FAILURE_PATTERN='502 Bad Gateway|504 Gateway Time-out|HTTP([^0-9]+)?50(2|4)|status([^0-9]+)?50(2|4)|connection (closed|reset|refused)|unexpected EOF|broken pipe|timeout awaiting response headers|context deadline exceeded|i/o timeout|Client\.Timeout exceeded' +WARP_TRANSPORT_FAILURE_PATTERN='502 Bad Gateway|504 Gateway Time-out|HTTP([^0-9]+)?50(2|4)|status([^0-9]+)?50(2|4)|connection (closed|reset|refused)|unexpected EOF|broken pipe|timeout awaiting response headers|context deadline exceeded|i/o timeout|Client\.Timeout exceeded|The service is unavailable|We encountered an internal error|upload error:' # Timeouts (seconds) REBALANCE_TIMEOUT=86400 @@ -248,6 +249,8 @@ normalize_http_endpoint() { resolve_endpoints() { [[ "${WARP_CONCURRENT}" =~ ^[1-9][0-9]*$ ]] \ || die "warp concurrency must be a positive integer: ${WARP_CONCURRENT}" + [[ "${WARP_ROUNDS}" =~ ^[1-9][0-9]*$ ]] \ + || die "warp rounds must be a positive integer: ${WARP_ROUNDS}" [[ "${STORAGEINFO_PROBE_RETRIES}" =~ ^[1-9][0-9]*$ ]] \ || die "storageinfo probe retries must be a positive integer: ${STORAGEINFO_PROBE_RETRIES}" [[ "${STORAGEINFO_PROBE_RETRY_DELAY}" =~ ^[0-9]+$ ]] \ @@ -650,8 +653,14 @@ service_action() { local action="$1" node="$2" log "${node}: systemctl ${action} ${RUSTFS_SERVICE}" if [ "${DRY_RUN}" -eq 1 ]; then return 0; fi + local remote_command + if [ "${action}" = "stop" ]; then + remote_command="if ! systemctl cat ${RUSTFS_SERVICE} >/dev/null 2>&1; then echo '${RUSTFS_SERVICE} is not installed; stop skipped'; exit 0; fi; if [ \"\$(id -u)\" -ne 0 ]; then sudo -n systemctl stop ${RUSTFS_SERVICE}; else systemctl stop ${RUSTFS_SERVICE}; fi" + else + remote_command="if [ \"\$(id -u)\" -ne 0 ]; then sudo -n systemctl ${action} ${RUSTFS_SERVICE}; else systemctl ${action} ${RUSTFS_SERVICE}; fi" + fi if ! ssh "${SSH_OPTS[@]}" "${SSH_USER}@${node}" \ - "if [ \"\$(id -u)\" -ne 0 ]; then sudo -n systemctl ${action} ${RUSTFS_SERVICE}; else systemctl ${action} ${RUSTFS_SERVICE}; fi"; then + "${remote_command}"; then if [ "${action}" = "start" ]; then diagnose_node_start_failure "${node}" fi @@ -895,7 +904,8 @@ monitor_storage() { log "current storage usage: ${usage}%" awk -v u="${usage}" -v t="${threshold}" 'BEGIN{exit !(u >= t)}' && break if [ -n "${warp_pid}" ] && ! kill -0 "${warp_pid}" 2>/dev/null; then - die "warp finished at ${usage}%, below the required storage threshold of ${threshold}%" + warn "warp finished at ${usage}%, below the required storage threshold of ${threshold}%" + return 3 fi sleep "${POLL_INTERVAL}" done @@ -1282,57 +1292,68 @@ step4_write_data() { verify_pools 1 if [ "${WITH_WARP}" -eq 1 ]; then need_cmd warp "warp benchmark tool" - confirm "About to run warp writes in the background (bucket=${WARP_BUCKET}, duration=${WARP_DURATION}) and monitor usage up to ${STORAGE_THRESHOLD}%. Continue?" + confirm "About to run up to ${WARP_ROUNDS} warp write round(s) against bucket=${WARP_BUCKET} and monitor usage up to ${STORAGE_THRESHOLD}%. Continue?" if [ "${DRY_RUN}" -eq 1 ]; then log "DRY-RUN: warp put --host $(endpoint_authority "${WARP_ENDPOINT}") --bucket ${WARP_BUCKET} --concurrent ${WARP_CONCURRENT} ... (background)" - log "DRY-RUN: monitoring storage usage until ${STORAGE_THRESHOLD}%" + log "DRY-RUN: allowing ${WARP_ROUNDS} round(s) to reach ${STORAGE_THRESHOLD}%" return 0 fi - local warp_log warp_pid + local warp_log warp_pid round monitor_rc target_reached=0 warp_stop_waited warp_log="${WARP_LOG_FILE:-$(mktemp "${TMPDIR:-/tmp}/rustfs-warp.XXXXXX.log")}" - log "starting warp writes against ${WARP_ENDPOINT} (background), log: ${warp_log}" - warp put --host "$(endpoint_authority "${WARP_ENDPOINT}")" \ - --bucket "${WARP_BUCKET}" \ - --access-key "${ACCESS_KEY}" \ - --secret-key "${SECRET_KEY}" \ - --obj.size "${WARP_OBJ_SIZE}" \ - --concurrent "${WARP_CONCURRENT}" \ - --noprefix --duration "${WARP_DURATION}" --noclear >"${warp_log}" 2>&1 & - warp_pid=$! - ACTIVE_WARP_PID="${warp_pid}" - log "warp PID=${warp_pid}" - monitor_storage "${STORAGE_THRESHOLD}" "${warp_pid}" - # Graceful stop first (SIGINT lets warp finish/abort in-flight multipart - # uploads), then SIGTERM/KILL as fallback; see rustfs/backlog#2063. - kill -INT "${warp_pid}" 2>/dev/null || true - local warp_stop_waited=0 - while kill -0 "${warp_pid}" 2>/dev/null && [ "${warp_stop_waited}" -lt 60 ]; do - sleep 1 - warp_stop_waited=$((warp_stop_waited + 1)) - done - if kill -0 "${warp_pid}" 2>/dev/null; then - warn "warp did not exit after SIGINT; sending SIGTERM" - kill -TERM "${warp_pid}" 2>/dev/null || true + : >"${warp_log}" + for ((round=1; round<=WARP_ROUNDS; round++)); do + log "starting warp round ${round}/${WARP_ROUNDS} against ${WARP_ENDPOINT} (background), log: ${warp_log}" + warp put --host "$(endpoint_authority "${WARP_ENDPOINT}")" \ + --bucket "${WARP_BUCKET}" \ + --access-key "${ACCESS_KEY}" \ + --secret-key "${SECRET_KEY}" \ + --obj.size "${WARP_OBJ_SIZE}" \ + --concurrent "${WARP_CONCURRENT}" \ + --noprefix --duration "${WARP_DURATION}" --noclear >>"${warp_log}" 2>&1 & + warp_pid=$! + ACTIVE_WARP_PID="${warp_pid}" + log "warp PID=${warp_pid}" + if monitor_storage "${STORAGE_THRESHOLD}" "${warp_pid}"; then + target_reached=1 + else + monitor_rc=$? + fi + # Graceful stop first (SIGINT lets warp finish/abort in-flight multipart + # uploads), then SIGTERM/KILL as fallback; see rustfs/backlog#2063. + kill -INT "${warp_pid}" 2>/dev/null || true warp_stop_waited=0 - while kill -0 "${warp_pid}" 2>/dev/null && [ "${warp_stop_waited}" -lt 30 ]; do + while kill -0 "${warp_pid}" 2>/dev/null && [ "${warp_stop_waited}" -lt 60 ]; do sleep 1 warp_stop_waited=$((warp_stop_waited + 1)) done - fi - if kill -0 "${warp_pid}" 2>/dev/null; then - warn "warp still running; sending SIGKILL" - kill -KILL "${warp_pid}" 2>/dev/null || true - fi - wait "${warp_pid}" 2>/dev/null || true - ACTIVE_WARP_PID="" - log "warp stopped" - local warp_transport_clean=1 - report_warp_errors "${warp_log}" || warp_transport_clean=0 - # Safety net: abort any incomplete multipart uploads left in the test - # bucket immediately, so no orphan can block the later decommission gate. - abort_stale_multipart_uploads "${WARP_BUCKET}" - [ "${warp_transport_clean}" -eq 1 ] \ - || die "warp observed an Nginx 502/504, timeout, or transport failure; see ${warp_log}" + if kill -0 "${warp_pid}" 2>/dev/null; then + warn "warp did not exit after SIGINT; sending SIGTERM" + kill -TERM "${warp_pid}" 2>/dev/null || true + warp_stop_waited=0 + while kill -0 "${warp_pid}" 2>/dev/null && [ "${warp_stop_waited}" -lt 30 ]; do + sleep 1 + warp_stop_waited=$((warp_stop_waited + 1)) + done + fi + if kill -0 "${warp_pid}" 2>/dev/null; then + warn "warp still running; sending SIGKILL" + kill -KILL "${warp_pid}" 2>/dev/null || true + fi + wait "${warp_pid}" 2>/dev/null || true + ACTIVE_WARP_PID="" + log "warp round ${round} stopped" + abort_stale_multipart_uploads "${WARP_BUCKET}" + report_warp_errors "${warp_log}" \ + || die "warp observed a service or transport failure; see ${warp_log}" + [ "${target_reached}" -eq 1 ] && break + [ "${monitor_rc:-0}" -eq 3 ] \ + || die "warp monitor failed unexpectedly (status=${monitor_rc:-unknown})" + [ "${round}" -lt "${WARP_ROUNDS}" ] \ + || die "warp exhausted ${WARP_ROUNDS} round(s) below the required storage threshold of ${STORAGE_THRESHOLD}%" + log "storage threshold not reached; continuing with another warp round" + done + [ "${target_reached}" -eq 1 ] \ + || die "warp did not reach the required storage threshold of ${STORAGE_THRESHOLD}%" else cat </dev/null 2>&1"; then + service_action stop "${node}" + else + log "${node}: ${RUSTFS_SERVICE} is not installed; stop skipped" + fi + done for node in "${NODES[@]}"; do run_remote "${node}" <<'REMOTE' set -euo pipefail @@ -81,6 +88,21 @@ disk_probe() { done } +collect_decommission_logs() { + local node="$1" since="$2" + ssh "${SSH_OPTS[@]}" "${SSH_USER}@${node}" " + SUDO=''; [ \"\$(id -u)\" -ne 0 ] && SUDO='sudo -n' + { + \${SUDO} journalctl -u ${RUSTFS_SERVICE} --since $(printf '%q' "${since}") \ + --no-pager --output=cat -n 5000 + for log_file in /var/log/rustfs/*.log; do + [ -f \"\${log_file}\" ] || continue + \${SUDO} tail -n 5000 \"\${log_file}\" + done + } 2>/dev/null || true + " +} + pool_hook() { local action="${1:-}" layout="" phase="" since="" node output0 output1 journal_since shift || true @@ -117,10 +139,8 @@ pool_hook() { diagnostic) if [ -n "${since}" ]; then journal_since="$(python3 -c 'from datetime import datetime,timezone; import sys; value=datetime.fromisoformat(sys.argv[1].replace("Z","+00:00")); assert value.tzinfo is not None; print(value.astimezone(timezone.utc).strftime("%Y-%m-%d %H:%M:%S.%f UTC"))' "${since}")" - output0="$(ssh "${SSH_OPTS[@]}" "${SSH_USER}@${NODES[0]}" \ - "journalctl -u rustfs --since $(printf '%q' "${journal_since}") --no-pager --output=cat -n 2000")" - output1="$(ssh "${SSH_OPTS[@]}" "${SSH_USER}@${NODES[1]}" \ - "journalctl -u rustfs --since $(printf '%q' "${journal_since}") --no-pager --output=cat -n 2000")" + output0="$(collect_decommission_logs "${NODES[0]}" "${journal_since}")" + output1="$(collect_decommission_logs "${NODES[1]}" "${journal_since}")" jq -n --arg phase "${phase}" --arg since "${since}" \ --arg node0 "${NODES[0]}" --arg node1 "${NODES[1]}" \ --arg output0 "${output0}" --arg output1 "${output1}" \ diff --git a/scripts/validation_registry.py b/scripts/validation_registry.py index fe5ddd81..5306fee4 100644 --- a/scripts/validation_registry.py +++ b/scripts/validation_registry.py @@ -168,7 +168,10 @@ def check_wiring(registry, root=ROOT): "workflow events or cadence differ from registry: " + item["workflow"]) for suite in registry["suites"]: source = (root / suite["workflow"]).read_text() - require("repository_dispatch" not in source and "Continue functional chain" not in source, + trigger = re.search(r"^on:\n(.*?)(?=^[^ #\n]|\Z)", source, re.M | re.S) + require(trigger is not None, "suite workflow has no event declaration") + suite_events = re.findall(r"^ (\w+):", trigger.group(1), re.M) + require("repository_dispatch" not in suite_events, "retired functional relay remains wired") require("group: " + suite["resource_lock"] in source and "cancel-in-progress: false" in source, "suite resource lock differs from registry") diff --git a/test_rustfs_performance_testing.py b/test_rustfs_performance_testing.py index 99fe6792..b1cda261 100644 --- a/test_rustfs_performance_testing.py +++ b/test_rustfs_performance_testing.py @@ -34,6 +34,7 @@ def setUp(self): [[ "$method" == put && "$objects" == 0 || "$method" != put && "$objects" == 1 ]] || return 21 printf 'diagnostic %s %s\n' "$method" "$size" >&2 if [[ "$mode" == fail || "$mode" == mixed-failure && "$method" == put ]]; then return 42; fi + if [[ "$mode" == hang ]]; then sleep 30; fi local sections=() section text case "$method" in get) sections=(GET);; @@ -112,6 +113,17 @@ def test_zero_exit_with_missing_metrics_is_not_a_result(self): self.assertEqual(len(rows), 4 if mode.startswith("missing-mixed-") else 0) self.assert_diagnostics(6) + def test_stuck_round_is_hard_timed_out_and_fails_the_run(self): + result, rows = self.run_benchmark( + "hang", WARP_METHODS="get", WARP_SIZES="1KiB", + WARP_ROUND_TIMEOUT="1", WARP_MAX_CONSECUTIVE_FAILURES="1", + ) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(rows, []) + master = (self.results / "master.log").read_text() + self.assertIn("rc=124", master) + self.assertIn("ABORT consecutive_failures=1 limit=1", master) + def test_each_required_single_method_metric_is_checked(self): for value in ("1.25 MiB/s", "1280.0", "1.2ms", "1.1ms", "1.5ms", "2.1ms"): with self.subTest(missing=value): diff --git a/test_rustfs_pool_expand.sh b/test_rustfs_pool_expand.sh index 97ec34fa..dd769411 100755 --- a/test_rustfs_pool_expand.sh +++ b/test_rustfs_pool_expand.sh @@ -295,8 +295,60 @@ test_warp_error_classification() { if report_warp_errors "${warp_log}" >/dev/null 2>&1; then fail "response-header timeouts must fail warp transport classification" fi + + printf '%s\n' 'warp: upload error: The service is unavailable. Please retry.' > "${warp_log}" + if report_warp_errors "${warp_log}" >/dev/null 2>&1; then + fail "service-unavailable upload errors must fail warp classification" + fi + + printf '%s\n' 'warp: upload error: We encountered an internal error, please try again.' > "${warp_log}" + if report_warp_errors "${warp_log}" >/dev/null 2>&1; then + fail "internal upload errors must fail warp classification" + fi } +test_stop_skips_an_uninstalled_service() ( + local fixture="${TEST_TMP}/stop-absent" bin="${TEST_TMP}/stop-absent-bin" + mkdir -p "${fixture}" "${bin}" + cat > "${bin}/systemctl" <<'EOF' +#!/usr/bin/env bash +[ "$1" = cat ] && exit 1 +exit 1 +EOF + chmod +x "${bin}/systemctl" + PATH="${bin}:${PATH}" + ssh() { + local remote="${*: -1}" + bash -c "${remote}" + } + service_action stop fixture-node +) + +test_warp_rounds_reach_threshold() ( + reset_endpoint_state + WITH_WARP=1 + DRY_RUN=0 + WARP_ROUNDS=2 + WARP_LOG_FILE="${TEST_TMP}/warp-rounds.log" + monitor_calls=0 + verify_stage_readiness() { :; } + configure_proxy_stage() { :; } + verify_pools() { :; } + confirm() { :; } + need_cmd() { :; } + warp() { :; } + abort_stale_multipart_uploads() { :; } + report_warp_errors() { :; } + monitor_storage() { + monitor_calls=$((monitor_calls + 1)) + if [ "${monitor_calls}" -eq 2 ]; then return 0; fi + return 3 + } + + step4_write_data >/dev/null 2>&1 + assert_eq 2 "${monitor_calls}" "warp rounds before threshold" +) + test_package_checksum_before_install() ( local script_name="$1" download_step="$2" install_step="$3" local fixture="${TEST_TMP}/${script_name}" expected_hash scenario status expected @@ -410,6 +462,8 @@ test_endpoint_resolution test_independent_routing_and_signing test_storageinfo_retry_is_sticky test_warp_must_reach_threshold +test_stop_skips_an_uninstalled_service +test_warp_rounds_reach_threshold test_stage_readiness_and_exact_pool_count test_stage_pool_boundaries 3 expand test_stage_pool_boundaries 4 expand diff --git a/test_rustfs_security.py b/test_rustfs_security.py index 159eb692..247795ec 100644 --- a/test_rustfs_security.py +++ b/test_rustfs_security.py @@ -269,12 +269,13 @@ def test_preview_does_not_require_runtime_dependencies(self): tools = root / "bin" tools.mkdir() # Deliberately omit curl, aws, ssh, openssl, jq, python3, and git. - for command in ("bash", "cat", "dirname", "head", "mktemp", "rm", "sed", "sort"): + for command in ("bash", "cat", "dirname", "head", "mkdir", "mktemp", "rm", "sed", "sort", "tee"): (tools / command).symlink_to(shutil.which(command)) report = root / "report.md" + log_file = root / "security.log" result = subprocess.run( [str(tools / "bash"), str(Path(__file__).with_name("rustfs-security-test.sh")), - "--dry-run", "--all-topologies", "-y"], + "--dry-run", "--all-topologies", "-y", "--log-file", str(log_file)], env={**os.environ, "PATH": str(tools), "TMPDIR": directory, "REPORT_FILE": str(report), "GITHUB_STEP_SUMMARY": str(root / "summary.md"), "RUSTFS_NODES": "fixture-node", "RUSTFS_ACCESS_KEY": "fixture-root", "RUSTFS_SECRET_KEY": "fixture-secret"}, @@ -284,6 +285,7 @@ def test_preview_does_not_require_runtime_dependencies(self): self.assertEqual(result.stdout.count("[DRY-RUN] STS-105"), 3) for topology in ("single-single", "single-multi", "multi-multi"): self.assertIn(f"## Topology: {topology}", report.read_text()) + self.assertEqual(result.stdout, log_file.read_text()) if __name__ == "__main__": diff --git a/test_rustfs_table.py b/test_rustfs_table.py index ac934512..d4e9e48c 100644 --- a/test_rustfs_table.py +++ b/test_rustfs_table.py @@ -10,6 +10,13 @@ class TableRemotePrivilegeTests(unittest.TestCase): + def test_package_download_uses_data_backed_staging_path(self): + script = (ROOT / "rustfs-table-test.sh").read_text() + self.assertIn('pkg_dir="/data/rustfs1"', script) + self.assertIn(".rustfs-table-", script) + self.assertIn("${RUN_ID}.deb", script) + self.assertIn("--retry-all-errors", script) + def test_generated_config_does_not_escape_quotes(self): script = ( f'source "{ROOT / "rustfs-table-test.sh"}"; ' diff --git a/test_rustfs_tier.sh b/test_rustfs_tier.sh index a6fddfd2..8bc65020 100755 --- a/test_rustfs_tier.sh +++ b/test_rustfs_tier.sh @@ -1085,7 +1085,8 @@ while True: esac printf "%s\n" "$3" if [ "$5" = killed-logger ]; then - kill -TERM "${LOG_CAPTURE_PID}" + trap '' PIPE + kill -KILL "${LOG_CAPTURE_PID}" sleep 0.1 fi exit "$4" @@ -1133,7 +1134,10 @@ test_log_capture_preserves_suite_and_logger_failures() { [ "${LOGGED_RC}" -ne 0 ] || fail "logger failure was not propagated" run_logged_child "${log_file}" both-failed-token 23 \ "${TEST_ROOT}/log-child-both-fail" killed-logger - assert_eq 23 "${LOGGED_RC}" "suite failure precedence over logger failure" + case "${LOGGED_RC}" in + 23|141) ;; + *) fail "combined suite/logger failures must stay nonzero: got ${LOGGED_RC}" ;; + esac } test_log_capture_does_not_wait_for_inherited_writers() { diff --git a/test_validation_heal_persistence_mrf.py b/test_validation_heal_persistence_mrf.py index f84c32d0..f57c0fbb 100644 --- a/test_validation_heal_persistence_mrf.py +++ b/test_validation_heal_persistence_mrf.py @@ -27,7 +27,8 @@ def bucket_gate_fixture(self, root): system = data / ".rustfs.sys"; system.mkdir(mode=0o700) (system / "format.json").write_bytes(b"format-control") bucket = data / "rv-heal-0123456789abcdef" - bucket.mkdir(mode=0o755) + bucket.mkdir() + bucket.chmod(0o755) obj = bucket / "objects/0000.bin"; obj.mkdir(parents=True) (obj / "xl.meta").write_bytes(b"original-fixture-metadata") (state / "owner.json").write_text(json.dumps({"runtimeProfile": "mrf-crash", "bucket": bucket.name})) diff --git a/test_validation_pool.py b/test_validation_pool.py index a7d1e7bf..d70c3cd4 100644 --- a/test_validation_pool.py +++ b/test_validation_pool.py @@ -1,12 +1,18 @@ #!/usr/bin/env python3 import unittest +from pathlib import Path from scripts.validation_pool.case_plan import POOL_CASE_IDS, pool_harness_scored_cases from scripts.validation_pool.pool_helpers import active_endpoint_indices, active_fixture_bytes class PoolContractTest(unittest.TestCase): + def test_environment_hooks_handle_missing_service_and_file_logs(self) -> None: + script = (Path(__file__).parent / "scripts/validation_pool/run_case.sh").read_text() + self.assertIn("is not installed; stop skipped", script) + self.assertIn("/var/log/rustfs/*.log", script) + def test_catalog_drives_case_selection(self) -> None: self.assertEqual(13, len(POOL_CASE_IDS)) self.assertEqual(("pool.status",), pool_harness_scored_cases(["pool.status"])) diff --git a/validation-migration-review.json b/validation-migration-review.json index e9ca52e9..823aac78 100644 --- a/validation-migration-review.json +++ b/validation-migration-review.json @@ -69,7 +69,7 @@ "scripts/validation_heal/versions.py": "fa77397a58b0db4499114c0e30a766ef67f997d1c88f22fcfe02752820b5eea0", "scripts/validation_pool/case_plan.py": "277777e040501c8a79e7104471f411b054d901afb920d18f3e222d3a0b7b03bc", "scripts/validation_pool/pool_behavior.py": "203bc3a2b7e85fb95482dd33efc937cff9f1f7e23f8ac2aa0b9ad2f40aff4632", - "scripts/validation_pool/run_case.sh": "a1de6d972964ae4232bdc1c7c4a61cb9e4a444825c7c1ced5c74cbc72f6a5ebe", + "scripts/validation_pool/run_case.sh": "597dac09fd77601701f29702504cc6b3255f50d369674ff136c7320a2ebdb1fb", "test_validation_heal.py": "6e4e25d2437a49db1af0b3020bb3a2509496b21cb3c27216a6b0322aea9bfbdc", "test_validation_heal_authority.py": "a4334246aa9572be05a3d6881779df1e2495d9706d3a9289ba9437ac866a688a", "test_validation_heal_bucket_metadata.py": "331bb5aa9bce132bd2d69b81262dae0eeffd76761139c9a382dd3884f400fb1c", @@ -102,12 +102,12 @@ "test_validation_heal_outcome_recovery.py": "46b21a03ad7fc656273d930178c1ebefb212bcc9faa125b3834c6f87a102305c", "test_validation_heal_performance.py": "92e3e09fca551c0a8a1daef4d7abdaba11e13dd4c021a33e601ad1e0bad7d924", "test_validation_heal_persistence.py": "d258b82634bfd19db7c4b1f6fb5dadcf33fafc595e3663a3a30a4720d0079f4f", - "test_validation_heal_persistence_mrf.py": "61dfa042c6b237e00b058b2660962411e201cda8908027c6900649da344b4cbe", + "test_validation_heal_persistence_mrf.py": "89b50ed67b22087fc04649239fc77b9d6bce429c80f3c3539f7723cf1bacd772", "test_validation_heal_race.py": "1f2097aff5ad08c4d2834bfd38e1eb1d1b373ecde8f7e6a3455b713114a07d3d", "test_validation_heal_race_native.py": "850c9683ac918834d22b002ec2f5a8c0d7fd2b74266461e4912776dfceb0d93d", "test_validation_heal_schema.py": "5df440df29e7085d48590665129f87b0791f9fa8c7f64c8c0432000bc2002606", "test_validation_heal_unformatted.py": "58bc43f3f0905c8c4b7d0f8e656d25f0a17e197b5a2dfb66a88b6af9747883a7", - "test_validation_pool.py": "300adcef396533b8cbb54864ccd043b9f1b897558042039ad9b9aec60ed2aade" + "test_validation_pool.py": "56b93f66befb32d0fd5a3957d752e10ae774b962caaee93b116ef0fd637c50ec" }, "cases": [ { From 746b8abd80b0bffc4adfcedbf0da5929cce806e9 Mon Sep 17 00:00:00 2001 From: cxymds Date: Wed, 4 Nov 2026 16:33:06 +0800 Subject: [PATCH 2/3] fix(test): bound tier probe process groups --- rustfs-tier-test.sh | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/rustfs-tier-test.sh b/rustfs-tier-test.sh index 7e220fde..206a7124 100755 --- a/rustfs-tier-test.sh +++ b/rustfs-tier-test.sh @@ -540,7 +540,22 @@ run_bounded() { run_hard_bounded() { local seconds="$1" shift - timeout --signal=KILL "${seconds}" "$@" + python3 - "$seconds" "$@" <<'PY' +import os +import signal +import subprocess +import sys + +timeout = float(sys.argv[1]) +process = subprocess.Popen(sys.argv[2:], start_new_session=True) +try: + status = process.wait(timeout=timeout) +except subprocess.TimeoutExpired: + os.killpg(process.pid, signal.SIGKILL) + process.wait() + raise SystemExit(137) +raise SystemExit(status if status >= 0 else 128 - status) +PY } bounded_safe_excerpt() { From dd7b55e97540ce58d97b2ea8c701accbeba5f4dc Mon Sep 17 00:00:00 2001 From: cxymds Date: Wed, 4 Nov 2026 16:43:14 +0800 Subject: [PATCH 3/3] fix(test): address VM rerun review findings --- rustfs-security-test.sh | 28 ++++++++++++++++++++++------ rustfs-table-test.sh | 2 +- rustfs_pool_expand.sh | 5 ++--- test_rustfs_pool_expand.sh | 18 ++++++++++++++++++ test_rustfs_security.py | 19 +++++++++++++++++++ test_rustfs_table.py | 3 +-- 6 files changed, 63 insertions(+), 12 deletions(-) diff --git a/rustfs-security-test.sh b/rustfs-security-test.sh index d6ba60d8..84beecb9 100755 --- a/rustfs-security-test.sh +++ b/rustfs-security-test.sh @@ -1109,11 +1109,6 @@ run_security_cases() { main() { parse_args "$@" - if [ -n "${LOG_FILE}" ]; then - command -v tee >/dev/null 2>&1 || die "missing command: tee (required by --log-file)" - mkdir -p "$(dirname "${LOG_FILE}")" - exec > >(tee -a "${LOG_FILE}") 2>&1 - fi need_cmd ssh "openssh client" need_cmd curl "curl" need_cmd openssl "openssl" @@ -1177,6 +1172,27 @@ main() { echo "security suite: ${PASS} passed, ${FAIL} failed, ${SKIP} skipped" } -if [ "${RUSTFS_SECURITY_SCRIPT_SOURCE_ONLY:-0}" != 1 ]; then +run_main() { + local log_file="" index + local -a args=("$@") + for ((index=0; index<${#args[@]}; index++)); do + if [ "${args[index]}" = "--log-file" ]; then + log_file="${args[index+1]:-}" + fi + done + if [ -n "${log_file}" ]; then + command -v tee >/dev/null 2>&1 || die "missing command: tee (required by --log-file)" + mkdir -p "$(dirname "${log_file}")" + set +e + main "$@" 2>&1 | tee -a "${log_file}" + local suite_rc=${PIPESTATUS[0]} tee_rc=${PIPESTATUS[1]} + set -e + [ "${suite_rc}" -eq 0 ] || exit "${suite_rc}" + exit "${tee_rc}" + fi main "$@" +} + +if [ "${RUSTFS_SECURITY_SCRIPT_SOURCE_ONLY:-0}" != 1 ]; then + run_main "$@" fi diff --git a/rustfs-table-test.sh b/rustfs-table-test.sh index 0bf29f31..f1dcf963 100755 --- a/rustfs-table-test.sh +++ b/rustfs-table-test.sh @@ -170,7 +170,7 @@ setup_environment() { remote_root "${v}" <= t)}' && break if [ -n "${warp_pid}" ] && ! kill -0 "${warp_pid}" 2>/dev/null; then diff --git a/test_rustfs_pool_expand.sh b/test_rustfs_pool_expand.sh index dd769411..140432cc 100755 --- a/test_rustfs_pool_expand.sh +++ b/test_rustfs_pool_expand.sh @@ -176,6 +176,19 @@ test_warp_must_reach_threshold() { fi } +test_monitor_propagates_storage_sample_failure() { + local status + if ( + storage_usage_percent() { return 1; } + monitor_storage 50 999999 + ) >/dev/null 2>&1; then + fail "storage sample failure must fail the monitor" + else + status=$? + fi + assert_eq 1 "${status}" "storage sample failure status" +} + test_stage_readiness_and_exact_pool_count() { reset_endpoint_state ADMIN_ENDPOINT="http://admin.test:9000" @@ -276,6 +289,10 @@ test_warp_error_classification() { report_warp_errors "${warp_log}" >/dev/null \ || fail "application-level SlowDown must not be classified as a proxy failure" + printf '%s\n' 'warp: upload error: SlowDown: foreground write concurrency limit reached' > "${warp_log}" + report_warp_errors "${warp_log}" >/dev/null \ + || fail "warp-prefixed SlowDown must not be classified as a transport failure" + printf '%s\n' '502 Bad Gateway' > "${warp_log}" if report_warp_errors "${warp_log}" >/dev/null 2>&1; then fail "Nginx 502 must fail warp transport classification" @@ -462,6 +479,7 @@ test_endpoint_resolution test_independent_routing_and_signing test_storageinfo_retry_is_sticky test_warp_must_reach_threshold +test_monitor_propagates_storage_sample_failure test_stop_skips_an_uninstalled_service test_warp_rounds_reach_threshold test_stage_readiness_and_exact_pool_count diff --git a/test_rustfs_security.py b/test_rustfs_security.py index 247795ec..2cb6500f 100644 --- a/test_rustfs_security.py +++ b/test_rustfs_security.py @@ -287,6 +287,25 @@ def test_preview_does_not_require_runtime_dependencies(self): self.assertIn(f"## Topology: {topology}", report.read_text()) self.assertEqual(result.stdout, log_file.read_text()) + def test_log_capture_failure_is_propagated(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + tools = root / "bin" + tools.mkdir() + for command in ("bash", "cat", "dirname", "head", "mkdir", "mktemp", "rm", "sed", "sort"): + (tools / command).symlink_to(shutil.which(command)) + (tools / "tee").write_text("#!/bin/sh\ncat >/dev/null\nexit 1\n") + (tools / "tee").chmod(0o755) + result = subprocess.run( + [str(tools / "bash"), str(Path(__file__).with_name("rustfs-security-test.sh")), + "--dry-run", "--all-topologies", "-y", "--log-file", str(root / "security.log")], + env={**os.environ, "PATH": str(tools), "TMPDIR": directory, "REPORT_FILE": str(root / "report.md"), + "GITHUB_STEP_SUMMARY": str(root / "summary.md"), "RUSTFS_NODES": "fixture-node", + "RUSTFS_ACCESS_KEY": "fixture-root", "RUSTFS_SECRET_KEY": "fixture-secret"}, + capture_output=True, text=True, timeout=10, + ) + self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr) + if __name__ == "__main__": unittest.main() diff --git a/test_rustfs_table.py b/test_rustfs_table.py index d4e9e48c..7509e691 100644 --- a/test_rustfs_table.py +++ b/test_rustfs_table.py @@ -13,8 +13,7 @@ class TableRemotePrivilegeTests(unittest.TestCase): def test_package_download_uses_data_backed_staging_path(self): script = (ROOT / "rustfs-table-test.sh").read_text() self.assertIn('pkg_dir="/data/rustfs1"', script) - self.assertIn(".rustfs-table-", script) - self.assertIn("${RUN_ID}.deb", script) + self.assertIn('pkg_tmp="\\${pkg_dir}/.rustfs-table-${RUN_ID}.deb"', script) self.assertIn("--retry-all-errors", script) def test_generated_config_does_not_escape_quotes(self):