fix(performance): 验证每轮清理并区分 HTTP 与逐对象删除错误

.github/workflows/environment-compatibility-tests.yml

@@ -8,6 +8,10 @@ on:
       - rustfs-security-test.sh
       - rustfs-upgrade-test.sh
       - rustfs_performance_test.sh
+      - rustfs_performance_testing.sh
+      - scripts/performance_s3.py
+      - scripts/performance_delete_probe.py
+      - test_performance_s3.py
       - scripts/check_execution_environment.py
       - scripts/check_shell_syntax.py
       - scripts/prepare_table_smoke.py
@@ -27,6 +31,10 @@ on:
       - rustfs-security-test.sh
       - rustfs-upgrade-test.sh
       - rustfs_performance_test.sh
+      - rustfs_performance_testing.sh
+      - scripts/performance_s3.py
+      - scripts/performance_delete_probe.py
+      - test_performance_s3.py
       - scripts/check_execution_environment.py
       - scripts/check_shell_syntax.py
       - scripts/prepare_table_smoke.py
@@ -55,13 +63,16 @@ jobs:
           python3 -B scripts/check_shell_syntax.py --report "${RUNNER_TEMP}/shell-syntax.json"
           test -x rustfs-security-test.sh
           test -x rustfs-upgrade-test.sh
+      - name: Install S3 test client
+        run: python3 -m pip install 'boto3>=1.35,<2'
       - name: Test portable execution helpers
         run: >-
           python3 -B -m unittest -v
           test_check_execution_environment.py
           test_check_shell_syntax.py
           test_rustfs_table.py
           test_rustfs_performance_testing.py
+          test_performance_s3.py
           test_rustfs_replication.py
       - name: Test missing HOME
         run: env -u HOME bash test_rustfs_pool_expand.sh

.github/workflows/performance-script-tests.yml

@@ -4,12 +4,20 @@ on:
   pull_request:
     paths:
       - rustfs_performance_testing.sh
+      - rustfs_performance_test.sh
+      - scripts/performance_s3.py
+      - scripts/performance_delete_probe.py
+      - test_performance_s3.py
       - test_rustfs_performance_testing.py
       - .github/workflows/performance-script-tests.yml
   push:
     branches: [main]
     paths:
       - rustfs_performance_testing.sh
+      - rustfs_performance_test.sh
+      - scripts/performance_s3.py
+      - scripts/performance_delete_probe.py
+      - test_performance_s3.py
       - test_rustfs_performance_testing.py
       - .github/workflows/performance-script-tests.yml
 
@@ -25,6 +33,10 @@ jobs:
         with:
           persist-credentials: false
       - name: Validate shell syntax
-        run: bash -n rustfs_performance_testing.sh
+        run: |
+          bash -n rustfs_performance_testing.sh
+          bash -n rustfs_performance_test.sh
+      - name: Install S3 test client
+        run: python3 -m pip install 'boto3>=1.35,<2'
       - name: Test benchmark result contracts without a server
-        run: python3 test_rustfs_performance_testing.py
+        run: python3 -B -m unittest -v test_rustfs_performance_testing.py test_performance_s3.py

.github/workflows/rustfs-performance-test.yml

@@ -155,6 +155,8 @@ jobs:
           uname -a
           jq --version
           warp --version || true
+          python3 -c 'import boto3; from botocore.config import Config'
+          command -v timeout
           df -h /data | tail -1
 
       - name: Admit clean shared environment
@@ -330,6 +332,11 @@ jobs:
             ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/get_*.txt
             ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/put_*.txt
             ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/mixed_*.txt
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/*.benchdata*
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/*.prepare.log
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/*.cleanup.log
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/*.s3.json
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/*.s3-events.jsonl
           if-no-files-found: error
 
       - name: Reset test environment (after)

README.md

@@ -26,6 +26,66 @@ port that does not conflict with the local package server or another service.
 All top-level suite entrypoints are directly executable; `bash <script>` also
 works when a local checkout or transport drops executable modes.
 
+## Performance benchmark lifecycle
+
+`rustfs_performance_testing.sh` needs Warp, GNU `timeout`, Python 3 and boto3
+(`python3 -m pip install 'boto3>=1.35,<2'`). It creates a different bucket for
+every selected round; `WARP_BUCKET` is now a name prefix. Warp runs with
+`--noclear`, and the harness owns preparation and cleanup. Credentials are
+passed through environment variables instead of Warp command-line arguments.
+
+`WARP_ROUND_TIMEOUT` (default 900 seconds) bounds Warp's fixture preparation
+and measured workload. `WARP_CLEANUP_TIMEOUT` (default 180 seconds) reserves a
+separate cleanup budget; `WARP_CLEANUP_ATTEMPTS` (default 3) bounds retries of
+retryable delete failures. The deployment wrapper exposes these as
+`RUSTFS_WARP_CLEANUP_TIMEOUT` and `RUSTFS_WARP_CLEANUP_ATTEMPTS`. Ordinary
+Warp failures and timeouts still run cleanup. An interrupted shell makes a
+best-effort cleanup attempt on exit; SIGKILL cannot run that handler. Keep
+the receipt if recovery is interrupted or incomplete.
+
+Each round retains its original `.txt`, native `.benchdata*`, `.prepare.log`,
+`.cleanup.log`, `.s3.json` receipt and `.s3-events.jsonl`. The events record
+HTTP status, request id, SDK retry count, RustFS error cause when supplied,
+and per-key delete errors. HTTP 200 with `DeleteResult/Errors` is a failure
+even though the HTTP request succeeded. `cleanup_http_503_attempts` counts
+HTTP 503 responses; `cleanup_error_entries` counts affected key failure
+entries across attempts, including keys affected by a failed request. These
+are different quantities. The event's `error_origin` distinguishes server
+per-key responses from request/protocol failures. The helper disables SDK
+retries; Warp's native measured-operation results do not prove that its SDK
+never retried an HTTP attempt.
+
+The round stays FAIL if cleanup retries encountered any error, even when
+cleanup ultimately succeeds with zero residue. Incomplete cleanup stops the
+matrix and marks later rounds NOT_RUN. `summary.tsv` contains only certified
+rounds, while `rounds.tsv` and `summary.md` include every selected round and
+its cleanup verdict. Delete error counters are saved after every attempt,
+before backoff or the next request. The sticky `cleanup_had_errors` field
+also retains deadline, malformed-response and interrupted-cleanup failures;
+successful recovery cannot clear it or certify the original round. Cleanup
+receipts carry `cleanup_in_progress` so recovery detects a process killed
+before its final checkpoint. Older receipts with a historical error code
+remain failed even if their final cleanup status is PASS.
+A new execution refuses to overwrite an existing
+`master.log`. `--parse-only` writes separate `.reparsed.*` reports; old outputs
+without a lifecycle marker remain readable, with unknown cleanup status.
+
+```bash
+python3 -B -m unittest -v test_rustfs_performance_testing.py test_performance_s3.py
+```
+
+These tests validate harness behavior, including real boto3 parsing against
+a local HTTP stub. They do not claim live RustFS coverage.
+
+For a separate bulk-delete investigation, `scripts/performance_delete_probe.py`
+compares batches of 1, 32, 128, 500 and 1000 objects at idle and under 32 GET
+workers. Supply each endpoint with `--endpoint http://node:9000` and a fresh
+`--result-dir`. It uses the same environment credentials as Warp and records
+each one-shot delete verdict before any recovery retry, UTC timestamps,
+per-key failures, request ids, GET results and residue before/after cleanup.
+Cleanup runs in `finally`; an incomplete cleanup aborts the remaining cases.
+The caller must hold the environment's test lock throughout this probe.
+
 ## Release validation migration
 
 `validation-cases.json` snapshots all 158 canonical cases from

rustfs_performance_test.sh

@@ -96,6 +96,8 @@ BENCH_SCRIPT="${RUSTFS_BENCH_SCRIPT:-${_BENCH_RESOLVED}}"
 RESULT_DIR="${RUSTFS_RESULT_DIR:-$(pwd)/warp-bench-results-$(date +%Y%m%d-%H%M%S)}"
 WARP_HOST="${RUSTFS_WARP_HOST:-rustfs-node1:9000,rustfs-node2:9000,rustfs-node3:9000,rustfs-node4:9000}"
 WARP_BUCKET="${RUSTFS_WARP_BUCKET:-warp-benchmark-bucket}"
+WARP_CLEANUP_TIMEOUT="${RUSTFS_WARP_CLEANUP_TIMEOUT:-180}"
+WARP_CLEANUP_ATTEMPTS="${RUSTFS_WARP_CLEANUP_ATTEMPTS:-3}"
 WARP_CONCURRENCY="${RUSTFS_WARP_CONCURRENCY:-64}"
 WARP_DURATION="${RUSTFS_WARP_DURATION:-5m}"
 WARP_GET_OBJECTS="${RUSTFS_WARP_GET_OBJECTS:-2500}"
@@ -265,6 +267,15 @@ preflight() {
   need_cmd ssh "openssh client"
   need_cmd curl "http client"
   need_cmd warp "warp benchmark tool (for step 5)"
+  local step
+  for step in "${SELECTED_STEPS[@]}"; do
+    if [[ "${step}" == 5 && "${DRY_RUN}" -eq 0 ]]; then
+      need_cmd python3 "verified benchmark cleanup"
+      need_cmd timeout "bounded benchmark cleanup"
+      python3 -c 'import boto3; from botocore.config import Config' \
+        || die "Python 3 with boto3 is required for verified benchmark cleanup"
+    fi
+  done
   if [ ! -x "${BENCH_SCRIPT}" ]; then
     die "benchmark script not found or not executable: ${BENCH_SCRIPT}"
   fi
@@ -480,6 +491,8 @@ step5_benchmark() {
   WARP_SLEEP_BETWEEN_ROUNDS="${WARP_SLEEP}" \
   WARP_ROUND_TIMEOUT="${WARP_ROUND_TIMEOUT}" \
   WARP_MAX_CONSECUTIVE_FAILURES="${WARP_MAX_CONSECUTIVE_FAILURES}" \
+  WARP_CLEANUP_TIMEOUT="${WARP_CLEANUP_TIMEOUT}" \
+  WARP_CLEANUP_ATTEMPTS="${WARP_CLEANUP_ATTEMPTS}" \
   WARP_METHODS="${WARP_METHODS}" \
   WARP_SIZES="${WARP_SIZES}" \
   WARP_RESULT_DIR="${RESULT_DIR}" \