修复全量复测中的套件准备、活动夹具与容量保护

.github/workflows/environment-compatibility-tests.yml

@@ -9,6 +9,10 @@ on:
       - rustfs-upgrade-test.sh
       - rustfs_performance_test.sh
       - scripts/check_execution_environment.py
+      - scripts/check_shell_syntax.py
+      - scripts/prepare_table_smoke.py
+      - test_check_shell_syntax.py
+      - .github/workflows/rustfs-table-test.yml
       - scripts/lib/**
       - test_check_execution_environment.py
       - test_rustfs_table.py
@@ -24,6 +28,10 @@ on:
       - rustfs-upgrade-test.sh
       - rustfs_performance_test.sh
       - scripts/check_execution_environment.py
+      - scripts/check_shell_syntax.py
+      - scripts/prepare_table_smoke.py
+      - test_check_shell_syntax.py
+      - .github/workflows/rustfs-table-test.yml
       - scripts/lib/**
       - test_check_execution_environment.py
       - test_rustfs_table.py
@@ -44,21 +52,14 @@ jobs:
           persist-credentials: false
       - name: Validate shell syntax
         run: |
-          for script in \
-            rustfs-table-test.sh \
-            rustfs-replication-test.sh \
-            rustfs-security-test.sh \
-            rustfs-upgrade-test.sh \
-            rustfs_performance_test.sh
-          do
-            bash -n "${script}"
-          done
+          python3 -B scripts/check_shell_syntax.py --report "${RUNNER_TEMP}/shell-syntax.json"
           test -x rustfs-security-test.sh
           test -x rustfs-upgrade-test.sh
       - name: Test portable execution helpers
         run: >-
           python3 -B -m unittest -v
           test_check_execution_environment.py
+          test_check_shell_syntax.py
           test_rustfs_table.py
           test_rustfs_performance_testing.py
           test_rustfs_replication.py

.github/workflows/package-install-tests.yml

@@ -7,6 +7,8 @@ on:
       - rustfs_pool_expand.sh
       - rustfs_performance_test.sh
       - test_rustfs_pool_expand.sh
+      - scripts/storage_capacity.py
+      - test_storage_capacity.py
       - .github/workflows/package-install-tests.yml
   push:
     branches: [main]
@@ -15,6 +17,8 @@ on:
       - rustfs_pool_expand.sh
       - rustfs_performance_test.sh
       - test_rustfs_pool_expand.sh
+      - scripts/storage_capacity.py
+      - test_storage_capacity.py
       - .github/workflows/package-install-tests.yml
 
 permissions:
@@ -35,3 +39,5 @@ jobs:
           done
       - name: Test package installation without a server
         run: bash test_rustfs_pool_expand.sh
+      - name: Test capacity stop decisions
+        run: python3 -B -m unittest -v test_storage_capacity.py

.github/workflows/rustfs-performance-test.yml

@@ -322,6 +322,11 @@ jobs:
             ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/master.log
             ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/summary.md
             ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/summary.tsv
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/rounds.tsv
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/master.reparsed.log
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/summary.reparsed.md
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/summary.reparsed.tsv
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/rounds.reparsed.tsv
             ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/get_*.txt
             ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/put_*.txt
             ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/mixed_*.txt

.github/workflows/rustfs-table-test.yml

@@ -22,6 +22,10 @@ on:
         required: false
         type: boolean
         default: false
+      source_ref:
+        description: 'RustFS source ref matching the standalone candidate (defaults to release tag or main)'
+        required: false
+        type: string
 
 concurrency:
   group: rustfs-shared-functional-tests-v2
@@ -80,6 +84,26 @@ jobs:
           aws --version
           python3 --version
 
+      - name: Checkout matching Table smoke sources
+        if: ${{ !inputs.skip_smoke }}
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          repository: rustfs/rustfs
+          ref: ${{ steps.chain.outputs.source_sha || inputs.source_ref || inputs.rustfs_version || 'main' }}
+          path: .rustfs-table-source
+          sparse-checkout: scripts/table-catalog
+          persist-credentials: false
+
+      - name: Prepare complete Table smoke module tree
+        if: ${{ !inputs.skip_smoke }}
+        env:
+          EXPECTED_SOURCE_SHA: ${{ steps.chain.outputs.source_sha }}
+        run: >-
+          python3 scripts/prepare_table_smoke.py prepare
+          --source-root "${GITHUB_WORKSPACE}/.rustfs-table-source"
+          --destination "${FUNCTIONAL_ARTIFACTS_DIR}/table-catalog"
+          --expected-sha "${EXPECTED_SOURCE_SHA}"
+
       - name: Admit clean shared environment
         id: environment
         timeout-minutes: 5
@@ -123,7 +147,7 @@ jobs:
           chmod +x rustfs-table-test.sh
           PACKAGE_URL='${{ steps.chain_package.outputs.package_url || inputs.package_url }}'
           RUSTFS_VERSION='${{ inputs.rustfs_version }}'
-          SMOKE_PATH="${GITHUB_WORKSPACE:-$PWD}/scripts/table-catalog/pyiceberg_smoke.py"
+          SMOKE_PATH="${FUNCTIONAL_ARTIFACTS_DIR}/table-catalog/pyiceberg_smoke.py"
           ARGS=(-y --smoke-script "${SMOKE_PATH}")
           if [ '${{ inputs.skip_smoke }}' = 'true' ]; then
             ARGS+=(--skip-smoke)
@@ -284,6 +308,7 @@ jobs:
             ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md
             ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/cases.md
             ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/table-catalog/source-manifest.json
           if-no-files-found: error
           retention-days: 30
 

.github/workflows/validation-catalog-tests.yml

@@ -14,6 +14,7 @@ on:
       - scripts/validation_ilm/**
       - test_validation_pool.py
       - scripts/validation_pool/**
+      - scripts/storage_capacity.py
       - rustfs_pool_expand.sh
       - test_validation_replication.py
       - test_rustfs_replication.py
@@ -88,6 +89,7 @@ on:
       - scripts/validation_ilm/**
       - test_validation_pool.py
       - scripts/validation_pool/**
+      - scripts/storage_capacity.py
       - rustfs_pool_expand.sh
       - test_validation_replication.py
       - test_rustfs_replication.py
@@ -164,9 +166,9 @@ jobs:
       - run: python3 -B scripts/validation_migration_review.py --require-complete
       - run: python3 -B -m unittest -v test_validation_migration.py
       - run: python3 -B -m unittest -v test_validation_ilm.py
+      - run: python3 -m pip install -r scripts/validation_ilm/requirements.txt
       - run: python3 -B -m unittest -v test_validation_pool.py
       - run: python3 -B -m unittest -v test_validation_replication.py test_rustfs_replication.py
-      - run: python3 -m pip install -r scripts/validation_ilm/requirements.txt
       - name: Install network namespace test dependency
         run: |
           if ! command -v nft >/dev/null; then

docs/suite-environment-retest.zh-CN.md

@@ -0,0 +1,45 @@
+# 测试套件与环境问题修复后的复测
+
+本次改动对应 [backlog #3173](https://github.com/rustfs/backlog/issues/3173) 的测试侧归因，基于已合并的 auto-testing #144。它不代表修复了服务端 503、命名空间锁超时或性能异常。
+
+| 原失败 | 修复行为 | 复测验收 |
+|---|---|---|
+| TBL-101 导入失败 | 从候选对应的 RustFS checkout 准备整个 `scripts/table-catalog`，记录源码 SHA 与文件哈希；先检查本地模块，再创建 venv | 端到端实际读写成功；缺模块必须报告工具准备错误 |
+| M-1x2EC1 启动非参与节点 | 显式传递参与节点数量，只对这些节点提交非阻塞启动并等待 active | 完整保留各拓扑 / EC / 故障探针，不能用未执行代替通过 |
+| Rebalance 活动判据超时 | 单独增大 Rebalance 活动夹具并先校验容量，保留 `Started` 与非零进度断言；提前完成明确报告活动窗口不足 | stop/resume、live I/O、operation guards 必须在活动窗口执行；完成态仍要求非零迁移进度及数据校验 |
+| 扩池负载写满单盘 | 2 秒容量采样，单盘利用率与剩余字节保护；预留在途对象和观测到的增长量，缩短停止等待 | 达到目标才通过；容量保护提前停止是环境容量失败，仍执行 MPU 清理 |
+| 性能轮次含服务错误却计入摘要 | 错误日志使该轮失败，原始指标保留作诊断；逐轮记录 Report 是否出现、退出码、错误行数 | `rounds.tsv` 和 Markdown 包含全部选定轮次；超时与未执行都不算通过 |
+| AppleDouble 输入 / 语法报告缺口 | 排除 `._*`，对每个真实脚本独立执行 `bash -n`，保存逐文件结果 | 任一真实脚本失败，整体返回非零；空选择也不通过 |
+
+## Table 源码准备
+
+```bash
+python3 scripts/prepare_table_smoke.py prepare \
+  --source-root /path/to/rustfs-checkout \
+  --destination /path/to/new-run/table-catalog \
+  --expected-sha FULL_CANDIDATE_SOURCE_SHA
+```
+
+`--destination` 必须是新目录，避免混入旧版本模块。链式 workflow 使用 manifest 的 source SHA；独立 workflow 可通过 `source_ref` 指定与产物匹配的源码。未指定时使用 release tag 或当次 checkout 的 main，并在 `source-manifest.json` 保留实际 SHA。
+
+Table workflow 将 `table-catalog/source-manifest.json` 与报告、用例表和日志一起上传，保留实际 smoke 源码 SHA 与文件哈希，供 runner 临时目录清理后追溯。
+
+## 容量配置
+
+Rebalance 源用例明确要求活动阶段已有非零迁移进度。上一轮每对象 32 MiB 的任务在首次周期保存进度前完成，因此不能通过。专用活动夹具默认改为每对象 128 MiB（默认 128 对象，其中 3 个边界对象更小），通过 `RUSTFS_ACTIVE_REBALANCE_FIXTURE_BYTES` / `--active-rebalance-fixture-bytes` 配置，并记录在 run manifest。写入前按 EC 占用上界与每盘至少 1 GiB 的剩余余量检查容量。其他 Decommission 活动夹具保留原大小；更快的机器仍可能需要增加夹具，窗口不足时用例失败而不是放松断言。
+
+容量采样与 Rebalance / Decommission 的 `--poll-interval` 分开。默认每 2 秒采样，一次直接 API 请求上限 3 秒，活动写入期间不做长重试或代理探测。代理仍在写入前后探测，错误继续保留。
+
+预留量为每盘最少剩余字节（默认 512 MiB）加 `对象大小 × 并发数`，再加观测到的集群增长速率乘以采样、请求与停止窗口。集群增长作为单盘增长的保守上界，不假设数据均匀。容量库存为空、无效或发生变化会停止继续写入。
+
+可配置 `RUSTFS_CAPACITY_POLL_INTERVAL`、`RUSTFS_CAPACITY_SAMPLE_TIMEOUT`、`RUSTFS_CAPACITY_MIN_FREE_BYTES`、`RUSTFS_CAPACITY_MAX_DISK_PERCENT`、`RUSTFS_WARP_STOP_GRACE`、`RUSTFS_WARP_OBJ_SIZE`。小容量环境应先按磁盘余量降低并发或目标利用率。旁路 `.capacity.jsonl` 保留每次判断的输入、余量与停止原因；这是一项负载保护，无法保证在不可预测的突发写入下绝不耗尽容量。
+
+MPU 列表错误会保留 HTTP status、`x-rustfs-error-cause`、request ID 与 retry-after（服务端返回时）；不采集完整 AWS debug 输出或响应 Cookie。
+
+## 结果边界
+
+性能轮次总超时仍覆盖准备、负载和收尾。`timeout_before_report` / `timeout_after_report` 仅说明是否出现 Warp Report，不能推断 Warp 内部卡在具体阶段。不要把扩池发生的磁盘满用来解释此前的性能超时。
+
+`--parse-only` 输出独立的 `rounds.reparsed.tsv`、`summary.reparsed.tsv`、`summary.reparsed.md` 与 `master.reparsed.log`，保留原执行台账、摘要及日志。重解析无法恢复耗时和原始运行配置；应以原 `rounds.tsv` 中的 `NOT_RUN / aborted` 及耗时为执行证据。性能 workflow 同时上传原执行台账与重解析产物。步骤 6 缺少原摘要时展示重解析摘要，重解析失败仍返回非零。
+
+复测需保留产品源码 SHA、CI 产物身份、二进制 SHA-256、测试提交 SHA、实际配置和完整逐条结果。VM 写操作及最终恢复使用同一个共享锁；恢复后核对四节点、24 个数据挂载与对象校验。

rustfs-fault-tolerance-test.sh

@@ -551,8 +551,20 @@ seed_baseline() { # seed_baseline <endpoint>
 }
 
 # cluster lifecycle ----------------------------------------------------------
-install_and_start() { # install_and_start <volumes> [extra env line]
-  local volumes="$1" extra_env="${2:-}" node
+install_and_start() { # install_and_start <volumes> [extra env line] [participant count]
+  local volumes="$1" extra_env="${2:-}" count="${3:-}" node
+  local participants=()
+  if [ -z "${count}" ] && [ "${volumes}" = "${VOLUMES_SNMD}" ]; then
+    participants=("${SNMD_NODE}")
+  else
+    if [ -z "${count}" ]; then
+      count=${#NODES[@]}
+      [ "${volumes}" != "${VOLUMES_3X4}" ] || count=3
+    fi
+    [[ "${count}" =~ ^[1-9][0-9]*$ ]] && [ "${count}" -le "${#NODES[@]}" ] \
+      || die "invalid topology participant count: ${count}"
+    participants=("${NODES[@]:0:count}")
+  fi
   [ -n "${PACKAGE_URL}" ] || die "--package-url is required for test runs"
   log "installing package on ${NODES[*]}"
   remote_all "set -e; sudo -n systemctl stop ${RUSTFS_SERVICE} 2>/dev/null || true; sudo -n dpkg --configure -a 2>/dev/null || true; curl -fsSL --retry 3 -o /tmp/rustfs-ft.deb '${PACKAGE_URL}' && sudo -n env DEBIAN_FRONTEND=noninteractive dpkg -i --force-confdef --force-confnew /tmp/rustfs-ft.deb && sudo -n systemctl daemon-reload"
@@ -573,27 +585,8 @@ install_and_start() { # install_and_start <volumes> [extra env line]
     ssh "${SSH_OPTS[@]}" "${SSH_USER}@${node}" "sudo -n tee ${RUSTFS_CONFIG_FILE} >/dev/null" < "${TMP}/envfile"
     run_remote "${node}" 'sudo -n mkdir -p /data/rustfs{1..4}/mnmd && for i in 1 2 3 4; do sudo -n chown rustfs:rustfs "/data/rustfs${i}" "/data/rustfs${i}/mnmd"; done 2>/dev/null || true'
   done
-  if [ "${volumes}" = "${VOLUMES_SNMD}" ]; then
-    log "single-node topology: starting ${RUSTFS_SERVICE} on ${SNMD_NODE} only"
-    if run_remote "${SNMD_NODE}" "sudo -n systemctl start ${RUSTFS_SERVICE}"; then
-      :
-    else
-      dump_service_diagnostics "${SNMD_NODE}"
-      die "start failed on ${SNMD_NODE}"
-    fi
-  else
-    log "starting ${RUSTFS_SERVICE} on ${NODES[*]}"
-    local start_pids=() start_i=0 start_pid start_fail=0
-    for node in "${NODES[@]}"; do
-      run_remote "${node}" "sudo -n systemctl start ${RUSTFS_SERVICE}" &
-      start_pids[${start_i}]=$!; start_i=$((start_i+1))
-    done
-    for start_pid in "${start_pids[@]}"; do wait "${start_pid}" || start_fail=1; done
-    if [ "${start_fail}" -ne 0 ]; then
-      dump_service_diagnostics "${NODES[@]}"
-      die "cluster start failed on one or more nodes"
-    fi
-  fi
+  log "topology participants: ${participants[*]}"
+  start_nodes "${participants[@]}"
   if [ "${DRY_RUN}" -eq 1 ]; then
     log "DRY-RUN: skipping cluster readiness polling"
     return 0
@@ -953,7 +946,7 @@ matrix_run_case() { # <topo-key> <nodes> <drives-per> <parity>
   done
   mc+=("${tag}-rec/read" "${tag}-rec/write")
   register_expected_cases "${mc[@]}"
-  install_and_start "$(matrix_volumes_expr "${nodes}" "${per}")" "RUSTFS_STORAGE_CLASS_STANDARD=\"EC:${parity}\""
+  install_and_start "$(matrix_volumes_expr "${nodes}" "${per}")" "RUSTFS_STORAGE_CLASS_STANDARD=\"EC:${parity}\"" "${nodes}"
   seed_baseline "${EP1}"
 
   # full capacity

rustfs-table-test.sh

@@ -246,11 +246,8 @@ cleanup_all() {
 # ==================== TBL-101: PyIceberg end-to-end (product smoke) ====================
 test_tbl101() {
   if [ "${SKIP_SMOKE}" -eq 1 ]; then echo "smoke skipped by flag"; return 2; fi
-  if [ "${SKIP_SMOKE}" -eq 1 ]; then echo "tbl101: smoke skipped by flag"; return 2; fi
-  if [ -z "${SMOKE_SCRIPT}" ] || [ ! -f "${SMOKE_SCRIPT}" ]; then
-    echo "tbl101: smoke script not found: '${SMOKE_SCRIPT:-<empty>}' (cwd=$(pwd), SKIP_SMOKE=${SKIP_SMOKE})"
-    return 2
-  fi
+  python3 "${SCRIPT_DIR}/scripts/prepare_table_smoke.py" check --smoke-script "${SMOKE_SCRIPT}" \
+    || return 1
   echo "tbl101: smoke script ok, creating venv"
   local venv="/tmp/tbl-venv-${RUN_ID}"
   if ! python3 -m venv "${venv}" >/tmp/tbl-venv-err.log 2>&1; then

rustfs_performance_test.sh

@@ -488,6 +488,7 @@ step5_benchmark() {
 }
 
 step6_analyze() {
+  local summary="${RESULT_DIR}/summary.md"
   log "step 6: analyze results from ${RESULT_DIR}"
   if [ "${DRY_RUN}" -eq 1 ]; then
     log "DRY-RUN: bash ${BENCH_SCRIPT} --parse-only ${RESULT_DIR}"
@@ -496,14 +497,17 @@ step6_analyze() {
   if [ ! -d "${RESULT_DIR}" ]; then
     die "result directory not found: ${RESULT_DIR}"
   fi
-  if [ -f "${RESULT_DIR}/summary.md" ]; then
-    log "summary already generated: ${RESULT_DIR}/summary.md"
+  if [ -f "${summary}" ]; then
+    log "summary already generated: ${summary}"
   else
     log "generating summary with --parse-only"
+    WARP_METHODS="${WARP_METHODS}" \
+    WARP_SIZES="${WARP_SIZES}" \
     bash "${BENCH_SCRIPT}" --parse-only "${RESULT_DIR}"
+    summary="${RESULT_DIR}/summary.reparsed.md"
   fi
-  log "----- summary.md -----"
-  cat "${RESULT_DIR}/summary.md"
+  log "----- ${summary} -----"
+  cat "${summary}"
   log "step 6 complete"
 }