diff --git a/.github/workflows/environment-compatibility-tests.yml b/.github/workflows/environment-compatibility-tests.yml index bf21500..eb31d15 100644 --- a/.github/workflows/environment-compatibility-tests.yml +++ b/.github/workflows/environment-compatibility-tests.yml @@ -9,6 +9,10 @@ on: - rustfs-upgrade-test.sh - rustfs_performance_test.sh - scripts/check_execution_environment.py + - scripts/check_shell_syntax.py + - scripts/prepare_table_smoke.py + - test_check_shell_syntax.py + - .github/workflows/rustfs-table-test.yml - scripts/lib/** - test_check_execution_environment.py - test_rustfs_table.py @@ -24,6 +28,10 @@ on: - rustfs-upgrade-test.sh - rustfs_performance_test.sh - scripts/check_execution_environment.py + - scripts/check_shell_syntax.py + - scripts/prepare_table_smoke.py + - test_check_shell_syntax.py + - .github/workflows/rustfs-table-test.yml - scripts/lib/** - test_check_execution_environment.py - test_rustfs_table.py @@ -44,21 +52,14 @@ jobs: persist-credentials: false - name: Validate shell syntax run: | - for script in \ - rustfs-table-test.sh \ - rustfs-replication-test.sh \ - rustfs-security-test.sh \ - rustfs-upgrade-test.sh \ - rustfs_performance_test.sh - do - bash -n "${script}" - done + python3 -B scripts/check_shell_syntax.py --report "${RUNNER_TEMP}/shell-syntax.json" test -x rustfs-security-test.sh test -x rustfs-upgrade-test.sh - name: Test portable execution helpers run: >- python3 -B -m unittest -v test_check_execution_environment.py + test_check_shell_syntax.py test_rustfs_table.py test_rustfs_performance_testing.py test_rustfs_replication.py diff --git a/.github/workflows/package-install-tests.yml b/.github/workflows/package-install-tests.yml index 73b4927..7e80347 100644 --- a/.github/workflows/package-install-tests.yml +++ b/.github/workflows/package-install-tests.yml @@ -7,6 +7,8 @@ on: - rustfs_pool_expand.sh - rustfs_performance_test.sh - test_rustfs_pool_expand.sh + - scripts/storage_capacity.py + - test_storage_capacity.py - .github/workflows/package-install-tests.yml push: branches: [main] @@ -15,6 +17,8 @@ on: - rustfs_pool_expand.sh - rustfs_performance_test.sh - test_rustfs_pool_expand.sh + - scripts/storage_capacity.py + - test_storage_capacity.py - .github/workflows/package-install-tests.yml permissions: @@ -35,3 +39,5 @@ jobs: done - name: Test package installation without a server run: bash test_rustfs_pool_expand.sh + - name: Test capacity stop decisions + run: python3 -B -m unittest -v test_storage_capacity.py diff --git a/.github/workflows/rustfs-performance-test.yml b/.github/workflows/rustfs-performance-test.yml index 2d1e5e6..2759386 100644 --- a/.github/workflows/rustfs-performance-test.yml +++ b/.github/workflows/rustfs-performance-test.yml @@ -322,6 +322,11 @@ jobs: ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/master.log ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/summary.md ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/summary.tsv + ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/rounds.tsv + ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/master.reparsed.log + ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/summary.reparsed.md + ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/summary.reparsed.tsv + ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/rounds.reparsed.tsv ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/get_*.txt ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/put_*.txt ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/mixed_*.txt diff --git a/.github/workflows/rustfs-table-test.yml b/.github/workflows/rustfs-table-test.yml index 4dd41df..edf2594 100644 --- a/.github/workflows/rustfs-table-test.yml +++ b/.github/workflows/rustfs-table-test.yml @@ -22,6 +22,10 @@ on: required: false type: boolean default: false + source_ref: + description: 'RustFS source ref matching the standalone candidate (defaults to release tag or main)' + required: false + type: string concurrency: group: rustfs-shared-functional-tests-v2 @@ -80,6 +84,26 @@ jobs: aws --version python3 --version + - name: Checkout matching Table smoke sources + if: ${{ !inputs.skip_smoke }} + uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7 + with: + repository: rustfs/rustfs + ref: ${{ steps.chain.outputs.source_sha || inputs.source_ref || inputs.rustfs_version || 'main' }} + path: .rustfs-table-source + sparse-checkout: scripts/table-catalog + persist-credentials: false + + - name: Prepare complete Table smoke module tree + if: ${{ !inputs.skip_smoke }} + env: + EXPECTED_SOURCE_SHA: ${{ steps.chain.outputs.source_sha }} + run: >- + python3 scripts/prepare_table_smoke.py prepare + --source-root "${GITHUB_WORKSPACE}/.rustfs-table-source" + --destination "${FUNCTIONAL_ARTIFACTS_DIR}/table-catalog" + --expected-sha "${EXPECTED_SOURCE_SHA}" + - name: Admit clean shared environment id: environment timeout-minutes: 5 @@ -123,7 +147,7 @@ jobs: chmod +x rustfs-table-test.sh PACKAGE_URL='${{ steps.chain_package.outputs.package_url || inputs.package_url }}' RUSTFS_VERSION='${{ inputs.rustfs_version }}' - SMOKE_PATH="${GITHUB_WORKSPACE:-$PWD}/scripts/table-catalog/pyiceberg_smoke.py" + SMOKE_PATH="${FUNCTIONAL_ARTIFACTS_DIR}/table-catalog/pyiceberg_smoke.py" ARGS=(-y --smoke-script "${SMOKE_PATH}") if [ '${{ inputs.skip_smoke }}' = 'true' ]; then ARGS+=(--skip-smoke) @@ -284,6 +308,7 @@ jobs: ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/cases.md ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log + ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/table-catalog/source-manifest.json if-no-files-found: error retention-days: 30 diff --git a/.github/workflows/validation-catalog-tests.yml b/.github/workflows/validation-catalog-tests.yml index 2a1d097..17ce043 100644 --- a/.github/workflows/validation-catalog-tests.yml +++ b/.github/workflows/validation-catalog-tests.yml @@ -14,6 +14,7 @@ on: - scripts/validation_ilm/** - test_validation_pool.py - scripts/validation_pool/** + - scripts/storage_capacity.py - rustfs_pool_expand.sh - test_validation_replication.py - test_rustfs_replication.py @@ -88,6 +89,7 @@ on: - scripts/validation_ilm/** - test_validation_pool.py - scripts/validation_pool/** + - scripts/storage_capacity.py - rustfs_pool_expand.sh - test_validation_replication.py - test_rustfs_replication.py @@ -164,9 +166,9 @@ jobs: - run: python3 -B scripts/validation_migration_review.py --require-complete - run: python3 -B -m unittest -v test_validation_migration.py - run: python3 -B -m unittest -v test_validation_ilm.py + - run: python3 -m pip install -r scripts/validation_ilm/requirements.txt - run: python3 -B -m unittest -v test_validation_pool.py - run: python3 -B -m unittest -v test_validation_replication.py test_rustfs_replication.py - - run: python3 -m pip install -r scripts/validation_ilm/requirements.txt - name: Install network namespace test dependency run: | if ! command -v nft >/dev/null; then diff --git a/docs/suite-environment-retest.zh-CN.md b/docs/suite-environment-retest.zh-CN.md new file mode 100644 index 0000000..0540362 --- /dev/null +++ b/docs/suite-environment-retest.zh-CN.md @@ -0,0 +1,45 @@ +# 测试套件与环境问题修复后的复测 + +本次改动对应 [backlog #3173](https://github.com/rustfs/backlog/issues/3173) 的测试侧归因，基于已合并的 auto-testing #144。它不代表修复了服务端 503、命名空间锁超时或性能异常。 + +| 原失败 | 修复行为 | 复测验收 | +|---|---|---| +| TBL-101 导入失败 | 从候选对应的 RustFS checkout 准备整个 `scripts/table-catalog`，记录源码 SHA 与文件哈希；先检查本地模块，再创建 venv | 端到端实际读写成功；缺模块必须报告工具准备错误 | +| M-1x2EC1 启动非参与节点 | 显式传递参与节点数量，只对这些节点提交非阻塞启动并等待 active | 完整保留各拓扑 / EC / 故障探针，不能用未执行代替通过 | +| Rebalance 活动判据超时 | 单独增大 Rebalance 活动夹具并先校验容量，保留 `Started` 与非零进度断言；提前完成明确报告活动窗口不足 | stop/resume、live I/O、operation guards 必须在活动窗口执行；完成态仍要求非零迁移进度及数据校验 | +| 扩池负载写满单盘 | 2 秒容量采样，单盘利用率与剩余字节保护；预留在途对象和观测到的增长量，缩短停止等待 | 达到目标才通过；容量保护提前停止是环境容量失败，仍执行 MPU 清理 | +| 性能轮次含服务错误却计入摘要 | 错误日志使该轮失败，原始指标保留作诊断；逐轮记录 Report 是否出现、退出码、错误行数 | `rounds.tsv` 和 Markdown 包含全部选定轮次；超时与未执行都不算通过 | +| AppleDouble 输入 / 语法报告缺口 | 排除 `._*`，对每个真实脚本独立执行 `bash -n`，保存逐文件结果 | 任一真实脚本失败，整体返回非零；空选择也不通过 | + +## Table 源码准备 + +```bash +python3 scripts/prepare_table_smoke.py prepare \ + --source-root /path/to/rustfs-checkout \ + --destination /path/to/new-run/table-catalog \ + --expected-sha FULL_CANDIDATE_SOURCE_SHA +``` + +`--destination` 必须是新目录，避免混入旧版本模块。链式 workflow 使用 manifest 的 source SHA；独立 workflow 可通过 `source_ref` 指定与产物匹配的源码。未指定时使用 release tag 或当次 checkout 的 main，并在 `source-manifest.json` 保留实际 SHA。 + +Table workflow 将 `table-catalog/source-manifest.json` 与报告、用例表和日志一起上传，保留实际 smoke 源码 SHA 与文件哈希，供 runner 临时目录清理后追溯。 + +## 容量配置 + +Rebalance 源用例明确要求活动阶段已有非零迁移进度。上一轮每对象 32 MiB 的任务在首次周期保存进度前完成，因此不能通过。专用活动夹具默认改为每对象 128 MiB（默认 128 对象，其中 3 个边界对象更小），通过 `RUSTFS_ACTIVE_REBALANCE_FIXTURE_BYTES` / `--active-rebalance-fixture-bytes` 配置，并记录在 run manifest。写入前按 EC 占用上界与每盘至少 1 GiB 的剩余余量检查容量。其他 Decommission 活动夹具保留原大小；更快的机器仍可能需要增加夹具，窗口不足时用例失败而不是放松断言。 + +容量采样与 Rebalance / Decommission 的 `--poll-interval` 分开。默认每 2 秒采样，一次直接 API 请求上限 3 秒，活动写入期间不做长重试或代理探测。代理仍在写入前后探测，错误继续保留。 + +预留量为每盘最少剩余字节（默认 512 MiB）加 `对象大小 × 并发数`，再加观测到的集群增长速率乘以采样、请求与停止窗口。集群增长作为单盘增长的保守上界，不假设数据均匀。容量库存为空、无效或发生变化会停止继续写入。 + +可配置 `RUSTFS_CAPACITY_POLL_INTERVAL`、`RUSTFS_CAPACITY_SAMPLE_TIMEOUT`、`RUSTFS_CAPACITY_MIN_FREE_BYTES`、`RUSTFS_CAPACITY_MAX_DISK_PERCENT`、`RUSTFS_WARP_STOP_GRACE`、`RUSTFS_WARP_OBJ_SIZE`。小容量环境应先按磁盘余量降低并发或目标利用率。旁路 `.capacity.jsonl` 保留每次判断的输入、余量与停止原因；这是一项负载保护，无法保证在不可预测的突发写入下绝不耗尽容量。 + +MPU 列表错误会保留 HTTP status、`x-rustfs-error-cause`、request ID 与 retry-after（服务端返回时）；不采集完整 AWS debug 输出或响应 Cookie。 + +## 结果边界 + +性能轮次总超时仍覆盖准备、负载和收尾。`timeout_before_report` / `timeout_after_report` 仅说明是否出现 Warp Report，不能推断 Warp 内部卡在具体阶段。不要把扩池发生的磁盘满用来解释此前的性能超时。 + +`--parse-only` 输出独立的 `rounds.reparsed.tsv`、`summary.reparsed.tsv`、`summary.reparsed.md` 与 `master.reparsed.log`，保留原执行台账、摘要及日志。重解析无法恢复耗时和原始运行配置；应以原 `rounds.tsv` 中的 `NOT_RUN / aborted` 及耗时为执行证据。性能 workflow 同时上传原执行台账与重解析产物。步骤 6 缺少原摘要时展示重解析摘要，重解析失败仍返回非零。 + +复测需保留产品源码 SHA、CI 产物身份、二进制 SHA-256、测试提交 SHA、实际配置和完整逐条结果。VM 写操作及最终恢复使用同一个共享锁；恢复后核对四节点、24 个数据挂载与对象校验。 diff --git a/rustfs-fault-tolerance-test.sh b/rustfs-fault-tolerance-test.sh index 902d519..2aef560 100755 --- a/rustfs-fault-tolerance-test.sh +++ b/rustfs-fault-tolerance-test.sh @@ -551,8 +551,20 @@ seed_baseline() { # seed_baseline } # cluster lifecycle ---------------------------------------------------------- -install_and_start() { # install_and_start [extra env line] - local volumes="$1" extra_env="${2:-}" node +install_and_start() { # install_and_start [extra env line] [participant count] + local volumes="$1" extra_env="${2:-}" count="${3:-}" node + local participants=() + if [ -z "${count}" ] && [ "${volumes}" = "${VOLUMES_SNMD}" ]; then + participants=("${SNMD_NODE}") + else + if [ -z "${count}" ]; then + count=${#NODES[@]} + [ "${volumes}" != "${VOLUMES_3X4}" ] || count=3 + fi + [[ "${count}" =~ ^[1-9][0-9]*$ ]] && [ "${count}" -le "${#NODES[@]}" ] \ + || die "invalid topology participant count: ${count}" + participants=("${NODES[@]:0:count}") + fi [ -n "${PACKAGE_URL}" ] || die "--package-url is required for test runs" log "installing package on ${NODES[*]}" remote_all "set -e; sudo -n systemctl stop ${RUSTFS_SERVICE} 2>/dev/null || true; sudo -n dpkg --configure -a 2>/dev/null || true; curl -fsSL --retry 3 -o /tmp/rustfs-ft.deb '${PACKAGE_URL}' && sudo -n env DEBIAN_FRONTEND=noninteractive dpkg -i --force-confdef --force-confnew /tmp/rustfs-ft.deb && sudo -n systemctl daemon-reload" @@ -573,27 +585,8 @@ install_and_start() { # install_and_start [extra env line] ssh "${SSH_OPTS[@]}" "${SSH_USER}@${node}" "sudo -n tee ${RUSTFS_CONFIG_FILE} >/dev/null" < "${TMP}/envfile" run_remote "${node}" 'sudo -n mkdir -p /data/rustfs{1..4}/mnmd && for i in 1 2 3 4; do sudo -n chown rustfs:rustfs "/data/rustfs${i}" "/data/rustfs${i}/mnmd"; done 2>/dev/null || true' done - if [ "${volumes}" = "${VOLUMES_SNMD}" ]; then - log "single-node topology: starting ${RUSTFS_SERVICE} on ${SNMD_NODE} only" - if run_remote "${SNMD_NODE}" "sudo -n systemctl start ${RUSTFS_SERVICE}"; then - : - else - dump_service_diagnostics "${SNMD_NODE}" - die "start failed on ${SNMD_NODE}" - fi - else - log "starting ${RUSTFS_SERVICE} on ${NODES[*]}" - local start_pids=() start_i=0 start_pid start_fail=0 - for node in "${NODES[@]}"; do - run_remote "${node}" "sudo -n systemctl start ${RUSTFS_SERVICE}" & - start_pids[${start_i}]=$!; start_i=$((start_i+1)) - done - for start_pid in "${start_pids[@]}"; do wait "${start_pid}" || start_fail=1; done - if [ "${start_fail}" -ne 0 ]; then - dump_service_diagnostics "${NODES[@]}" - die "cluster start failed on one or more nodes" - fi - fi + log "topology participants: ${participants[*]}" + start_nodes "${participants[@]}" if [ "${DRY_RUN}" -eq 1 ]; then log "DRY-RUN: skipping cluster readiness polling" return 0 @@ -953,7 +946,7 @@ matrix_run_case() { # done mc+=("${tag}-rec/read" "${tag}-rec/write") register_expected_cases "${mc[@]}" - install_and_start "$(matrix_volumes_expr "${nodes}" "${per}")" "RUSTFS_STORAGE_CLASS_STANDARD=\"EC:${parity}\"" + install_and_start "$(matrix_volumes_expr "${nodes}" "${per}")" "RUSTFS_STORAGE_CLASS_STANDARD=\"EC:${parity}\"" "${nodes}" seed_baseline "${EP1}" # full capacity diff --git a/rustfs-table-test.sh b/rustfs-table-test.sh index f1dcf96..612af79 100755 --- a/rustfs-table-test.sh +++ b/rustfs-table-test.sh @@ -246,11 +246,8 @@ cleanup_all() { # ==================== TBL-101: PyIceberg end-to-end (product smoke) ==================== test_tbl101() { if [ "${SKIP_SMOKE}" -eq 1 ]; then echo "smoke skipped by flag"; return 2; fi - if [ "${SKIP_SMOKE}" -eq 1 ]; then echo "tbl101: smoke skipped by flag"; return 2; fi - if [ -z "${SMOKE_SCRIPT}" ] || [ ! -f "${SMOKE_SCRIPT}" ]; then - echo "tbl101: smoke script not found: '${SMOKE_SCRIPT:-}' (cwd=$(pwd), SKIP_SMOKE=${SKIP_SMOKE})" - return 2 - fi + python3 "${SCRIPT_DIR}/scripts/prepare_table_smoke.py" check --smoke-script "${SMOKE_SCRIPT}" \ + || return 1 echo "tbl101: smoke script ok, creating venv" local venv="/tmp/tbl-venv-${RUN_ID}" if ! python3 -m venv "${venv}" >/tmp/tbl-venv-err.log 2>&1; then diff --git a/rustfs_performance_test.sh b/rustfs_performance_test.sh index 3f1931d..d5c2e46 100755 --- a/rustfs_performance_test.sh +++ b/rustfs_performance_test.sh @@ -488,6 +488,7 @@ step5_benchmark() { } step6_analyze() { + local summary="${RESULT_DIR}/summary.md" log "step 6: analyze results from ${RESULT_DIR}" if [ "${DRY_RUN}" -eq 1 ]; then log "DRY-RUN: bash ${BENCH_SCRIPT} --parse-only ${RESULT_DIR}" @@ -496,14 +497,17 @@ step6_analyze() { if [ ! -d "${RESULT_DIR}" ]; then die "result directory not found: ${RESULT_DIR}" fi - if [ -f "${RESULT_DIR}/summary.md" ]; then - log "summary already generated: ${RESULT_DIR}/summary.md" + if [ -f "${summary}" ]; then + log "summary already generated: ${summary}" else log "generating summary with --parse-only" + WARP_METHODS="${WARP_METHODS}" \ + WARP_SIZES="${WARP_SIZES}" \ bash "${BENCH_SCRIPT}" --parse-only "${RESULT_DIR}" + summary="${RESULT_DIR}/summary.reparsed.md" fi - log "----- summary.md -----" - cat "${RESULT_DIR}/summary.md" + log "----- ${summary} -----" + cat "${summary}" log "step 6 complete" } diff --git a/rustfs_performance_testing.sh b/rustfs_performance_testing.sh index 436a0a2..65d37eb 100755 --- a/rustfs_performance_testing.sh +++ b/rustfs_performance_testing.sh @@ -54,13 +54,20 @@ CONSECUTIVE_FAILURES=0 # --parse-only : parse an existing result dir (${method}_${size}.txt) without running # Only round files with a successful terminal execution status can be certified. +OUTPUT_TAG="" if [[ "${1:-}" == "--parse-only" && -n "${2:-}" ]]; then RESULT_DIR="$2" fi +if [[ "${1:-}" == "--parse-only" ]]; then + # Reanalysis must not rewrite the execution's abort states, timings or reports. + OUTPUT_TAG=".reparsed" +fi -LOG_FILE="${RESULT_DIR}/master.log" -SUMMARY_TSV="${RESULT_DIR}/summary.tsv" -SUMMARY_MD="${RESULT_DIR}/summary.md" +LOG_FILE="${RESULT_DIR}/master${OUTPUT_TAG}.log" +SUMMARY_TSV="${RESULT_DIR}/summary${OUTPUT_TAG}.tsv" +SUMMARY_MD="${RESULT_DIR}/summary${OUTPUT_TAG}.md" +ROUNDS_TSV="${RESULT_DIR}/rounds${OUTPUT_TAG}.tsv" +PARSE_FAILURE_REASON="" if [[ "${1:-}" != "--parse-only" ]] && ! command -v warp >/dev/null 2>&1; then echo "error: warp command not found; install MinIO warp first" >&2 @@ -73,6 +80,32 @@ log() { # ---- Result parsing ---- +round_error_lines() { + awk ' + { line = tolower($0) + if (line ~ /|serviceunavailable|the service is unavailable|errors:[[:space:]]*[1-9][0-9]*/) count++ } + END { print count + 0 } + ' "$1" +} + +record_round() { + local method="$1" size="$2" file="$3" verdict="$4" reason="$5" warp_rc="$6" capture_rc="$7" elapsed="$8" + local reported=0 errors=0 + if [[ "${verdict}" != NOT_RUN && -f "${file}" ]]; then + grep -q '^Report: ' "${file}" && reported=1 + errors=$(round_error_lines "${file}") + fi + if [[ "${warp_rc}" == 124 ]]; then + reason=timeout_before_report + [[ "${reported}" == 0 ]] || reason=timeout_after_report + elif [[ "${warp_rc}" == 137 ]]; then + reason=killed_before_report + [[ "${reported}" == 0 ]] || reason=killed_after_report + fi + printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \ + "${method}" "${size}" "${verdict}" "${reason}" "${warp_rc}" "${capture_rc}" "${elapsed}" "${reported}" "${errors}" >> "${ROUNDS_TSV}" +} + # Extract the raw Average / Reqs / TTFB lines of a section (GET/PUT/Total etc.) section_lines() { awk -v sec="$2" ' @@ -107,10 +140,17 @@ field() { # Parse one round's output and append a row to summary.tsv parse_round() { local method="$1" size="$2" file="$3" + PARSE_FAILURE_REASON="" if [[ "$(tail -n 1 "$file")" != "RustFS round status: warp=0 capture=0" ]]; then + PARSE_FAILURE_REASON=missing_or_failed_round_status log "PARSE_FAILED method=${method} size=${size} reason=missing_or_failed_round_status file=${file}" return 1 fi + if [[ "$(round_error_lines "$file")" -gt 0 ]]; then + PARSE_FAILURE_REASON=logged_errors + log "PARSE_FAILED method=${method} size=${size} reason=logged_errors file=${file} (metrics retained in raw output)" + return 1 + fi local line if [[ "${method}" == "mixed" ]]; then local total get put @@ -145,6 +185,7 @@ parse_round() { { last = $1 == "mixed" ? 6 : 8 for (i = 3; i <= last; i++) if ($i !~ /[^[:space:]]/) exit 1 } '; then + PARSE_FAILURE_REASON=missing_required_metrics log "PARSE_FAILED method=${method} size=${size} reason=missing_required_metrics file=${file}" return 1 fi @@ -161,6 +202,16 @@ gen_summary_md() { echo "- Concurrency: ${CONCURRENCY}; duration: ${DURATION}; sleep: ${SLEEP_BETWEEN_ROUNDS}s; GET objects: ${GET_OBJECTS}" echo "- Methods: ${METHODS[*]-}; sizes: ${SIZES[*]-}" echo "- Rounds: selected=${TOTAL_ROUNDS}; parsed=${PARSED_ROUNDS}; failed=${FAILED_ROUNDS}" + if [[ -n "${OUTPUT_TAG}" ]]; then + echo "- Reparsed raw outputs: elapsed time is unknown; configuration above is from this parser invocation. See the original rounds.tsv, master.log and summary files for execution evidence." + fi + echo "- Timeout covers preparation, workload and teardown. Report presence only distinguishes timeout before/after a report; it does not prove the internal phase." + echo "" + echo "## Every selected round" + echo "" + echo "| Method | Size | Verdict | Reason | Warp exit | Capture exit | Elapsed seconds | Report present | Error lines |" + echo "|---|---|---|---|---|---|---|---|---|" + awk -F '\t' 'NR > 1 { printf "| %s | %s | %s | %s | %s | %s | %s | %s | %s |\n", $1,$2,$3,$4,$5,$6,$7,$8,$9 }' "${ROUNDS_TSV}" echo "" } > "${SUMMARY_MD}" @@ -196,6 +247,7 @@ gen_summary_md() { mkdir -p "${RESULT_DIR}" || exit 1 log "CONFIG host=${HOST} bucket=${BUCKET} concurrency=${CONCURRENCY} duration=${DURATION} get_objects=${GET_OBJECTS} sleep_between_rounds=${SLEEP_BETWEEN_ROUNDS}s" || exit 1 printf 'method\tsize\tthroughput\tobj_per_s\treq_avg\treq_p50\treq_p90\treq_p99\tttfb_avg\tttfb_p99\tttfb_worst\n' > "${SUMMARY_TSV}" || exit 1 +printf 'method\tsize\tverdict\treason\twarp_exit\tcapture_exit\telapsed_seconds\treport_present\terror_lines\n' > "${ROUNDS_TSV}" || exit 1 if [[ ${TOTAL_ROUNDS} -eq 0 ]]; then log "NO_ROUNDS_SELECTED" @@ -207,11 +259,21 @@ if [[ "${1:-}" == "--parse-only" ]]; then for method in "${METHODS[@]}"; do for size in "${SIZES[@]}"; do outfile="${RESULT_DIR}/${method}_${size}.txt" + PARSE_FAILURE_REASON=missing_output + warp_rc=unknown; capture_rc=unknown + if [[ -s "${outfile}" ]]; then + terminal=$(tail -n 1 "${outfile}") + if [[ "${terminal}" =~ ^RustFS\ round\ status:\ warp=([0-9]+)\ capture=([0-9]+)$ ]]; then + warp_rc=${BASH_REMATCH[1]}; capture_rc=${BASH_REMATCH[2]} + fi + fi if [[ -s "${outfile}" ]] && parse_round "${method}" "${size}" "${outfile}"; then PARSED_ROUNDS=$((PARSED_ROUNDS + 1)) + record_round "${method}" "${size}" "${outfile}" PASS clean "${warp_rc}" "${capture_rc}" unknown else FAILED_ROUNDS=$((FAILED_ROUNDS + 1)) log "PARSE_FAILED method=${method} size=${size} file=${outfile}" + record_round "${method}" "${size}" "${outfile}" FAIL "${PARSE_FAILURE_REASON}" "${warp_rc}" "${capture_rc}" unknown fi done done @@ -257,15 +319,19 @@ for method in "${METHODS[@]}"; do capture_rc=${round_status[1]} end_epoch=$(date +%s) printf '\nRustFS round status: warp=%s capture=%s\n' "${rc}" "${capture_rc}" >> "${outfile}" || capture_rc=1 + PARSE_FAILURE_REASON=warp_exit + [[ ${capture_rc} -eq 0 ]] || PARSE_FAILURE_REASON=output_capture if [[ ${rc} -eq 0 && ${capture_rc} -eq 0 ]] && parse_round "${method}" "${size}" "${outfile}"; then PARSED_ROUNDS=$((PARSED_ROUNDS + 1)) CONSECUTIVE_FAILURES=0 log "END round=${ROUND}/${TOTAL_ROUNDS} method=${method} size=${size} rc=${rc} elapsed=$((end_epoch - start_epoch))s parsed=ok" + record_round "${method}" "${size}" "${outfile}" PASS clean "${rc}" "${capture_rc}" "$((end_epoch - start_epoch))" else FAILED_ROUNDS=$((FAILED_ROUNDS + 1)) CONSECUTIVE_FAILURES=$((CONSECUTIVE_FAILURES + 1)) log "END round=${ROUND}/${TOTAL_ROUNDS} method=${method} size=${size} rc=${rc} capture_rc=${capture_rc} elapsed=$((end_epoch - start_epoch))s parsed=failed" + record_round "${method}" "${size}" "${outfile}" FAIL "${PARSE_FAILURE_REASON}" "${rc}" "${capture_rc}" "$((end_epoch - start_epoch))" if [[ "${MAX_CONSECUTIVE_FAILURES}" =~ ^[1-9][0-9]*$ ]] \ && (( CONSECUTIVE_FAILURES >= MAX_CONSECUTIVE_FAILURES )); then log "ABORT consecutive_failures=${CONSECUTIVE_FAILURES} limit=${MAX_CONSECUTIVE_FAILURES}" @@ -280,6 +346,16 @@ for method in "${METHODS[@]}"; do done done +# An early abort leaves explicit unexecuted rows rather than shortening the +# selected matrix silently. Raw metrics from failed rounds remain diagnostic. +for method in "${METHODS[@]}"; do + for size in "${SIZES[@]}"; do + if ! awk -F '\t' -v m="${method}" -v s="${size}" 'NR>1 && $1==m && $2==s {found=1} END {exit !found}' "${ROUNDS_TSV}"; then + record_round "${method}" "${size}" "" NOT_RUN aborted unknown unknown 0 + fi + done +done + gen_summary_md || exit 1 log "ALL_ROUNDS_COMPLETE selected=${TOTAL_ROUNDS} parsed=${PARSED_ROUNDS} failed=${FAILED_ROUNDS} summary_tsv=${SUMMARY_TSV} summary_md=${SUMMARY_MD}" || exit 1 echo "" diff --git a/rustfs_pool_expand.sh b/rustfs_pool_expand.sh index ec8e890..f1830d6 100755 --- a/rustfs_pool_expand.sh +++ b/rustfs_pool_expand.sh @@ -34,6 +34,7 @@ # order when expanding. # set -Eeuo pipefail +POOL_SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # ==================== Configuration (adjust to your environment) ==================== @@ -107,7 +108,7 @@ RUSTFS_OBS_LOG_DIRECTORY="/var/log/rustfs/" # Data writing & monitoring (step 4) WARP_BUCKET="test-10mb" -WARP_OBJ_SIZE="100MiB" +WARP_OBJ_SIZE="${RUSTFS_WARP_OBJ_SIZE:-100MiB}" WARP_CONCURRENT="${RUSTFS_WARP_CONCURRENT:-32}" WARP_DURATION="5m" WARP_ROUNDS="${RUSTFS_WARP_ROUNDS:-1}" @@ -116,6 +117,14 @@ WARP_ROUNDS="${RUSTFS_WARP_ROUNDS:-1}" WARP_LOG_FILE="${RUSTFS_WARP_LOG_FILE:-}" STORAGE_THRESHOLD=85 # stop writing when usage reaches N% (note suggests 80-85) POLL_INTERVAL=30 # status polling interval (seconds) +CAPACITY_POLL_INTERVAL="${RUSTFS_CAPACITY_POLL_INTERVAL:-2}" +CAPACITY_SAMPLE_TIMEOUT="${RUSTFS_CAPACITY_SAMPLE_TIMEOUT:-3}" +CAPACITY_MIN_FREE_BYTES="${RUSTFS_CAPACITY_MIN_FREE_BYTES:-536870912}" +CAPACITY_MAX_DISK_PERCENT="${RUSTFS_CAPACITY_MAX_DISK_PERCENT:-95}" +WARP_STOP_GRACE="${RUSTFS_WARP_STOP_GRACE:-5}" +CAPACITY_PREVIOUS_SAMPLE="" +CAPACITY_LOG_FILE="" +CAPACITY_EMERGENCY=0 WARP_TRANSPORT_FAILURE_PATTERN='502 Bad Gateway|504 Gateway Time-out|HTTP([^0-9]+)?50(2|4)|status([^0-9]+)?50(2|4)|connection (closed|reset|refused)|unexpected EOF|broken pipe|timeout awaiting response headers|context deadline exceeded|i/o timeout|Client\.Timeout exceeded|The service is unavailable|We encountered an internal error' # Timeouts (seconds) @@ -247,6 +256,10 @@ normalize_http_endpoint() { } resolve_endpoints() { + local capacity_value + for capacity_value in "${CAPACITY_POLL_INTERVAL}" "${CAPACITY_SAMPLE_TIMEOUT}" "${CAPACITY_MIN_FREE_BYTES}" "${WARP_STOP_GRACE}"; do + [[ "${capacity_value}" =~ ^[1-9][0-9]*$ ]] || die "capacity bounds must be positive integers: ${capacity_value}" + done [[ "${WARP_CONCURRENT}" =~ ^[1-9][0-9]*$ ]] \ || die "warp concurrency must be a positive integer: ${WARP_CONCURRENT}" [[ "${WARP_ROUNDS}" =~ ^[1-9][0-9]*$ ]] \ @@ -282,6 +295,7 @@ cleanup_runtime() { kill "${ACTIVE_WARP_PID}" 2>/dev/null || true fi rm -f "${ADMIN_API_CODE_FILE}" "${ADMIN_API_TIMING_FILE}" \ + "${ADMIN_API_CODE_FILE}.headers" "${PROXY_API_CODE_FILE}.headers" \ "${PROXY_API_CODE_FILE}" "${PROXY_API_TIMING_FILE}" \ "${ADMIN_ANOMALY_FILE}" "${PROXY_ANOMALY_FILE}" } @@ -349,11 +363,12 @@ api_request() { local amz_date date_stamp host_port local canonical_headers signed_headers canonical_request string_to_sign local scope k_date k_region k_service k_signing signature auth - local url tmp metrics code connect_time starttransfer_time total_time curl_exit=0 + local url tmp header_tmp metrics code connect_time starttransfer_time total_time curl_exit=0 if [ "${DRY_RUN}" -eq 1 ]; then log "DRY-RUN: API ${method} ${endpoint}${path}${query:+?${query}}" printf '200' > "${code_file}" + : > "${code_file}.headers" printf 'http=200 connect=0 ttfb=0 total=0 curl_exit=0' > "${timing_file}" return 0 fi @@ -391,7 +406,8 @@ $(sha256_hex "${canonical_request}")" url="${endpoint}${path}${query:+?${query}}" tmp="$(mktemp)" - if metrics="$(curl -sS --max-time "${API_REQUEST_TIMEOUT}" -o "${tmp}" \ + header_tmp="$(mktemp)" + if metrics="$(curl -sS --max-time "${API_REQUEST_TIMEOUT}" -o "${tmp}" -D "${header_tmp}" \ -w $'%{http_code}\t%{time_connect}\t%{time_starttransfer}\t%{time_total}' \ -H "Host: ${host_port}" \ -H "x-amz-content-sha256: UNSIGNED-PAYLOAD" \ @@ -408,11 +424,15 @@ $(sha256_hex "${canonical_request}")" fi code="${code:-000}" printf '%s' "${code}" > "${code_file}" + # Only retain diagnostic response headers. Never publish Authorization, + # cookies or the full AWS debug stream to collect an error cause. + awk 'tolower($0) ~ /^(http\/|x-rustfs-error-cause:|x-amz-request-id:|x-amz-id-2:|retry-after:)/ {sub(/\r$/, ""); print}' \ + "${header_tmp}" > "${code_file}.headers" printf 'http=%s connect=%s ttfb=%s total=%s curl_exit=%s' \ "${code}" "${connect_time:-unknown}" "${starttransfer_time:-unknown}" \ "${total_time:-unknown}" "${curl_exit}" > "${timing_file}" cat "${tmp}" - rm -f "${tmp}" + rm -f "${tmp}" "${header_tmp}" } admin_api() { @@ -432,6 +452,10 @@ admin_api_timing() { cat "${ADMIN_API_TIMING_FILE}" } +admin_api_headers() { + [ ! -f "${ADMIN_API_CODE_FILE}.headers" ] || cat "${ADMIN_API_CODE_FILE}.headers" +} + proxy_api_code() { cat "${PROXY_API_CODE_FILE}" } @@ -869,44 +893,71 @@ assert_no_endpoint_anomalies() { || die "endpoint anomalies occurred; retries preserved diagnostics but do not qualify as a clean pass" } -# Cluster storage usage percentage from the admin API (aggregate across disks) -storage_usage_percent() { +# Storage sampling while a writer is active must be bounded: no long proxy +# probe or retry loop can delay the capacity stop decision. +storage_capacity_sample() { local body sample_time if [ "${DRY_RUN}" -eq 1 ]; then log "DRY-RUN: GET ${ADMIN_ENDPOINT}/rustfs/admin/v3/storageinfo (storage usage)" - printf '0\n' + printf '{"usage_percent":0}\n' return 0 fi sample_time="$(date -u +%Y-%m-%dT%H:%M:%SZ)" printf '\033[1;36m[PROBE]\033[0m %s storageinfo sample direct=%s proxy=%s\n' \ "${sample_time}" "${ADMIN_ENDPOINT}" "${PROXY_ENDPOINT:-disabled}" >&2 - body="$(storageinfo_with_retry admin)" \ - || die "direct storageinfo API remained unavailable at ${ADMIN_ENDPOINT}" - probe_proxy_storageinfo - if ! printf '%s' "${body}" | jq -e '(.info.disks // .disks // []) | length' >/dev/null 2>&1; then - printf '\033[1;31m[ERROR]\033[0m storageinfo response unexpected (expected JSON with .info.disks or .disks):\n%s\n' "${body}" >&2 - die "storageinfo response could not be parsed" + body="$(API_REQUEST_TIMEOUT="${CAPACITY_SAMPLE_TIMEOUT}" STORAGEINFO_PROBE_RETRIES=1 storageinfo_with_retry admin)" \ + || return 1 + printf '%s' "${body}" | python3 "${POOL_SCRIPT_DIR}/scripts/storage_capacity.py" sample +} + +storage_usage_percent() { + storage_capacity_sample | jq -r '.usage_percent' +} + +assess_capacity() { + local sample="$1" threshold="$2" assessment + assessment="$(printf '%s' "${sample}" | python3 "${POOL_SCRIPT_DIR}/scripts/storage_capacity.py" assess \ + --previous "${CAPACITY_PREVIOUS_SAMPLE}" --threshold "${threshold}" \ + --max-disk-percent "${CAPACITY_MAX_DISK_PERCENT}" --min-free-bytes "${CAPACITY_MIN_FREE_BYTES}" \ + --object-size "${WARP_OBJ_SIZE}" --concurrent "${WARP_CONCURRENT}" \ + --poll-seconds "${CAPACITY_POLL_INTERVAL}" --stop-seconds "$((WARP_STOP_GRACE + 3))" \ + --sample-timeout "${CAPACITY_SAMPLE_TIMEOUT}")" || return 1 + [ -z "${CAPACITY_LOG_FILE}" ] || printf '%s\n' "${assessment}" >> "${CAPACITY_LOG_FILE}" + printf '%s' "${assessment}" +} + +check_capacity_before_write() { + local assessment + CAPACITY_PREVIOUS_SAMPLE="$(storage_capacity_sample)" || return 1 + assessment="$(assess_capacity "${CAPACITY_PREVIOUS_SAMPLE}" "${STORAGE_THRESHOLD}")" || return 1 + if [ "$(printf '%s' "${assessment}" | jq -r '.action')" = capacity_guard ]; then + warn "[ENVIRONMENT-CAPACITY] refusing to start warp: ${assessment}" + return 4 fi - printf '%s' "${body}" | jq -r ' - ([.info.disks // .disks // [] | .[] | select(.state != "offline" and .totalspace > 0) | .usedspace] | add // 0) as $used - | ([.info.disks // .disks // [] | .[] | select(.state != "offline" and .totalspace > 0) | .totalspace] | add // 0) as $total - | if $total > 0 then ($used / $total * 100) else 0 end - ' } # Poll storage usage until it reaches the threshold monitor_storage() { - local threshold="${1:-${STORAGE_THRESHOLD}}" warp_pid="${2:-}" usage - log "monitoring storage usage until ${threshold}% (every ${POLL_INTERVAL}s)" + local threshold="${1:-${STORAGE_THRESHOLD}}" warp_pid="${2:-}" sample assessment action usage + log "monitoring storage usage until ${threshold}% (every ${CAPACITY_POLL_INTERVAL}s, with per-disk reserves)" while :; do - usage="$(storage_usage_percent)" || return 1 - log "current storage usage: ${usage}%" - awk -v u="${usage}" -v t="${threshold}" 'BEGIN{exit !(u >= t)}' && break + sample="$(storage_capacity_sample)" || return 1 + assessment="$(assess_capacity "${sample}" "${threshold}")" || return 1 + CAPACITY_PREVIOUS_SAMPLE="${sample}" + usage="$(printf '%s' "${assessment}" | jq -r '.usage_percent')" + action="$(printf '%s' "${assessment}" | jq -r '.action')" + log "capacity sample: ${assessment}" + if [ "${action}" = capacity_guard ]; then + [ "$(printf '%s' "${assessment}" | jq -r '.emergency')" != true ] || CAPACITY_EMERGENCY=1 + warn "[ENVIRONMENT-CAPACITY] stopping before a disk exhausts its reserved space" + return 4 + fi + [ "${action}" != target_reached ] || break if [ -n "${warp_pid}" ] && ! kill -0 "${warp_pid}" 2>/dev/null; then warn "warp finished at ${usage}%, below the required storage threshold of ${threshold}%" return 3 fi - sleep "${POLL_INTERVAL}" + sleep "${CAPACITY_POLL_INTERVAL}" done log "storage usage reached ${usage}%, stopping writes" } @@ -1175,6 +1226,7 @@ preflight() { fi if [ "${WITH_WARP}" -eq 1 ]; then need_cmd warp "warp benchmark tool" + need_cmd python3 "capacity monitor" fi log "preflight OK" } @@ -1291,6 +1343,7 @@ step4_write_data() { verify_pools 1 if [ "${WITH_WARP}" -eq 1 ]; then need_cmd warp "warp benchmark tool" + need_cmd python3 "capacity monitor" confirm "About to run up to ${WARP_ROUNDS} warp write round(s) against bucket=${WARP_BUCKET} and monitor usage up to ${STORAGE_THRESHOLD}%. Continue?" if [ "${DRY_RUN}" -eq 1 ]; then log "DRY-RUN: warp put --host $(endpoint_authority "${WARP_ENDPOINT}") --bucket ${WARP_BUCKET} --concurrent ${WARP_CONCURRENT} ... (background)" @@ -1300,7 +1353,11 @@ step4_write_data() { local warp_log warp_pid round monitor_rc target_reached=0 warp_stop_waited warp_log="${WARP_LOG_FILE:-$(mktemp "${TMPDIR:-/tmp}/rustfs-warp.XXXXXX.log")}" : >"${warp_log}" + CAPACITY_LOG_FILE="${warp_log}.capacity.jsonl" + probe_proxy_storageinfo for ((round=1; round<=WARP_ROUNDS; round++)); do + CAPACITY_EMERGENCY=0 + check_capacity_before_write || die "[ENVIRONMENT-CAPACITY] capacity admission failed; see ${CAPACITY_LOG_FILE}" log "starting warp round ${round}/${WARP_ROUNDS} against ${WARP_ENDPOINT} (background), log: ${warp_log}" warp put --host "$(endpoint_authority "${WARP_ENDPOINT}")" \ --bucket "${WARP_BUCKET}" \ @@ -1319,9 +1376,13 @@ step4_write_data() { fi # Graceful stop first (SIGINT lets warp finish/abort in-flight multipart # uploads), then SIGTERM/KILL as fallback; see rustfs/backlog#2063. - kill -INT "${warp_pid}" 2>/dev/null || true + if [ "${CAPACITY_EMERGENCY}" -eq 1 ] || [ "${monitor_rc:-0}" -eq 1 ]; then + kill -TERM "${warp_pid}" 2>/dev/null || true + else + kill -INT "${warp_pid}" 2>/dev/null || true + fi warp_stop_waited=0 - while kill -0 "${warp_pid}" 2>/dev/null && [ "${warp_stop_waited}" -lt 60 ]; do + while kill -0 "${warp_pid}" 2>/dev/null && [ "${warp_stop_waited}" -lt "${WARP_STOP_GRACE}" ]; do sleep 1 warp_stop_waited=$((warp_stop_waited + 1)) done @@ -1329,7 +1390,7 @@ step4_write_data() { warn "warp did not exit after SIGINT; sending SIGTERM" kill -TERM "${warp_pid}" 2>/dev/null || true warp_stop_waited=0 - while kill -0 "${warp_pid}" 2>/dev/null && [ "${warp_stop_waited}" -lt 30 ]; do + while kill -0 "${warp_pid}" 2>/dev/null && [ "${warp_stop_waited}" -lt 3 ]; do sleep 1 warp_stop_waited=$((warp_stop_waited + 1)) done @@ -1344,6 +1405,9 @@ step4_write_data() { abort_stale_multipart_uploads "${WARP_BUCKET}" report_warp_errors "${warp_log}" \ || die "warp observed a service or transport failure; see ${warp_log}" + probe_proxy_storageinfo + [ "${monitor_rc:-0}" -ne 4 ] \ + || die "[ENVIRONMENT-CAPACITY] workload stopped for disk safety; target was not certified (see ${CAPACITY_LOG_FILE})" [ "${target_reached}" -eq 1 ] && break [ "${monitor_rc:-0}" -eq 3 ] \ || die "warp monitor failed unexpectedly (status=${monitor_rc:-unknown})" @@ -1441,7 +1505,7 @@ abort_stale_multipart_uploads() { [ -n "${marker_id}" ] && q="${q}&upload-id-marker=${marker_id}" body="$(admin_api GET "/${bucket}" "${q}")" code="$(admin_api_code)" - [ "${code}" = "200" ] || die "list multipart uploads failed (HTTP ${code}): ${body}" + [ "${code}" = "200" ] || die "list multipart uploads failed (HTTP ${code}; $(admin_api_headers)): ${body}" loops=$((loops + 1)) [ "${loops}" -gt 100 ] && die "multipart upload listing did not terminate (pagination loop)" # Split the one-line XML into one ... block per line. @@ -1465,7 +1529,7 @@ abort_stale_multipart_uploads() { # Confirm the bucket has no incomplete uploads left before decommission. body="$(admin_api GET "/${bucket}" "uploads")" code="$(admin_api_code)" - [ "${code}" = "200" ] || die "re-list multipart uploads failed (HTTP ${code}): ${body}" + [ "${code}" = "200" ] || die "re-list multipart uploads failed (HTTP ${code}; $(admin_api_headers)): ${body}" if printf '%s' "${body}" | grep -q ''; then die "multipart upload cleanup incomplete: ${bucket} still has incomplete uploads" fi @@ -1652,6 +1716,9 @@ Options: --decommission-timeout N Decommission wait timeout in seconds (default 86400) --service-timeout N Service start wait timeout in seconds (default 300) --poll-interval N Status polling interval in seconds (default 30) + --capacity-poll-interval N Capacity sample interval in seconds (default 2) + --capacity-min-free-bytes N Minimum reserved bytes on every disk (default 536870912) + --capacity-max-disk-percent N Per-disk emergency stop limit (default 95) --log-file FILE Append all output to FILE --dry-run Preview commands without executing them -y, --yes Skip all confirmation prompts @@ -1765,6 +1832,9 @@ main() { --decommission-timeout) DECOMMISSION_TIMEOUT="$1"; shift ;; --service-timeout) SERVICE_TIMEOUT="$1"; shift ;; --poll-interval) POLL_INTERVAL="$1"; shift ;; + --capacity-poll-interval) CAPACITY_POLL_INTERVAL="$1"; shift ;; + --capacity-min-free-bytes) CAPACITY_MIN_FREE_BYTES="$1"; shift ;; + --capacity-max-disk-percent) CAPACITY_MAX_DISK_PERCENT="$1"; shift ;; --log-file) LOG_FILE="$1"; shift ;; --dry-run) DRY_RUN=1 ;; -y|--yes) ASSUME_YES=1 ;; diff --git a/scripts/check_shell_syntax.py b/scripts/check_shell_syntax.py new file mode 100644 index 0000000..2df6b15 --- /dev/null +++ b/scripts/check_shell_syntax.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 +"""Check every real shell script and retain each command's actual exit status.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +import subprocess + + +EXCLUDED = {".git", "node_modules", ".venv", "venv", "__pycache__"} + + +def check(root: Path, *, bash: str = "bash") -> dict: + files = sorted(path for path in root.rglob("*.sh") + if path.is_file() and not any(part in EXCLUDED or part.startswith("._") + for part in path.relative_to(root).parts)) + results = [] + for path in files: + try: + completed = subprocess.run([bash, "-n", str(path.resolve())], capture_output=True, text=True) + result = {"file": str(path.relative_to(root)), "exit_code": completed.returncode, + "stdout": completed.stdout, "stderr": completed.stderr} + except OSError as exc: + result = {"file": str(path.relative_to(root)), "exit_code": 127, + "stdout": "", "stderr": str(exc)} + results.append(result) + return {"complete": bool(results), "passed": bool(results) and all(r["exit_code"] == 0 for r in results), + "results": results} + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", type=Path, default=Path(__file__).resolve().parents[1]) + parser.add_argument("--report", type=Path) + args = parser.parse_args() + document = check(args.root) + if args.report: + args.report.parent.mkdir(parents=True, exist_ok=True) + args.report.write_text(json.dumps(document, indent=2) + "\n") + for result in document["results"]: + print(f'{"PASS" if result["exit_code"] == 0 else "FAIL"} {result["file"]} exit={result["exit_code"]}') + if result["stderr"]: + print(result["stderr"], end="") + if not document["complete"]: + print("FAIL no shell scripts selected") + return 0 if document["passed"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/prepare_table_smoke.py b/scripts/prepare_table_smoke.py new file mode 100644 index 0000000..f699220 --- /dev/null +++ b/scripts/prepare_table_smoke.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +"""Stage the complete product Table smoke sources without importing dependencies.""" + +from __future__ import annotations + +import argparse +import ast +import hashlib +import json +from pathlib import Path +import shutil +import subprocess +import sys + + +ENTRY = "pyiceberg_smoke.py" +LOCAL_MODULES = ("engine_compatibility", "failure_coverage") + + +def validate(smoke_script: Path) -> list[Path]: + files = [smoke_script] + if not smoke_script.is_file(): + raise ValueError(f"Table smoke entry is missing: {smoke_script}") + tree = ast.parse(smoke_script.read_text(), filename=str(smoke_script)) + imports = { + alias.name.split(".")[0] + for node in ast.walk(tree) if isinstance(node, ast.Import) + for alias in node.names + } | { + node.module.split(".")[0] + for node in ast.walk(tree) + if isinstance(node, ast.ImportFrom) and node.module + } + # Older product refs may not use both modules. Only require those imported + # by this entry; never import the smoke just to validate local preparation. + for name in LOCAL_MODULES: + if name not in imports: + continue + module = smoke_script.with_name(f"{name}.py") + if not module.is_file(): + raise ValueError(f"Table smoke local dependency is missing: {module.name}") + ast.parse(module.read_text(), filename=str(module)) + files.append(module) + return files + + +def prepare(source_root: Path, destination: Path, expected_sha: str = "") -> Path: + source_root = source_root.resolve() + source = source_root / "scripts/table-catalog" + validate(source / ENTRY) + result = subprocess.run( + ["git", "-C", str(source_root), "rev-parse", "HEAD"], + check=True, capture_output=True, text=True, + ) + sha = result.stdout.strip() + if expected_sha and sha != expected_sha: + raise ValueError(f"Table source SHA mismatch: expected {expected_sha}, got {sha}") + dirty = subprocess.run( + ["git", "-C", str(source_root), "status", "--porcelain", "--", "scripts/table-catalog"], + check=True, capture_output=True, text=True, + ) + if dirty.stdout.strip(): + raise ValueError("Table smoke source tree has uncommitted changes; use a clean candidate checkout") + # Never merge a new source version with leftovers from an older run. + shutil.copytree(source, destination, ignore=shutil.ignore_patterns("._*", "__pycache__", "*.pyc")) + entry = destination / ENTRY + validate(entry) + manifest = { + "source_sha": sha, + "files": { + str(path.relative_to(destination)): hashlib.sha256(path.read_bytes()).hexdigest() + for path in sorted(destination.rglob("*")) if path.is_file() + }, + } + (destination / "source-manifest.json").write_text(json.dumps(manifest, indent=2) + "\n") + return entry + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="command", required=True) + check = sub.add_parser("check") + check.add_argument("--smoke-script", type=Path, required=True) + stage = sub.add_parser("prepare") + stage.add_argument("--source-root", type=Path, required=True) + stage.add_argument("--destination", type=Path, required=True) + stage.add_argument("--expected-sha", default="") + args = parser.parse_args() + try: + if args.command == "check": + validate(args.smoke_script) + else: + print(prepare(args.source_root, args.destination, args.expected_sha)) + except (OSError, ValueError, SyntaxError, subprocess.CalledProcessError) as exc: + print(f"[HARNESS-ERROR] Table smoke preparation: {exc}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/storage_capacity.py b/scripts/storage_capacity.py new file mode 100644 index 0000000..fa09e70 --- /dev/null +++ b/scripts/storage_capacity.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +"""Bound a capacity-filling workload using the least available online disk.""" + +from __future__ import annotations + +import argparse +import json +import math +import re +import sys +import time + + +def object_bytes(value: str) -> int: + match = re.fullmatch(r"([1-9][0-9]*)(B|KiB|MiB|GiB)?", value) + if not match: + raise ValueError(f"unsupported object size: {value}") + return int(match[1]) * {None: 1, "B": 1, "KiB": 1024, "MiB": 1024**2, "GiB": 1024**3}[match[2]] + + +def sample(document: dict, *, now: float | None = None) -> dict: + disks = document.get("info", document).get("disks") + if not isinstance(disks, list) or not disks: + raise ValueError("storageinfo must contain a non-empty disk inventory") + online = [] + for disk in disks: + if not isinstance(disk, dict): + raise ValueError("storageinfo disk must be an object") + if str(disk.get("state", "")).lower() == "offline": + continue + total, used = disk.get("totalspace"), disk.get("usedspace") + if any(type(value) not in (int, float) or not math.isfinite(value) for value in (total, used)): + raise ValueError("online disk capacity must be numeric") + if total <= 0 or used < 0 or used > total: + raise ValueError("online disk capacity is outside its valid range") + online.append((total, used)) + if not online: + raise ValueError("storageinfo has no usable online disks") + total, used = sum(d[0] for d in online), sum(d[1] for d in online) + return { + "sample_time": time.monotonic() if now is None else now, + "disk_count": len(online), "total_bytes": total, "used_bytes": used, + "usage_percent": used / total * 100, + "max_disk_percent": max(d[1] / d[0] * 100 for d in online), + "min_free_bytes": min(d[0] - d[1] for d in online), + } + + +def assess(current: dict, previous: dict | None = None, *, threshold: float = 85, + max_disk_percent: float = 95, min_free_bytes: int = 512 * 1024**2, + in_flight_bytes: int, poll_seconds: float = 2, stop_seconds: float = 5, + sample_timeout: float = 3) -> dict: + if not 0 < threshold < max_disk_percent <= 100: + raise ValueError("require 0 < storage threshold < max disk percent <= 100") + if min_free_bytes <= 0 or in_flight_bytes <= 0 or poll_seconds <= 0 or stop_seconds <= 0 or sample_timeout <= 0: + raise ValueError("capacity reserve and time bounds must be positive") + rate = 0.0 + if previous: + if (current["total_bytes"], current["disk_count"]) != (previous["total_bytes"], previous["disk_count"]): + raise ValueError("disk inventory changed during capacity filling") + elapsed = current["sample_time"] - previous["sample_time"] + if elapsed > 0: + # Cluster growth is a conservative bound for any one disk's growth; + # do not assume the workload or EC shards are evenly distributed. + rate = max(0, current["used_bytes"] - previous["used_bytes"]) / elapsed + reserve = math.ceil(min_free_bytes + in_flight_bytes + rate * (poll_seconds + stop_seconds + sample_timeout)) + guarded = current["max_disk_percent"] >= max_disk_percent or current["min_free_bytes"] <= reserve + action = "capacity_guard" if guarded else "target_reached" if current["usage_percent"] >= threshold else "continue" + return dict(current, action=action, reserve_bytes=reserve, growth_bytes_per_second=rate, + emergency=current["max_disk_percent"] >= max_disk_percent or current["min_free_bytes"] <= min_free_bytes) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("command", choices=("sample", "assess")) + parser.add_argument("--previous", default="") + parser.add_argument("--threshold", type=float, default=85) + parser.add_argument("--max-disk-percent", type=float, default=95) + parser.add_argument("--min-free-bytes", type=int, default=512 * 1024**2) + parser.add_argument("--object-size", default="100MiB") + parser.add_argument("--concurrent", type=int, default=32) + parser.add_argument("--poll-seconds", type=float, default=2) + parser.add_argument("--stop-seconds", type=float, default=5) + parser.add_argument("--sample-timeout", type=float, default=3) + args = parser.parse_args() + try: + current = json.load(sys.stdin) + if args.command == "sample": + result = sample(current) + else: + if args.concurrent <= 0: + raise ValueError("warp concurrency must be positive") + result = assess(current, json.loads(args.previous) if args.previous else None, + threshold=args.threshold, max_disk_percent=args.max_disk_percent, + min_free_bytes=args.min_free_bytes, + in_flight_bytes=object_bytes(args.object_size) * args.concurrent, + poll_seconds=args.poll_seconds, stop_seconds=args.stop_seconds, + sample_timeout=args.sample_timeout) + print(json.dumps(result, separators=(",", ":"))) + except (TypeError, AttributeError, KeyError, ValueError) as exc: + print(f"[HARNESS-ERROR] capacity monitor: {exc}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/validation_pool/pool_behavior.py b/scripts/validation_pool/pool_behavior.py index d47e348..4d7069e 100644 --- a/scripts/validation_pool/pool_behavior.py +++ b/scripts/validation_pool/pool_behavior.py @@ -26,7 +26,7 @@ from typing import Any, Callable from case_plan import POOL_CASE_IDS as CASE_IDS, pool_harness_scored_cases -from pool_helpers import active_endpoint_indices, active_fixture_bytes +from pool_helpers import active_endpoint_indices, active_fixture_bytes, active_rebalance_fixture_bytes from pool_status import ( PoolStatusError, assert_pool_parity, @@ -49,6 +49,7 @@ validate_pool_inventory, ) from rc_binary import resolve_rc_binary +from scripts.storage_capacity import sample as capacity_sample try: import boto3 @@ -1179,6 +1180,14 @@ def probe() -> tuple[bool, Any]: document = json.loads(body) try: matched = predicates[state](document, operation_id) + if state == "active" and not allow_fast_completion and not matched: + pools = rebalance_entries(document, operation_id) + if all(str(pool["status"]).lower() in {"completed", "none"} for pool in pools): + self.save_json(case_id, f"rebalance-{artifact_label}-window-ended.json", document) + raise AssertionError( + "rebalance completed before the required active window; " + "increase the active fixture or reduce probe overhead" + ) except PoolStatusError: self.save_json( case_id, @@ -1206,12 +1215,17 @@ def probe() -> tuple[bool, Any]: operationId=operation_id, evidence="terminal status carried nonzero movement progress", ) - self.rebalance_status( + verified = self.rebalance_status( case_id, operation_id, f"rebalance-{artifact_label}-parity", allow_stopped=state == "stopped", ) + if state == "active" and not allow_fast_completion and not rebalance_active( + verified, operation_id, require_progress=True + ): + self.save_json(case_id, f"rebalance-{artifact_label}-window-ended.json", verified) + raise AssertionError("rebalance active window ended during rc/Admin parity checks") if state in {"complete", "stopped"}: inventory, _ = self.pool_snapshot( case_id, @@ -1518,9 +1532,31 @@ def case_expand_and_rebalance(self, case_id: str) -> None: self.census(case_id, "after-rebalance") self.assert_inventory(case_id, self.client, bucket, inventory, "after-rebalance") + def rebalance_fixture_size(self, case_id: str) -> int: + size = active_rebalance_fixture_bytes( + self.args.fixture_bytes, self.args.active_rebalance_fixture_bytes + ) + # The strict active oracle needs published movement before stop/live I/O. + # Reserve room for an EC ratio up to 2 and metadata; refuse an oversized + # fixture before writing it instead of filling the environment's disks. + document = self.admin_json(case_id, "rebalance-fixture-capacity", "storageinfo") + current = capacity_sample(document) + logical_bytes = (self.args.fixture_objects - 3) * size + 4098 + required_free = logical_bytes * 2 / current["disk_count"] + 1024**3 + self.save_json(case_id, "rebalance-fixture-budget.json", { + "fixtureBytes": size, "logicalBytes": logical_bytes, + "requiredFreePerDisk": required_free, "capacity": current, + }) + if current["min_free_bytes"] < required_free: + raise RuntimeError( + "[ENVIRONMENT-CAPACITY] insufficient per-disk space for the active " + "rebalance fixture; reduce fixture count/size or prepare more capacity" + ) + return size + def case_expansion_restart_persistence(self, case_id: str) -> None: bucket, inventory = self.seed_and_expand( - case_id, size=active_fixture_bytes(self.args.fixture_bytes) + case_id, size=self.rebalance_fixture_size(case_id) ) before, _ = self.pool_snapshot(case_id, "before-restart", 2) operation_id = self.start_rebalance(case_id) @@ -1577,7 +1613,7 @@ def rebalance_counters(document: Any) -> tuple[int, int, int]: def case_rebalance_stop_resume(self, case_id: str) -> None: bucket, inventory = self.seed_and_expand( - case_id, size=active_fixture_bytes(self.args.fixture_bytes) + case_id, size=self.rebalance_fixture_size(case_id) ) first_id = self.start_rebalance(case_id) self.wait_rebalance(case_id, first_id, "active") @@ -1700,7 +1736,7 @@ def read_live_object() -> bytes: def case_rebalance_live_io(self, case_id: str) -> None: bucket, inventory = self.seed_and_expand( - case_id, size=active_fixture_bytes(self.args.fixture_bytes) + case_id, size=self.rebalance_fixture_size(case_id) ) operation_id = self.start_rebalance(case_id) self.wait_rebalance(case_id, operation_id, "active") @@ -1708,6 +1744,9 @@ def case_rebalance_live_io(self, case_id: str) -> None: status, _, body = self.signed_request( self.args.endpoint, "GET", f"{ADMIN_PREFIX}/rebalance/status" ) + self.save_json(case_id, "rebalance-live-after-io.json", { + "httpStatus": status, "body": body.decode(errors="replace") + }) if status != 200 or not rebalance_active(json.loads(body), operation_id, require_progress=True): raise AssertionError("rebalance did not remain active across the live-I/O window") self.wait_rebalance(case_id, operation_id, "complete") @@ -2276,11 +2315,12 @@ def case_retired_config_removal(self, case_id: str) -> None: self.census(case_id, "after-retired-config-removal") def case_operation_guards(self, case_id: str) -> None: + fixture_size = self.rebalance_fixture_size(case_id) bucket = self.create_bucket(case_id) inventory = self.put_inventory( case_id, bucket, - size=active_fixture_bytes(self.args.fixture_bytes), + size=fixture_size, ) self.run_rc( case_id, @@ -2620,6 +2660,9 @@ def write_results(self, started_at: str, started: float) -> str: "fixtureObjects": self.args.fixture_objects, "fixtureBytes": self.args.fixture_bytes, "activeFixtureBytes": active_fixture_bytes(self.args.fixture_bytes), + "activeRebalanceFixtureBytes": active_rebalance_fixture_bytes( + self.args.fixture_bytes, self.args.active_rebalance_fixture_bytes + ), "liveObjects": self.args.live_objects, "boto3Version": boto3.__version__, "botocoreVersion": botocore.__version__, @@ -2754,6 +2797,8 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--poll-interval", type=float, default=0.5) parser.add_argument("--fixture-objects", type=int, default=128) parser.add_argument("--fixture-bytes", type=int, default=1024 * 1024) + parser.add_argument("--active-rebalance-fixture-bytes", type=int, + default=int(os.environ.get("RUSTFS_ACTIVE_REBALANCE_FIXTURE_BYTES", 128 * 1024**2))) parser.add_argument("--live-objects", type=int, default=16) parser.add_argument("--workers", type=int, default=8) parser.add_argument("--rc-bin", default="rc") @@ -2787,6 +2832,7 @@ def parse_args() -> argparse.Namespace: or args.poll_interval <= 0 or args.fixture_objects < 4 or args.fixture_bytes < 1 + or args.active_rebalance_fixture_bytes < 32 * 1024**2 or args.live_objects < 1 or args.workers < 1 ): diff --git a/scripts/validation_pool/pool_helpers.py b/scripts/validation_pool/pool_helpers.py index cd96be1..a1fef4e 100644 --- a/scripts/validation_pool/pool_helpers.py +++ b/scripts/validation_pool/pool_helpers.py @@ -1,6 +1,7 @@ """Pool topology and fixture rules used by the source behavior checks.""" MIN_ACTIVE_FIXTURE_BYTES = 32 * 1024 * 1024 +MIN_ACTIVE_REBALANCE_FIXTURE_BYTES = 128 * 1024 * 1024 def active_endpoint_indices(endpoint_count: int, layout: str) -> tuple[int, ...]: @@ -23,3 +24,9 @@ def active_endpoint_indices(endpoint_count: int, layout: str) -> tuple[int, ...] def active_fixture_bytes(configured_bytes: int) -> int: return max(configured_bytes, MIN_ACTIVE_FIXTURE_BYTES) + + +def active_rebalance_fixture_bytes(configured_bytes: int, minimum_bytes: int = MIN_ACTIVE_REBALANCE_FIXTURE_BYTES) -> int: + if minimum_bytes < MIN_ACTIVE_FIXTURE_BYTES: + raise ValueError("active rebalance fixtures must be at least 32 MiB per object") + return max(configured_bytes, minimum_bytes) diff --git a/test_check_shell_syntax.py b/test_check_shell_syntax.py new file mode 100644 index 0000000..c3affe1 --- /dev/null +++ b/test_check_shell_syntax.py @@ -0,0 +1,35 @@ +import json +from pathlib import Path +import subprocess +import tempfile +import unittest + +from scripts.check_shell_syntax import check + + +class ShellSyntaxTests(unittest.TestCase): + def test_sidecars_are_excluded_and_each_real_result_is_retained(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "._good.sh").write_bytes(b"\x00AppleDouble") + (root / "a-bad.sh").write_text("if then\n") + (root / "z-good.sh").write_text("echo good\n") + report = root / "report.json" + run = subprocess.run(["python3", str(Path(__file__).parent / "scripts/check_shell_syntax.py"), + "--root", directory, "--report", str(report)], capture_output=True, text=True) + self.assertNotEqual(run.returncode, 0) + results = json.loads(report.read_text()) + self.assertFalse(results["passed"]) + self.assertTrue(results["complete"]) + self.assertEqual([r["file"] for r in results["results"]], ["a-bad.sh", "z-good.sh"]) + self.assertNotEqual(results["results"][0]["exit_code"], 0) + self.assertEqual(results["results"][1]["exit_code"], 0) + self.assertIn("syntax error", results["results"][0]["stderr"]) + + def test_no_actual_scripts_cannot_certify_a_syntax_pass(self): + with tempfile.TemporaryDirectory() as directory: + self.assertFalse(check(Path(directory))["passed"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/test_rustfs_fault_tolerance.py b/test_rustfs_fault_tolerance.py index 885271d..f823793 100644 --- a/test_rustfs_fault_tolerance.py +++ b/test_rustfs_fault_tolerance.py @@ -535,7 +535,7 @@ def test_matrix_topologies_are_single_pool_expressions(self) -> None: def test_matrix_applies_parity_and_product_write_quorum(self) -> None: result = self.run_bash( """ - install_and_start() { echo "env=$2"; } + install_and_start() { echo "env=$2 participants=$3"; } seed_baseline() { :; } stop_node() { :; } start_nodes() { :; } @@ -561,6 +561,37 @@ def test_matrix_applies_parity_and_product_write_quorum(self) -> None: self.assertIn("M-4x2EC4-n2/read=ok", out) self.assertIn("M-4x2EC4-n2/write=fail", out) self.assertIn("M-4x2EC3-n2/read=fail", out) + self.assertIn("participants=4", out) + + def test_install_starts_only_the_topology_participants(self) -> None: + for count in (1, 2, 3, 4): + with self.subTest(count=count): + result = self.run_bash(f''' + calls="${{TMP}}/starts" + PACKAGE_URL=https://example.invalid/rustfs.deb + remote_all() {{ :; }} + run_remote() {{ + case "$2" in *systemctl*) printf '%s|%s\\n' "$1" "$2" >> "${{calls}}";; esac + }} + ssh() {{ cat >/dev/null; }} + curl() {{ :; }} + install_and_start "$(matrix_volumes_expr {count} 2)" 'RUSTFS_STORAGE_CLASS_STANDARD="EC:1"' {count} + cat "${{calls}}" + ''') + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + starts = [line.split("|")[0] for line in result.stdout.splitlines() if "|sudo -n systemctl start --no-block" in line] + self.assertEqual(sorted(starts), [f"vm00{i}" for i in range(count)]) + self.assertEqual(result.stdout.count("systemctl is-active --quiet"), count) + + def test_invalid_participant_count_is_rejected_before_mutation(self) -> None: + result = self.run_bash(''' + PACKAGE_URL=https://example.invalid/rustfs.deb + remote_all() { echo UNEXPECTED_MUTATION; } + install_and_start "$VOLUMES_4X4" "" 5 + ''') + self.assertNotEqual(result.returncode, 0) + self.assertIn("invalid topology participant", result.stderr) + self.assertNotIn("UNEXPECTED_MUTATION", result.stdout) def test_fault_injection_reaches_mount_lease_and_recovery_is_nonblocking(self) -> None: result = self.run_bash( diff --git a/test_rustfs_performance_testing.py b/test_rustfs_performance_testing.py index b1cda26..cd4c6d4 100644 --- a/test_rustfs_performance_testing.py +++ b/test_rustfs_performance_testing.py @@ -53,6 +53,10 @@ def setUp(self): printf '%s\n' "${text//"$omit"/}" done [[ "$mode" != failure-with-metrics ]] || return 42 + if [[ "$mode" == errors-with-success ]]; then + printf '%s\n' 'warp: upload error: The service is unavailable. Please retry.' + fi + if [[ "$mode" == hang-after-report ]]; then sleep 30; fi } ''') self.calls = self.root / "calls" @@ -124,6 +128,33 @@ def test_stuck_round_is_hard_timed_out_and_fails_the_run(self): self.assertIn("rc=124", master) self.assertIn("ABORT consecutive_failures=1 limit=1", master) + def round_evidence(self, filename="rounds.tsv"): + with (self.results / filename).open() as stream: + return list(csv.DictReader(stream, delimiter="\t")) + + def test_zero_exit_with_service_errors_retains_metrics_but_is_not_a_pass(self): + result, rows = self.run_benchmark("errors-with-success", WARP_METHODS="get", WARP_SIZES="1KiB") + self.assertNotEqual(result.returncode, 0) + self.assertEqual(rows, []) + evidence = self.round_evidence()[0] + self.assertEqual(evidence["warp_exit"], "0") + self.assertEqual(evidence["verdict"], "FAIL") + self.assertEqual(evidence["reason"], "logged_errors") + self.assertEqual(evidence["error_lines"], "1") + self.assertIn("Average:", (self.results / "get_1KiB.txt").read_text()) + + def test_timeout_before_and_after_report_preserve_distinct_evidence(self): + for mode, reason in (("hang", "timeout_before_report"), ("hang-after-report", "timeout_after_report")): + with self.subTest(mode=mode): + result, rows = self.run_benchmark(mode, WARP_METHODS="get", WARP_SIZES="1KiB,4KiB", + WARP_ROUND_TIMEOUT="1", WARP_MAX_CONSECUTIVE_FAILURES="1") + self.assertNotEqual(result.returncode, 0) + self.assertEqual(rows, []) + evidence = self.round_evidence() + self.assertEqual(evidence[0]["reason"], reason) + self.assertEqual(evidence[0]["warp_exit"], "124") + self.assertEqual(evidence[1]["verdict"], "NOT_RUN") + def test_each_required_single_method_metric_is_checked(self): for value in ("1.25 MiB/s", "1280.0", "1.2ms", "1.1ms", "1.5ms", "2.1ms"): with self.subTest(missing=value): @@ -155,13 +186,76 @@ def test_one_round_and_default_full_matrix_are_valid_selections(self): def test_parse_only_rejects_missing_or_incomplete_selected_results(self): result, _ = self.run_benchmark() self.assertEqual(result.returncode, 0, result.stderr) + original = {path.name: path.read_bytes() for path in self.results.iterdir()} + calls = self.calls.read_bytes() command = ["bash", str(self.script), "--parse-only", str(self.results)] parsed = subprocess.run(command, cwd=self.root, env=self.env, capture_output=True, text=True) self.assertEqual(parsed.returncode, 0, parsed.stderr) + self.assertEqual(original, {name: (self.results / name).read_bytes() for name in original}) + self.assertEqual(self.calls.read_bytes(), calls) + self.assertEqual(len(self.round_evidence("rounds.reparsed.tsv")), 6) + self.assertIn("parsed=6", (self.results / "summary.reparsed.md").read_text()) (self.results / "get_1KiB.txt").unlink() parsed = subprocess.run(command, cwd=self.root, env=self.env, capture_output=True, text=True) self.assertNotEqual(parsed.returncode, 0) - self.assertIn("failed=1", (self.results / "summary.md").read_text()) + self.assertIn("failed=1", (self.results / "summary.reparsed.md").read_text()) + + def test_parse_only_preserves_aborted_execution_and_writes_separate_evidence(self): + result, _ = self.run_benchmark("fail", WARP_MAX_CONSECUTIVE_FAILURES="1") + self.assertNotEqual(result.returncode, 0) + execution = self.round_evidence() + self.assertEqual(execution[0]["warp_exit"], "42") + self.assertTrue(execution[0]["elapsed_seconds"].isdigit()) + self.assertEqual(execution[1]["verdict"], "NOT_RUN") + self.assertEqual(execution[1]["reason"], "aborted") + original = {path.name: path.read_bytes() for path in self.results.iterdir()} + calls = self.calls.read_bytes() + + parsed = subprocess.run(["bash", str(self.script), "--parse-only", str(self.results)], + cwd=self.root, env=self.env, capture_output=True, text=True) + self.assertNotEqual(parsed.returncode, 0) + self.assertEqual(original, {name: (self.results / name).read_bytes() for name in original}) + self.assertEqual(self.calls.read_bytes(), calls) + reparsed = self.round_evidence("rounds.reparsed.tsv") + self.assertEqual(len(reparsed), 6) + self.assertEqual(reparsed[0]["warp_exit"], "42") + self.assertEqual(reparsed[0]["elapsed_seconds"], "unknown") + self.assertEqual(reparsed[1]["reason"], "missing_output") + self.assertIn("failed=6", (self.results / "summary.reparsed.md").read_text()) + self.assertIn("PARSE_FAILED", (self.results / "master.reparsed.log").read_text()) + with (self.results / "summary.reparsed.tsv").open() as stream: + self.assertEqual(list(csv.DictReader(stream, delimiter="\t")), []) + + def test_analysis_without_original_summary_uses_reparsed_report_and_propagates_failure(self): + for mode, expected in (("success", 0), ("failure-with-metrics", 1)): + with self.subTest(mode=mode): + self.run_benchmark(mode) + (self.results / "summary.md").unlink() + original = {path.name: path.read_bytes() for path in self.results.iterdir() + if ".reparsed." not in path.name} + calls = self.calls.read_bytes() + environment = {**self.env, "RUSTFS_RESULT_DIR": str(self.results), + "RUSTFS_BENCH_SCRIPT": str(self.script), + "RUSTFS_PERF_SCRIPT_SOURCE_ONLY": "1", + "RUSTFS_WARP_METHODS": self.env["WARP_METHODS"], + "RUSTFS_WARP_SIZES": self.env["WARP_SIZES"]} + environment.pop("WARP_METHODS") + environment.pop("WARP_SIZES") + parsed = subprocess.run( + ["bash", "-c", 'source "$1"; step6_analyze', "bash", + str(ROOT / "rustfs_performance_test.sh")], + cwd=self.root, env=environment, + capture_output=True, text=True, timeout=10, + ) + self.assertEqual(parsed.returncode, expected, parsed.stdout + parsed.stderr) + self.assertFalse((self.results / "summary.md").exists()) + self.assertEqual(original, {name: (self.results / name).read_bytes() for name in original}) + self.assertEqual(self.calls.read_bytes(), calls) + if expected == 0: + self.assertIn("step 6 complete", parsed.stdout) + else: + self.assertNotIn("step 6 complete", parsed.stdout) + self.assertTrue((self.results / "summary.reparsed.md").is_file()) def test_parse_only_cannot_certify_failed_or_unverified_metrics(self): result, rows = self.run_benchmark("failure-with-metrics") @@ -170,7 +264,7 @@ def test_parse_only_cannot_certify_failed_or_unverified_metrics(self): command = ["bash", str(self.script), "--parse-only", str(self.results)] parsed = subprocess.run(command, cwd=self.root, env=self.env, capture_output=True, text=True) self.assertNotEqual(parsed.returncode, 0) - self.assertIn("failed=6", (self.results / "summary.md").read_text()) + self.assertIn("failed=6", (self.results / "summary.reparsed.md").read_text()) result, _ = self.run_benchmark() self.assertEqual(result.returncode, 0, result.stderr) output = self.results / "get_1KiB.txt" @@ -180,7 +274,7 @@ def test_parse_only_cannot_certify_failed_or_unverified_metrics(self): output.write_text(original.replace("RustFS round status: warp=0 capture=0\n", terminal)) parsed = subprocess.run(command, cwd=self.root, env=self.env, capture_output=True, text=True) self.assertNotEqual(parsed.returncode, 0) - self.assertIn("failed=1", (self.results / "summary.md").read_text()) + self.assertIn("failed=1", (self.results / "summary.reparsed.md").read_text()) class PerformanceEnvironmentTests(unittest.TestCase): diff --git a/test_rustfs_pool_expand.sh b/test_rustfs_pool_expand.sh index 140432c..d20c873 100755 --- a/test_rustfs_pool_expand.sh +++ b/test_rustfs_pool_expand.sh @@ -70,12 +70,13 @@ MOCK_STATUS_FILE="${TEST_TMP}/curl-statuses" MOCK_CALL_FILE="${TEST_TMP}/curl-calls" curl() { - local output_file="" write_format="" method="GET" url="" host_header="" arg status body + local output_file="" header_file="" write_format="" method="GET" url="" host_header="" arg status body while [ "$#" -gt 0 ]; do arg="$1" shift case "${arg}" in -o) output_file="$1"; shift ;; + -D) header_file="$1"; shift ;; -w) write_format="$1"; shift ;; -X) method="$1"; shift ;; -H) @@ -101,6 +102,10 @@ curl() { if [ -n "${output_file}" ]; then printf '%s' "${body}" > "${output_file}" fi + if [ -n "${header_file}" ]; then + printf '%s\r\n' "HTTP/1.1 ${status}" 'x-rustfs-error-cause: namespace_lock_timeout' \ + 'x-amz-request-id: request-test' 'retry-after: 1' 'Set-Cookie: secret-cookie' > "${header_file}" + fi if [ -n "${write_format}" ]; then if [[ "${write_format}" == *time_connect* ]]; then printf '%s\t0.010\t0.020\t0.030' "${status}" @@ -169,7 +174,7 @@ test_storageinfo_retry_is_sticky() { test_warp_must_reach_threshold() { if ( - storage_usage_percent() { printf '10\n'; } + storage_capacity_sample() { printf '{"sample_time":1,"disk_count":1,"total_bytes":107374182400,"used_bytes":10737418240,"usage_percent":10,"max_disk_percent":10,"min_free_bytes":96636764160}\n'; } monitor_storage 50 999999 ) >/dev/null 2>&1; then fail "warp completion below the storage threshold must fail" @@ -179,7 +184,7 @@ test_warp_must_reach_threshold() { test_monitor_propagates_storage_sample_failure() { local status if ( - storage_usage_percent() { return 1; } + storage_capacity_sample() { return 1; } monitor_storage 50 999999 ) >/dev/null 2>&1; then fail "storage sample failure must fail the monitor" @@ -189,6 +194,38 @@ test_monitor_propagates_storage_sample_failure() { assert_eq 1 "${status}" "storage sample failure status" } +test_monitor_rejects_an_unsafe_disk_even_when_average_is_low() ( + local status=0 + storage_capacity_sample() { printf '{"sample_time":1,"disk_count":4,"total_bytes":40000000000,"used_bytes":12000000000,"usage_percent":30,"max_disk_percent":99.9,"min_free_bytes":200704}\n'; } + monitor_storage 85 999999 >/dev/null 2>&1 || status=$? + assert_eq 4 "${status}" "capacity guard must not certify the storage target" + assert_eq 1 "${CAPACITY_EMERGENCY}" "near-full disk needs urgent writer stop" +) + +test_active_capacity_sampling_does_not_retry_a_failed_api() ( + reset_endpoint_state + resolve_endpoints + set_mock_statuses 503 200 + if storage_capacity_sample >/dev/null 2>&1; then + fail "failed capacity probe must stop the writer rather than continue retries" + fi + assert_eq 1 "$(wc -l < "${MOCK_CALL_FILE}" | tr -d ' ')" "bounded active capacity sampling" +) + +test_mpu_failure_keeps_cause_and_request_id_without_sensitive_headers() ( + reset_endpoint_state + resolve_endpoints + set_mock_statuses 503 + if (abort_stale_multipart_uploads fixture-bucket) >"${TEST_TMP}/mpu-error.log" 2>&1; then + fail "MPU 503 must fail cleanup" + fi + grep -q 'namespace_lock_timeout' "${TEST_TMP}/mpu-error.log" || fail "MPU error cause missing" + grep -q 'request-test' "${TEST_TMP}/mpu-error.log" || fail "MPU request ID missing" + if grep -q 'secret-cookie' "${TEST_TMP}/mpu-error.log"; then + fail "response cookies must not be published" + fi +) + test_stage_readiness_and_exact_pool_count() { reset_endpoint_state ADMIN_ENDPOINT="http://admin.test:9000" @@ -356,6 +393,7 @@ test_warp_rounds_reach_threshold() ( warp() { :; } abort_stale_multipart_uploads() { :; } report_warp_errors() { :; } + check_capacity_before_write() { :; } monitor_storage() { monitor_calls=$((monitor_calls + 1)) if [ "${monitor_calls}" -eq 2 ]; then return 0; fi @@ -366,6 +404,30 @@ test_warp_rounds_reach_threshold() ( assert_eq 2 "${monitor_calls}" "warp rounds before threshold" ) +test_capacity_guard_stops_writer_and_cleans_mpu_but_fails_step() ( + reset_endpoint_state + WITH_WARP=1 + DRY_RUN=0 + WARP_LOG_FILE="${TEST_TMP}/guard-warp.log" + WARP_STOP_GRACE=1 + verify_stage_readiness() { :; } + configure_proxy_stage() { :; } + verify_pools() { :; } + confirm() { :; } + need_cmd() { :; } + check_capacity_before_write() { :; } + warp() { exec sleep 30; } + abort_stale_multipart_uploads() { touch "${TEST_TMP}/guard-mpu-cleaned"; } + report_warp_errors() { :; } + monitor_storage() { CAPACITY_EMERGENCY=1; return 4; } + if (step4_write_data) >"${TEST_TMP}/guard-step.log" 2>&1; then + fail "capacity guard must fail the fill step" + fi + [ -f "${TEST_TMP}/guard-mpu-cleaned" ] || fail "capacity stop must retain MPU cleanup" + grep -q 'ENVIRONMENT-CAPACITY.*target was not certified' "${TEST_TMP}/guard-step.log" \ + || fail "capacity stop must be reported separately from target reached" +) + test_package_checksum_before_install() ( local script_name="$1" download_step="$2" install_step="$3" local fixture="${TEST_TMP}/${script_name}" expected_hash scenario status expected @@ -480,8 +542,12 @@ test_independent_routing_and_signing test_storageinfo_retry_is_sticky test_warp_must_reach_threshold test_monitor_propagates_storage_sample_failure +test_monitor_rejects_an_unsafe_disk_even_when_average_is_low +test_active_capacity_sampling_does_not_retry_a_failed_api +test_mpu_failure_keeps_cause_and_request_id_without_sensitive_headers test_stop_skips_an_uninstalled_service test_warp_rounds_reach_threshold +test_capacity_guard_stops_writer_and_cleans_mpu_but_fails_step test_stage_readiness_and_exact_pool_count test_stage_pool_boundaries 3 expand test_stage_pool_boundaries 4 expand diff --git a/test_rustfs_table.py b/test_rustfs_table.py index 7509e69..5ffd18b 100644 --- a/test_rustfs_table.py +++ b/test_rustfs_table.py @@ -2,9 +2,14 @@ """Check that table-suite privileged commands are decided on the remote host.""" from pathlib import Path +import json +import os import subprocess +import tempfile import unittest +from scripts.prepare_table_smoke import prepare, validate + ROOT = Path(__file__).resolve().parent @@ -47,5 +52,56 @@ def test_remote_root_checks_uid_after_ssh(self): self.assertIn("sudo -n bash -s", result.stdout) +class TableSmokePreparationTests(unittest.TestCase): + def test_complete_product_tree_is_staged_and_bound_to_its_sha(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + source = root / "source" + tree = source / "scripts/table-catalog" + tree.mkdir(parents=True) + (tree / "pyiceberg_smoke.py").write_text("import engine_compatibility\nimport failure_coverage\n") + (tree / "engine_compatibility.py").write_text("VALUE = 1\n") + (tree / "failure_coverage.py").write_text("VALUE = 2\n") + (tree / "README.md").write_text("Supporting source documentation\n") + for command in (["init", "-q"], ["add", "."], + ["-c", "user.name=Test", "-c", "user.email=test@example.invalid", "commit", "-qm", "fixture"]): + subprocess.run(["git", "-C", str(source), *command], check=True, capture_output=True) + sha = subprocess.check_output(["git", "-C", str(source), "rev-parse", "HEAD"], text=True).strip() + destination = root / "staged" + entry = prepare(source, destination, sha) + self.assertEqual(len(validate(entry)), 3) + self.assertTrue((destination / "README.md").is_file()) + manifest = json.loads((destination / "source-manifest.json").read_text()) + self.assertEqual(manifest["source_sha"], sha) + self.assertEqual(len(manifest["files"]), 4) + with self.assertRaises(ValueError): + prepare(source, root / "wrong", "f" * 40) + self.assertFalse((root / "wrong").exists()) + (tree / "engine_compatibility.py").write_text("VALUE = 99\n") + with self.assertRaisesRegex(ValueError, "uncommitted"): + prepare(source, root / "dirty", sha) + self.assertFalse((root / "dirty").exists()) + + def test_missing_module_fails_before_venv_or_s3_actions(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + entry = root / "pyiceberg_smoke.py" + entry.write_text("import engine_compatibility\n") + command = f'''source "{ROOT / 'rustfs-table-test.sh'}" +SMOKE_SCRIPT="$SMOKE_FIXTURE" +python3() {{ + if [ "${{1:-}}" = -m ]; then echo "unexpected-venv"; return 99; fi + command python3 "$@" +}} +test_tbl101 +''' + result = subprocess.run(["bash", "-c", command], capture_output=True, text=True, + env={**os.environ, "SMOKE_FIXTURE": str(entry)}, timeout=10) + self.assertNotEqual(result.returncode, 0) + self.assertIn("[HARNESS-ERROR]", result.stderr) + self.assertIn("engine_compatibility.py", result.stderr) + self.assertNotIn("unexpected-venv", result.stdout) + + if __name__ == "__main__": unittest.main() diff --git a/test_storage_capacity.py b/test_storage_capacity.py new file mode 100644 index 0000000..78f08c5 --- /dev/null +++ b/test_storage_capacity.py @@ -0,0 +1,57 @@ +import unittest + +from scripts.storage_capacity import assess, object_bytes, sample + + +class CapacitySafetyTests(unittest.TestCase): + def disks(self, *used, now=0): + return sample({"info": {"disks": [{"state": "ok", "totalspace": 10000, "usedspace": value} + for value in used]}}, now=now) + + def decision(self, current, previous=None, **kwargs): + return assess(current, previous, min_free_bytes=100, in_flight_bytes=200, **kwargs) + + def test_pool_average_cannot_hide_an_almost_full_disk(self): + result = self.decision(self.disks(9990, 1000, 1000, 1000)) + self.assertLess(result["usage_percent"], 40) + self.assertEqual(result["action"], "capacity_guard") + self.assertTrue(result["emergency"]) + + def test_in_flight_reserve_stops_before_the_emergency_limit(self): + result = assess( + self.disks(9400), min_free_bytes=100, in_flight_bytes=600, max_disk_percent=99) + self.assertEqual(result["action"], "capacity_guard") + self.assertFalse(result["emergency"]) + + def test_growth_reserves_the_next_sample_and_shutdown_window(self): + previous = self.disks(5000, 5000, now=10) + result = self.decision(self.disks(6000, 6000, now=12), previous) + self.assertEqual(result["growth_bytes_per_second"], 1000) + self.assertEqual(result["action"], "capacity_guard") + + def test_safe_target_reached_and_insufficient_load_are_distinct(self): + self.assertEqual(self.decision(self.disks(8500, 8600))["action"], "target_reached") + self.assertEqual(self.decision(self.disks(1000))["action"], "continue") + + def test_inventory_change_or_invalid_capacity_does_not_allow_more_writes(self): + with self.assertRaises(ValueError): + self.decision(self.disks(2000, 2000, now=2), self.disks(1000, now=1)) + for document in ({"disks": []}, {"disks": [{"state": "offline"}]}, + {"disks": [{"totalspace": 10, "usedspace": -1}]}, + {"disks": [{"totalspace": 10, "usedspace": 11}]}, + {"disks": [{"totalspace": 10, "usedspace": "1"}]}): + with self.subTest(document=document), self.assertRaises(ValueError): + sample(document) + + def test_invalid_bounds_and_object_sizes_are_rejected(self): + self.assertEqual(object_bytes("100MiB"), 100 * 1024**2) + for invalid in ("0MiB", "-1", "100MB"): + with self.assertRaises(ValueError): + object_bytes(invalid) + for kwargs in ({"threshold": 95}, {"poll_seconds": 0}, {"stop_seconds": -1}): + with self.assertRaises(ValueError): + self.decision(self.disks(1000), **kwargs) + + +if __name__ == "__main__": + unittest.main() diff --git a/test_validation_pool.py b/test_validation_pool.py index 67ff04d..80f6029 100644 --- a/test_validation_pool.py +++ b/test_validation_pool.py @@ -1,10 +1,18 @@ #!/usr/bin/env python3 import unittest +import json from pathlib import Path +import sys +from types import SimpleNamespace +from unittest.mock import Mock from scripts.validation_pool.case_plan import POOL_CASE_IDS, pool_harness_scored_cases -from scripts.validation_pool.pool_helpers import active_endpoint_indices, active_fixture_bytes +from scripts.validation_pool.pool_helpers import active_endpoint_indices, active_fixture_bytes, active_rebalance_fixture_bytes + +sys.path.insert(0, str(Path(__file__).parent / "scripts/validation_pool")) +from pool_behavior import PoolHarness +from pool_status import rebalance_complete class PoolContractTest(unittest.TestCase): @@ -27,6 +35,79 @@ def test_two_node_layout_and_active_fixture_size(self) -> None: self.assertEqual((0, 1), active_endpoint_indices(2, "expanded")) self.assertEqual((1,), active_endpoint_indices(2, "retired")) self.assertEqual(32 * 1024 * 1024, active_fixture_bytes(1024)) + self.assertEqual(128 * 1024 * 1024, active_rebalance_fixture_bytes(1024)) + + +class RebalanceWindowTests(unittest.TestCase): + def document(self, status="Started", objects=0): + return {"id": "operation-1", "pools": [{"id": 0, "status": status, "used": 0.5, + "cleanupWarnings": {"count": 0}, "progress": {"objects": objects}}]} + + def harness(self, document): + harness = PoolHarness.__new__(PoolHarness) + harness.args = SimpleNamespace(endpoint="http://test.invalid", active_timeout=1, timeout=1, fixture_bytes=1024, + fixture_objects=128, active_rebalance_fixture_bytes=128 * 1024**2) + harness.signed_request = Mock(return_value=(200, {}, json.dumps(document).encode())) + harness.save_json = Mock() + harness.rebalance_status = Mock(return_value=document) + harness.rebalance_fixture_size = Mock(return_value=128 * 1024**2) + def wait_once(case, description, probe, **kwargs): + matched, value = probe() + if not matched: + raise AssertionError("predicate did not match") + return value + harness.wait_until = wait_once + return harness + + def test_started_with_progress_enters_active_window(self): + document = self.document(objects=10) + harness = self.harness(document) + self.assertEqual(harness.wait_rebalance("case", "operation-1", "active"), document) + + def test_unpublished_progress_does_not_satisfy_the_source_oracle(self): + harness = self.harness(self.document()) + with self.assertRaisesRegex(AssertionError, "predicate did not match"): + harness.wait_rebalance("case", "operation-1", "active") + + def test_completed_does_not_replace_active_window(self): + harness = self.harness(self.document("Completed", 10)) + with self.assertRaisesRegex(AssertionError, "completed before"): + harness.wait_rebalance("case", "operation-1", "active") + self.assertTrue(harness.save_json.called) + + def test_completion_during_parity_checks_fails_the_window(self): + harness = self.harness(self.document(objects=10)) + harness.rebalance_status.return_value = self.document("Completed", 10) + with self.assertRaisesRegex(AssertionError, "parity checks"): + harness.wait_rebalance("case", "operation-1", "active") + + def test_final_completion_still_requires_migration_progress(self): + self.assertFalse(rebalance_complete(self.document("Completed"), "operation-1")) + self.assertTrue(rebalance_complete(self.document("Completed", 10), "operation-1")) + + def test_live_io_cannot_pass_when_operation_completed_during_io(self): + harness = self.harness(self.document("Completed", 10)) + harness.seed_and_expand = Mock(return_value=("bucket", [])) + harness.start_rebalance = Mock(return_value="operation-1") + harness.wait_rebalance = Mock() + harness.exercise_live_io = Mock(return_value=[]) + harness.assert_inventory = Mock() + with self.assertRaisesRegex(AssertionError, "did not remain active"): + harness.case_rebalance_live_io("case") + self.assertEqual(harness.wait_rebalance.call_count, 1) + harness.assert_inventory.assert_not_called() + + def test_large_rebalance_fixture_has_a_capacity_admission_check(self): + harness = self.harness(self.document()) + harness.admin_json = Mock(return_value={"disks": [ + {"state": "ok", "totalspace": 20 * 1024**3, "usedspace": 1024**3} for _ in range(4) + ]}) + self.assertEqual(PoolHarness.rebalance_fixture_size(harness, "case"), 128 * 1024**2) + harness.admin_json.return_value = {"disks": [ + {"state": "ok", "totalspace": 1024**3, "usedspace": 0} for _ in range(4) + ]} + with self.assertRaisesRegex(RuntimeError, "ENVIRONMENT-CAPACITY"): + PoolHarness.rebalance_fixture_size(harness, "case") if __name__ == "__main__": diff --git a/validation-migration-review.json b/validation-migration-review.json index 647a8a4..9ef68ad 100644 --- a/validation-migration-review.json +++ b/validation-migration-review.json @@ -1,11 +1,11 @@ { "schema_version": 1, - "as_of": "2026-10-28", + "as_of": "2026-11-10", "source_commit": "524eb5b32c3236497fdd8a9271df0494fa3d9974", "source_sha256": "463fdd02dee43e42fef4f3aacf7506709ce87af535e1a53d91c521bc2e361cad", - "reviewed_testing_commit": "f0744d198769feecb2cb038ab96b507a1784d44d", + "reviewed_testing_commit": "50baebb79f52b3b817f41c45db551588a4b28bd7", "scope": "Test implementation migration and candidate live acceptance are independent. This review does not rewrite original results, change migration_status semantics or relax the canonical gate. Environment procedures remain disabled by default under the accepted delivery scope.", - "evidence_basis": "20261026-canonical-migration-ledger-7094.json plus later G01/G02 attempts and source review on 2026-10-28; historical result bytes and audit reports remain in the separate evidence store.", + "evidence_basis": "20261026-canonical-migration-ledger-7094.json plus later G01/G02 attempts and source review on 2026-10-28; historical result bytes and audit reports remain in the separate evidence store. 2026-11-10: 对 Rebalance 活动夹具、容量准入、Started 非零进度与提前完成诊断进行限定复核；本次本地回归不新增产品 live acceptance。", "reviewed_files": { "scripts/run_validation_case.sh": "ac51328ff860074f285999236a07ec8d2274d56607dc7c6c6389747fd7a0c498", "scripts/validation_heal/authority.py": "8f2225f184427674de1f45aff4c4976c812f9d70fd4bc927ce460726a4ff74a0", @@ -68,7 +68,7 @@ "scripts/validation_heal/version_race_runner.py": "5dd244ae2aa816907d3711a6a7c9bf3dac76265c76967f7803437195894ff0e0", "scripts/validation_heal/versions.py": "fa77397a58b0db4499114c0e30a766ef67f997d1c88f22fcfe02752820b5eea0", "scripts/validation_pool/case_plan.py": "277777e040501c8a79e7104471f411b054d901afb920d18f3e222d3a0b7b03bc", - "scripts/validation_pool/pool_behavior.py": "203bc3a2b7e85fb95482dd33efc937cff9f1f7e23f8ac2aa0b9ad2f40aff4632", + "scripts/validation_pool/pool_behavior.py": "3850809ca92e809656abf15e89bee28f8baea6b539dbefafe19e79d43afea375", "scripts/validation_pool/run_case.sh": "e846559a26ccc45b0bba1fbba272ba8a921ea78079948b7f6f2ef6efa60bc25d", "test_validation_heal.py": "6e4e25d2437a49db1af0b3020bb3a2509496b21cb3c27216a6b0322aea9bfbdc", "test_validation_heal_authority.py": "a4334246aa9572be05a3d6881779df1e2495d9706d3a9289ba9437ac866a688a", @@ -107,7 +107,9 @@ "test_validation_heal_race_native.py": "850c9683ac918834d22b002ec2f5a8c0d7fd2b74266461e4912776dfceb0d93d", "test_validation_heal_schema.py": "5df440df29e7085d48590665129f87b0791f9fa8c7f64c8c0432000bc2002606", "test_validation_heal_unformatted.py": "58bc43f3f0905c8c4b7d0f8e656d25f0a17e197b5a2dfb66a88b6af9747883a7", - "test_validation_pool.py": "e0c894b9df84be1cafe03d2fa6cf011d44cb1810f25e5543e9eb17c0188d6e18" + "test_validation_pool.py": "c9653c3f30babb493eb532b188d55371f96b147941890681448d74c6df03449f", + "scripts/validation_pool/pool_helpers.py": "0022d0f60c0928411dc85c9fa2a2176f50c0a36b930e45d78d369429cf03c6fd", + "scripts/storage_capacity.py": "01fe5315c8e842fe5c53bf1d1fe530f0c866f25be6a98afd39a5c2d2b91910b7" }, "cases": [ { @@ -145,9 +147,13 @@ { "case_id": "pool.expansion-restart-persistence", "implementation": "implemented", - "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance. 2026-11-10 复核：独立 Rebalance 活动夹具在写入前检查容量；Started 与非零进度、操作后仍 active、终态迁移和对象完整性断言保持严格。提前 Completed 报告活动窗口不足；历史 accepted runs 仅保留原产物的验收含义。", "references": [ - "scripts/run_validation_case.sh" + "scripts/run_validation_case.sh", + "scripts/validation_pool/pool_behavior.py", + "scripts/validation_pool/pool_helpers.py", + "scripts/storage_capacity.py", + "test_validation_pool.py" ], "gaps": [], "live": { @@ -161,9 +167,13 @@ { "case_id": "pool.rebalance-stop-resume", "implementation": "implemented", - "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance. 2026-11-10 复核：独立 Rebalance 活动夹具在写入前检查容量；Started 与非零进度、操作后仍 active、终态迁移和对象完整性断言保持严格。提前 Completed 报告活动窗口不足；历史 accepted runs 仅保留原产物的验收含义。", "references": [ - "scripts/run_validation_case.sh" + "scripts/run_validation_case.sh", + "scripts/validation_pool/pool_behavior.py", + "scripts/validation_pool/pool_helpers.py", + "scripts/storage_capacity.py", + "test_validation_pool.py" ], "gaps": [], "live": { @@ -177,9 +187,13 @@ { "case_id": "pool.rebalance-live-io", "implementation": "implemented", - "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance. 2026-11-10 复核：独立 Rebalance 活动夹具在写入前检查容量；Started 与非零进度、操作后仍 active、终态迁移和对象完整性断言保持严格。提前 Completed 报告活动窗口不足；历史 accepted runs 仅保留原产物的验收含义。", "references": [ - "scripts/run_validation_case.sh" + "scripts/run_validation_case.sh", + "scripts/validation_pool/pool_behavior.py", + "scripts/validation_pool/pool_helpers.py", + "scripts/storage_capacity.py", + "test_validation_pool.py" ], "gaps": [], "live": { @@ -314,9 +328,13 @@ { "case_id": "pool.operation-guards", "implementation": "implemented", - "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance. 2026-11-10 复核：独立 Rebalance 活动夹具在写入前检查容量；Started 与非零进度、操作后仍 active、终态迁移和对象完整性断言保持严格。提前 Completed 报告活动窗口不足；历史 accepted runs 仅保留原产物的验收含义。", "references": [ - "scripts/run_validation_case.sh" + "scripts/run_validation_case.sh", + "scripts/validation_pool/pool_behavior.py", + "scripts/validation_pool/pool_helpers.py", + "scripts/storage_capacity.py", + "test_validation_pool.py" ], "gaps": [], "live": {