diff --git a/.github/workflows/kms-script-tests.yml b/.github/workflows/kms-script-tests.yml index e53a32ea..60b58201 100644 --- a/.github/workflows/kms-script-tests.yml +++ b/.github/workflows/kms-script-tests.yml @@ -5,12 +5,14 @@ on: paths: - rustfs-kms-test.sh - test_rustfs_kms_presign.py + - test_validation_kms.sh - .github/workflows/kms-script-tests.yml push: branches: [main] paths: - rustfs-kms-test.sh - test_rustfs_kms_presign.py + - test_validation_kms.sh - .github/workflows/kms-script-tests.yml permissions: @@ -28,3 +30,5 @@ jobs: run: bash -n rustfs-kms-test.sh - name: Test presigned KMS case without a server run: python3 -m unittest test_rustfs_kms_presign.py + - name: Test canonical KMS round trip without a server + run: bash test_validation_kms.sh diff --git a/.github/workflows/s3-script-tests.yml b/.github/workflows/s3-script-tests.yml index 64f74358..45aeec1e 100644 --- a/.github/workflows/s3-script-tests.yml +++ b/.github/workflows/s3-script-tests.yml @@ -6,6 +6,7 @@ on: - rustfs-s3-compat-test.sh - test_rustfs_s3_usage.py - test_rustfs_s3_signing.py + - test_validation_s3_basic.sh - .github/workflows/s3-script-tests.yml push: branches: [main] @@ -13,6 +14,7 @@ on: - rustfs-s3-compat-test.sh - test_rustfs_s3_usage.py - test_rustfs_s3_signing.py + - test_validation_s3_basic.sh - .github/workflows/s3-script-tests.yml permissions: @@ -32,3 +34,5 @@ jobs: run: python3 -m pip install botocore==1.43.19 - name: Test S3 cases and signatures without a cluster run: python3 -m unittest test_rustfs_s3_usage.py test_rustfs_s3_signing.py + - name: Test validation S3 round trip without a cluster + run: bash test_validation_s3_basic.sh diff --git a/.github/workflows/validation-catalog-tests.yml b/.github/workflows/validation-catalog-tests.yml new file mode 100644 index 00000000..89f2ed9d --- /dev/null +++ b/.github/workflows/validation-catalog-tests.yml @@ -0,0 +1,185 @@ +name: Validation Catalog Tests + +on: + pull_request: + paths: + - validation-cases.json + - validation-migration-review.json + - scripts/validation_migration_review.py + - scripts/validation_catalog.py + - scripts/validation_report.py + - scripts/validation_result.py + - test_validation_migration.py + - test_validation_ilm.py + - scripts/validation_ilm/** + - test_validation_pool.py + - scripts/validation_pool/** + - rustfs_pool_expand.sh + - test_validation_replication.py + - scripts/validation_replication/** + - test_validation_heal.py + - test_validation_heal*.py + - test_validation_heal_recovery.py + - test_validation_heal_authorization.py + - test_validation_heal_full_restart.py + - test_validation_heal_mrf.py + - test_validation_heal_unformatted.py + - test_validation_heal_performance.py + - test_validation_heal_multi_pool.py + - test_validation_heal_scopes.py + - test_validation_heal_bucket_metadata.py + - test_validation_heal_write_quorum.py + - test_validation_heal_coordinator_crash.py + - test_validation_heal_flapping.py + - test_validation_heal_capacity.py + - test_validation_heal_network.py + - test_validation_heal_encrypted.py + - test_validation_heal_schema.py + - test_validation_heal_native.py + - test_validation_heal_lock_successor.py + - test_validation_heal_race_native.py + - test_validation_heal_incarnation_native.py + - test_validation_heal_manager_native.py + - test_validation_heal_manager_fixture.py + - test_validation_heal_journal_fault.py + - test_validation_heal_checkpoint.py + - test_validation_heal_authority.py + - test_validation_heal_orphan.py + - test_validation_heal_options.py + - test_validation_heal_environment.py + - test_validation_heal_queue_pressure.py + - test_validation_heal_nolock.py + - test_validation_heal_race.py + - test_validation_heal_g06_source.py + - test_validation_heal_environment_gate.py + - test_validation_heal_outcome.py + - test_validation_heal_outcome_grace.py + - test_validation_heal_persistence.py + - test_validation_heal_persistence_mrf.py + - test_validation_heal_incarnation.py + - test_validation_heal_incarnation_replay.py + - scripts/validation_heal/** + - testdata/i05-*.ndjson + - testdata/validation/i05-*.zip + - docs/validation-heal-*.md + - rustfs_heal_test.sh + - test_validation_oidc.py + - scripts/validation_oidc.py + - scripts/validation_oidc/** + - test_validation_webhooks.py + - scripts/validation_webhooks.py + - scripts/run_validation_case.sh + - rustfs-replication-test.sh + - rustfs-storage-test.sh + - test_validation_scanner_usage.sh + - .github/workflows/validation-catalog-tests.yml + push: + branches: [main] + paths: + - validation-cases.json + - validation-migration-review.json + - scripts/validation_migration_review.py + - scripts/validation_catalog.py + - scripts/validation_report.py + - scripts/validation_result.py + - test_validation_migration.py + - test_validation_ilm.py + - scripts/validation_ilm/** + - test_validation_pool.py + - scripts/validation_pool/** + - rustfs_pool_expand.sh + - test_validation_replication.py + - scripts/validation_replication/** + - test_validation_heal.py + - test_validation_heal*.py + - test_validation_heal_recovery.py + - test_validation_heal_authorization.py + - test_validation_heal_full_restart.py + - test_validation_heal_mrf.py + - test_validation_heal_unformatted.py + - test_validation_heal_performance.py + - test_validation_heal_multi_pool.py + - test_validation_heal_scopes.py + - test_validation_heal_bucket_metadata.py + - test_validation_heal_write_quorum.py + - test_validation_heal_coordinator_crash.py + - test_validation_heal_flapping.py + - test_validation_heal_capacity.py + - test_validation_heal_network.py + - test_validation_heal_encrypted.py + - test_validation_heal_schema.py + - test_validation_heal_native.py + - test_validation_heal_lock_successor.py + - test_validation_heal_race_native.py + - test_validation_heal_incarnation_native.py + - test_validation_heal_manager_native.py + - test_validation_heal_manager_fixture.py + - test_validation_heal_journal_fault.py + - test_validation_heal_checkpoint.py + - test_validation_heal_authority.py + - test_validation_heal_orphan.py + - test_validation_heal_options.py + - test_validation_heal_environment.py + - test_validation_heal_queue_pressure.py + - test_validation_heal_nolock.py + - test_validation_heal_race.py + - test_validation_heal_g06_source.py + - test_validation_heal_environment_gate.py + - test_validation_heal_outcome.py + - test_validation_heal_outcome_grace.py + - test_validation_heal_persistence.py + - test_validation_heal_persistence_mrf.py + - test_validation_heal_incarnation.py + - test_validation_heal_incarnation_replay.py + - scripts/validation_heal/** + - testdata/i05-*.ndjson + - testdata/validation/i05-*.zip + - docs/validation-heal-*.md + - rustfs_heal_test.sh + - test_validation_oidc.py + - scripts/validation_oidc.py + - scripts/validation_oidc/** + - test_validation_webhooks.py + - scripts/validation_webhooks.py + - scripts/run_validation_case.sh + - rustfs-replication-test.sh + - rustfs-storage-test.sh + - test_validation_scanner_usage.sh + - .github/workflows/validation-catalog-tests.yml + +permissions: + contents: read + +jobs: + contract-tests: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + with: + persist-credentials: false + - run: python3 scripts/validation_catalog.py check + - run: python3 -B scripts/validation_migration_review.py --require-complete + - run: python3 -B -m unittest -v test_validation_migration.py + - run: python3 -B -m unittest -v test_validation_ilm.py + - run: python3 -B -m unittest -v test_validation_pool.py + - run: python3 -B -m unittest -v test_validation_replication.py + - run: python3 -m pip install -r scripts/validation_ilm/requirements.txt + - name: Install network namespace test dependency + run: | + if ! command -v nft >/dev/null; then + sudo apt-get update + sudo apt-get install --no-install-recommends -y nftables + fi + - run: python3 -B -m unittest -v test_validation_heal.py test_validation_heal_recovery.py test_validation_heal_authorization.py test_validation_heal_full_restart.py test_validation_heal_mrf.py test_validation_heal_unformatted.py test_validation_heal_performance.py test_validation_heal_scopes.py test_validation_heal_multi_pool.py test_validation_heal_bucket_metadata.py test_validation_heal_native.py test_validation_heal_manager_native.py test_validation_heal_manager_fixture.py test_validation_heal_journal_fault.py test_validation_heal_checkpoint.py test_validation_heal_authority.py test_validation_heal_orphan.py test_validation_heal_options.py test_validation_heal_environment.py test_validation_heal_environment_gate.py test_validation_heal_queue_pressure.py test_validation_heal_nolock.py test_validation_heal_race.py test_validation_heal_write_quorum.py test_validation_heal_schema.py test_validation_heal_encrypted.py test_validation_heal_coordinator_crash.py test_validation_heal_flapping.py test_validation_heal_capacity.py test_validation_heal_network.py test_validation_oidc.py test_validation_webhooks.py + - run: python3 -B -m unittest -v test_validation_heal_timeout_retention.py test_validation_heal_race_retention.py test_validation_heal_retained_scene.py test_validation_heal_lock_timeout_runner.py test_validation_heal_lock_source.py test_validation_heal_lock_timeout.py test_validation_heal_lock_observer.py test_validation_heal_lock_successor.py test_validation_heal_caller_abort.py test_validation_heal_status_keepalive.py test_validation_heal_mpu_race_runner.py test_validation_heal_race_io.py test_validation_heal_race_physical.py test_validation_heal_g03_source.py test_validation_heal_version_race_fixture.py test_validation_heal_version_race_physical.py test_validation_heal_version_race_adapter.py test_validation_heal_version_race_runner.py + - run: python3 -B scripts/validation_heal/matrix.py check + - run: python3 -B -m unittest -v test_validation_heal_outcome.py test_validation_heal_outcome_grace.py + - run: python3 -B -m unittest -v test_validation_heal_persistence.py test_validation_heal_persistence_mrf.py + - run: python3 -B -m unittest -v test_validation_heal_incarnation.py test_validation_heal_incarnation_replay.py + - run: python3 -B -m unittest -v test_validation_heal_incarnation_native.py + - run: python3 -B -m unittest -v test_validation_heal_race_native.py test_validation_heal_g06_source.py + - run: python3 -B -m unittest -v test_validation_heal_g02_source.py test_validation_heal_endpoint_resolution.py + - run: bash -n scripts/run_validation_case.sh scripts/validation_heal/run_case.sh scripts/validation_oidc/run_case.sh + - run: bash -n rustfs-storage-test.sh rustfs_pool_expand.sh rustfs-replication-test.sh scripts/validation_pool/run_case.sh scripts/validation_replication/run_case.sh test_validation_scanner_usage.sh + - run: bash test_validation_scanner_usage.sh diff --git a/README.md b/README.md index 006a8da2..b7c0171a 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,84 @@ # auto-testing Auto testing scripts +## Release validation migration + +`validation-cases.json` snapshots all 158 canonical cases from +`rustfs-release-validation/scenarios/catalog.yaml` (66 implemented, 84 +procedure-ready, 8 planned). The source status is historical context. A run +result carries `migration_status: verified` only after its assertions, topology, +and live evidence are checked in this repository. `reuse_candidates` names +existing suites to inspect and never grants coverage by itself. + +```bash +python3 scripts/validation_catalog.py check +python3 scripts/validation_catalog.py summary +python3 -B -m unittest -v test_validation_migration.py +``` + +To refresh after an intentional source catalog change: + +```bash +python3 scripts/validation_catalog.py sync \ + --source /absolute/path/to/rustfs-release-validation/scenarios/catalog.yaml +``` + +`scripts/validation_report.py` accepts one JSON result per canonical case ID. +Results must separately report source status, migration status, verdict, +cleanup, candidate/rc provenance, and evidence for each source assertion. +Pass `--gate-status implemented` only after the 66 migrated executors are +ready; missing or unverified cases fail that gate. Without a gate status, the +report shows migration progress and still rejects malformed or false PASS +results. + +The executable canonical cases currently include `s3.basic-object-round-trip`, +`scanner.usage-convergence`, `kms.sse-kms-round-trip`, +`tiering.transition-and-restore`, `events.delivery`, +`audit.request-correlation`, `iam.policy-enforcement`, and +`scanner.status-readiness`. The S3, KMS, and IAM +cases use single-disk +deployment. The scanner case uses the storage suite's single-node four-disk +deployment, waits for a post-upload successful scanner cycle, and requires a +complete usage snapshot with exactly 20 objects and 20,690 logical bytes. +Each requires a candidate manifest with `source_sha`, `package_url`, and +`package_sha256`. The installed binary +SHA-256 is measured on the test node and included in result provenance: + +```bash +./rustfs-s3-compat-test.sh \ + --validation-case s3.basic-object-round-trip \ + --candidate-manifest /absolute/path/candidate.json \ + --testing-commit "$(git rev-parse HEAD)" \ + --validation-artifacts-dir /absolute/path/new-artifacts-dir \ + --yes + +./rustfs-storage-test.sh \ + --validation-case scanner.usage-convergence \ + --candidate-manifest /absolute/path/candidate.json \ + --testing-commit "$(git rev-parse HEAD)" \ + --validation-artifacts-dir /absolute/path/another-new-artifacts-dir \ + --yes + +./rustfs-kms-test.sh \ + --validation-case kms.sse-kms-round-trip \ + --candidate-manifest /absolute/path/candidate.json \ + --testing-commit "$(git rev-parse HEAD)" \ + --validation-artifacts-dir /absolute/path/kms-new-artifacts-dir \ + --yes + +python3 scripts/validation_report.py \ + --results-dir /absolute/path/new-artifacts-dir/cases \ + --output /absolute/path/new-artifacts-dir/report.md +``` + +The report's full implemented-case gate is intentionally not enabled by these +runs. Run it with `--gate-status implemented` only after all 66 +implemented source cases have verified executors and live results. + +S3, scanner, KMS, tiering, and IAM have live PASS evidence against nightly +build `37958901959` attempt `1` on the current auto-testing fleet. Events and +audit have local executor tests but still need live verification. + ## Fault-tolerance suite `rustfs-fault-tolerance-test.sh` writes a machine-readable completion result diff --git a/docs/validation-heal-cases.md b/docs/validation-heal-cases.md new file mode 100644 index 00000000..72b29d74 --- /dev/null +++ b/docs/validation-heal-cases.md @@ -0,0 +1,1977 @@ +# Heal validation procedures + +These source procedures require live execution and branch evidence on the current auto-testing Heal fleet. +Use [the operations reference](validation-heal-operations.md) before injecting a fault. A procedure card or complete archive is not a PASS. + + + +## A01: heal.admin-healthy-noop + +Topologies: H1,H2. Branches: normal, deep. + +1. 用 DB 夹具生成完整 manifest；先对每个对象保存 S3 基线和四盘分片摘要，关闭 scanner/MRF/auto-heal。 +2. 依次执行 START object（scanMode=1、2），WAIT 原 token 直至终态；再 STATUS 同 token 两次。 +3. 对每个对象的 outcome 区分 Normal 的 metadata_healthy 与 Deep 的 verified_healthy；两者均要求 healed=0、unchanged>0，Normal 不得冒充 payload 完整性证明。比较目标/控制对象物理摘要和 S3 清单，重复请求不得新建版本。 + +Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [BASELINE](validation-heal-operations.md#BASELINE) → [START](validation-heal-operations.md#START) → [WAIT](validation-heal-operations.md#WAIT) → [STATUS](validation-heal-operations.md#STATUS) → [VERIFY](validation-heal-operations.md#VERIFY) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) + +Source assertions: + +- 对健康对象分别执行 normal/deep，再重复查询 +- 不修改用户数据或版本；Normal 只接受 `metadata_healthy`，Deep 才要求 `verified_healthy`，healthy 与 repaired 分开计数，不能把检查盘数当修复盘数。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## A02: heal.admin-scope-isolation + +Topologies: H1,H2. Branches: root, bucket, prefix, object, pool-set. + +1. 用 DS 在两个桶的 a/、ab/、b/ 写入不同内容；另在未选 set 放控制对象；逐对象预先记录实际 pool/set。 +2. 分别隔离选中对象、相邻前缀及另一桶/未选 set 的一个 part。每个 selector 使用新夹具和故障 ID。 +3. 对象请求路径到完整 key；prefix 路径到 a/ 且 recursive=true；bucket 路径只含桶；root 为 /；pool/set 写入 body。START 后 WAIT。 +4. 先 PHYSICAL：选择范围内全部重建、范围外控制故障仍缺失；然后 GET 验证，避免验证请求提前修好控制故障。root 必须包含整个专用服务清单。 + +Operations: [SEED_DS](validation-heal-operations.md#SEED_DS) → [BASELINE](validation-heal-operations.md#BASELINE) → [FAULT](validation-heal-operations.md#FAULT) → [START](validation-heal-operations.md#START) → [WAIT](validation-heal-operations.md#WAIT) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 分别以 root、bucket、prefix、object、pool/set 选择器启动 +- 目标范围实际遍历；相邻前缀、其他桶及未选 set 的故障保持原状；按对象清单而非目录数量验收。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## A03: heal.admin-token-cross-node-status + +Topologies: H2. Branches: coordinator-to-peer, coordinator-to-stable-endpoint. + +1. 在 H2 同一跨节点 set 写入 DB；保存协调者、目标、稳定入口分别对应的端点，不把同一节点的两个别名当跨节点。 +2. 隔离目标节点 part；从 n0 START，记录原始 clientToken；通过 n1 和 stable 对同一路径、同 token 执行 STATUS。 +3. 按同一任务身份对齐三端状态，最多等候冻结的状态收敛预算；终态及核心计数必须一致，404/unknown 不能视为结束。 +4. WAIT 后先物理证明目标重建，再从所有 S3 入口 VERIFY；保存每个请求实际连接的节点日志。 + +Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [FAULT](validation-heal-operations.md#FAULT) → [START](validation-heal-operations.md#START) → [STATUS](validation-heal-operations.md#STATUS) → [WAIT](validation-heal-operations.md#WAIT) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 从节点甲启动，经乙及稳定入口按原 token 查询 +- 任务身份、scope、终态及核心计数有界收敛；状态缺失/未知不能解释为完成。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## A04: heal.admin-dry-run + +Topologies: H1,H2. Branches: missing-remove0-recreate0, missing-remove1-recreate0, bitflip-remove0-recreate1, bitflip-remove1-recreate1, versioned-missing, versioned-bitflip. + +1. 分别用 DB 与 DV 建夹具，冻结全部 versionId/marker；关闭后台来源，不在注入后 GET。 +2. 在原始 part 上做 missing/bitflip，每组独立新目录；保存故障后的 PHYSICAL 快照作为 dry-run 前基线。 +3. START body 同时传 dryRun=true、scanMode=2 和该实例的 remove/recreate；WAIT 仅接受 dry_run_observed，不能要求 dry-run 持有 incarnation。 +4. 前后所有分片/元数据/marker 字节与版本清单相同；没有 repaired ACK。完成非变更证明后恢复故障，再 VERIFY。 + +Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [SEED_DV](validation-heal-operations.md#SEED_DV) → [BASELINE](validation-heal-operations.md#BASELINE) → [FAULT](validation-heal-operations.md#FAULT) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [START](validation-heal-operations.md#START) → [WAIT](validation-heal-operations.md#WAIT) → [RECOVER](validation-heal-operations.md#RECOVER) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 对缺片与损坏对象执行 dryRun，参数化 remove/recreate +- 有明确 dry_run_observed 诊断；用户对象分片、xl.meta、版本与删除标记不变；后台修复干扰须排除。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## A05: heal.admin-overlap-and-force-start + +Topologies: H1,H2. Branches: same-scope, parent-child, disjoint, force-start. + +1. 用 DX 超过 128 项的外部分片夹具注入多对象缺片；预先从固定候选记录 overlap policy（复用 token 或冲突错误），禁止接受任意结果。 +2. START 递归范围，WINDOW 证明 0 + +## A06: heal.admin-stop-and-stale-token + +Topologies: H1,H2. Branches: stop-active, repeat-stop, old-status, old-stop-new-task. + +1. DX 批量缺片后 START；WINDOW 观察非零且未完成进度，再 STOP 原 token。 +2. 状态显示 aborted/cancelled 后仍观察物理写入静默窗口；停止受理成功不能替代在途退出。 +3. 新 START 得到不同 token，再对旧 token STATUS/STOP；新任务不应被旧请求终止，重复 stop 按冻结的 HTTP/错误码验收。 +4. 新任务完成后完整 VERIFY；记录取消项不能算 repaired，所有暂态错误保留原始 body。 + +Operations: [SEED_DX](validation-heal-operations.md#SEED_DX) → [FAULT](validation-heal-operations.md#FAULT) → [START](validation-heal-operations.md#START) → [WINDOW](validation-heal-operations.md#WINDOW) → [STOP](validation-heal-operations.md#STOP) → [STATUS](validation-heal-operations.md#STATUS) → [WAIT](validation-heal-operations.md#WAIT) → [VERIFY](validation-heal-operations.md#VERIFY) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) + +Source assertions: + +- 非零修复进度时 stop，稳定后新建任务，再发旧 token 的查询/停止 +- 停止需等在途写结束并稳定；新任务身份独立，旧请求不得终止新任务；重复 stop 按公开契约处理。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## A07: heal.admin-invalid-request + +Topologies: H1,H2. Branches: duplicate-query, invalid-bool, invalid-scan, conflicting-flags, invalid-selector, oversize-body. + +1. 建立健康 DB 和 PHYSICAL 基线；保存后台/任务状态及对象版本集合，用于检查负请求未意外启动任务。 +2. 逐个发送服务端负例：重复 clientToken；forceStart=wat；scanMode=99；forceStart=true&forceStop=true；pool=-1/set 越界；超过候选限制的 body。每个例固定预期 HTTP 及结构化错误码。 +3. 使用 ADMIN_RAW 的重复 --query 和原始 --body-file，不能只调用 rc 的参数校验；不使用超出专用 Heal 路由的请求。 +4. 全部响应无有效新任务受理，服务仍可用；PHYSICAL、版本、后台责任无改变；若返回 2xx 或错误无明确原因保留 FAIL。 + +Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [BASELINE](validation-heal-operations.md#BASELINE) → [ADMIN_RAW](validation-heal-operations.md#ADMIN_RAW) → [STATUS](validation-heal-operations.md#STATUS) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 发送非法 selector、重复 query、非法布尔/scanMode、冲突 flags、超长 body +- 服务端返回已核实的结构化错误；无误启动、范围扩大或持久化副作用；CLI 参数错误不能替代服务端负例。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## A08: heal.admin-authorization + +Topologies: H1,H2. Branches: anonymous, bad-signature, expired-signature, denied-user, least-privilege-user. + +1. 通过 IAM 手册在专用服务创建两个临时用户，分别没有和仅有 HealAdminAction；用受控凭据环境运行，先证明限制用户不能读未授权对象。 +2. 管理员创建待查询/停止任务；分别对 start/status/stop 发匿名、坏签名、有效签名但时间超期、无授权用户请求；过期请求由隔离客户端签名时间设置产生，不改服务器时间。 +3. 检查冻结的 AccessDenied/签名/时间错误；响应不暴露对象结果。最小授权用户完成受理/查询/停止，证明接口本身可工作。 +4. 移除临时用户和策略并验证凭据失效；所有原始证据先脱敏再归档，不把管理员 token 暴露给无关主体。 + +Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [IAM](validation-heal-operations.md#IAM) → [START](validation-heal-operations.md#START) → [ADMIN_RAW](validation-heal-operations.md#ADMIN_RAW) → [STATUS](validation-heal-operations.md#STATUS) → [STOP](validation-heal-operations.md#STOP) → [VERIFY](validation-heal-operations.md#VERIFY) → [RECOVER](validation-heal-operations.md#RECOVER) + +Source assertions: + +- 匿名、坏签名、过期签名、无 HealAdminAction 用户与最小授权用户分别请求 +- 未授权 start/status/stop 均不能越权或泄漏对象信息；授权用户可完成规定操作，凭据不得写入证据。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## A09: heal.admin-nolock-safety + +Topologies: H1,H2. Branches: overwrite-nolock, delete-nolock, native-lock-fence. + +1. DB/DV 准备同一目标并缺片；两客户端使用同 key，冻结每个 PUT/DELETE 的唯一 nonce 和 ACK ledger。 +2. 一端 START body nolock=true，另一端按 WORKLOAD 提交覆盖或精确 versionId 删除；记录确实交叠的运行区间。 +3. 按 ACK 因果顺序重建期望清单并 VERIFY；最后成功覆盖不回退、已删除版本不复活。 +4. 另外运行 NATIVE 的锁/提交屏障回归并保存精确测试名、源码 SHA、命中日志；黑盒竞态与 native-lock-fence 单独结果，不以偶然重叠证明强制锁时点。 + +Operations: [SEED_DV](validation-heal-operations.md#SEED_DV) → [FAULT](validation-heal-operations.md#FAULT) → [WORKLOAD](validation-heal-operations.md#WORKLOAD) → [START](validation-heal-operations.md#START) → [WINDOW](validation-heal-operations.md#WINDOW) → [VERIFY](validation-heal-operations.md#VERIFY) → [NATIVE](validation-heal-operations.md#NATIVE) + +Source assertions: + +- 通过签名 API 传 nolock=true，并与同对象提交制造确定性争用 +- 兼容字段不得跳过存储层对象锁；最后成功写入和已确认删除保持正确，原生屏障与真机竞争证据分开。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## A10: heal.admin-option-contract + +Topologies: H1,H2. Branches: recursive-false, recursive-true, recreate, remove-false, remove-true, update-parity, read-repair, selector-valid, selector-invalid. + +1. 将候选 HealOpts 支持表和危险参数副作用写入本批 contract.json；不根据 CLI help 推导 API 未暴露字段。 +2. 为每个 variant 使用新 DB/DV：固定范围、足够权威和控制对象；remove=true 只对经 C10 证明无活引用的本次孤儿执行。 +3. 逐一 START 并对比只改变一个字段的基线；recursive 检查子前缀，pool/set 检查实际目标身份，非法组合要求结构化拒绝且无副作用。 +4. WAIT、PHYSICAL、VERIFY 三者验收；updateParity/readRepair 若候选不支持必须显示明确拒绝，不可悄悄按默认修复算覆盖。 + +Source implementation detail: frozen `heal_h1_options_batch.py` and +`heal_h1_options_node.py` at `524eb5b32c3236497fdd8a9271df0494fa3d9974` +run the remove=true orphan with a recursive bucket sweep after exact-version +DELETE and one-member full-object replay (including metadata). Its false +baseline is an option reference; remove=false is a separate repairable-object +branch. The adapter keeps every original H1/H2 DB/DV branch, with 68 registered +execution instances, and rejects the four formerly invented orphan baseline +instances. Original failures remain archived. Physical closure requires the +whole orphan to disappear, every post-DELETE control/format to remain exact, +and complete version-addressed task receipts/counters before reads or rescue. + +Complete live acceptance: [run 39197088183/1](https://github.com/rustfs/rustfs/actions/runs/39197088183) +on helper `b16e6072679d6a60a044b9d4520b30b4a79b5557` passed all 68 instances +and independent ACK/task/physical/public/cleanup audit, including 170 node +finalizations. Candidate `072b57b37559fbe626aeda8abf61617fdcbdd926`, package +`38043283428/1` and inspector `38715612825/1` remain immutable. Earlier failures +are retained; control rescue is not counted as product repair. + +Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [SEED_DV](validation-heal-operations.md#SEED_DV) → [FAULT](validation-heal-operations.md#FAULT) → [START](validation-heal-operations.md#START) → [WAIT](validation-heal-operations.md#WAIT) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 参数化 recursive、recreate、remove、updateParity、readRepair 与合法/非法 pool/set 组合 +- 仅测试已核实的候选语义；normal/deep 与危险 remove 的副作用明确；API 独有选项不得伪称 rc 已覆盖。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## B01: heal.disk-loss-and-repair + +Topologies: H1,H2. Branches: original-drive-return, offline-ack-repair. + +1. 由 layout.json 选一成员盘，记录同版本 N/R/W；使用 DRIVE_GATE 仅让本服务看不到该逻辑盘，不卸载共享 XFS。 +2. 故障窗口经测试进程 mount namespace 的读失败与 Admin offline 状态双重见证；在 quorum 允许范围执行 GET 和唯一 key PUT，逐次记 ACK。 +3. DRIVE_GATE 恢复原盘身份后，等待所选手工/自动来源处理离线期间责任；不得复制备份补盘。 +4. 验证所有故障前对象及成功 ACK 对象、目标盘实际新增分片；节点 ready/GET 成功不是补片证据。 + +Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [BASELINE](validation-heal-operations.md#BASELINE) → [DRIVE_GATE](validation-heal-operations.md#DRIVE_GATE) → [WORKLOAD](validation-heal-operations.md#WORKLOAD) → [BACKGROUND](validation-heal-operations.md#BACKGROUND) → [START](validation-heal-operations.md#START) → [WAIT](validation-heal-operations.md#WAIT) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 让一个已确认 set 成员盘对 heal 服务不可用，持续读取，恢复原盘 +- 记录真实故障窗口和源/目标盘身份；在可恢复条件下最终补齐目标分片及完整 manifest，不只验证降级 GET。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## B02: heal.disk-readonly-and-permission + +Topologies: H1,H2. Branches: read-only, permission-denied. + +1. 在专用测试服务的 mount namespace 设置目标根只读；permission 实例用专用非 root 服务 UID 和独立目录 ACL，不能对 root 仅 chmod 后假称生效。 +2. 在服务相同身份/namespace 执行有界写探针，要求 EROFS/EACCES；先保存并隔离目标 part，再触发 deep heal。 +3. 目标不可写期间必须保留失败/待修责任，不能 reported repaired；PHYSICAL 确認没有写到父目录/系统盘。 +4. 解除只读或恢复原 ACL，重新修复并 VERIFY；保留故障期间错误和恢复后物理补片证据。 + +permission-denied 分支释放 ACL gate 后，保持同一候选进程运行，并用进程 birth、候选摘要及释放后的物理快照证明状态。释放后、提交新的唯一 Admin token 前，目标分片必须仍保持故障态；若自动恢复先改变分片，该分支记为 `NOT_OBSERVED`，不能将恢复归因给新的手工任务。新的修复 token 终态后再停止服务并验证原始分片摘要。read-only 分支按其 unit mount gate 的生命周期清理 gate，再执行独立修复任务。 + +Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [FAULT](validation-heal-operations.md#FAULT) → [DRIVE_GATE](validation-heal-operations.md#DRIVE_GATE) → [START](validation-heal-operations.md#START) → [WAIT](validation-heal-operations.md#WAIT) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [RECOVER](validation-heal-operations.md#RECOVER) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 仅对独占测试卷制造只读/拒绝访问，恢复后重试 +- 不可写目标不得 reported repaired；错误及待修责任可见；恢复后真实写入目标且不落到系统盘或其他目录。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## B03: heal.disk-replacement-formatted + +Topologies: H1,H2. Branches: new-disk-data, new-disk-versioned. + +1. 完整基线后停止本例服务，将目标逻辑盘 data 整体隔离到本 run 的 quarantine；使用 REPLACE 建空的新 data 根。 +2. 按候选初始化契约启动，读取 format.json 证明新 this UUID 属于原 pool/set/slot；其他盘格式不得变成新集群。 +3. 显式 START root/deep 或本实例指定重建来源，旧盘备份不可读给测试进程；WAIT 全部责任。 +4. 逐对象/版本验证目标盘分片与同角色基线摘要；确认不是从 quarantine 拷贝回去；恢复操作只在取证后做并标为 rescue。 + +Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [SEED_DV](validation-heal-operations.md#SEED_DV) → [BASELINE](validation-heal-operations.md#BASELINE) → [REPLACE](validation-heal-operations.md#REPLACE) → [FORMAT](validation-heal-operations.md#FORMAT) → [START](validation-heal-operations.md#START) → [WAIT](validation-heal-operations.md#WAIT) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 换入空的新逻辑盘，确认新 disk UUID/正确格式后触发重建 +- 新盘与旧盘身份可区分且属于正确 pool/set/slot；逐版本重建完成，不能从隔离备份复制回来冒充 heal。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## B04: heal.disk-replacement-unformatted + +Topologies: H1,H2. Branches: blank-unformatted, foreign-format. + +The Functional Chain runs the shared read-only replacement admission gate +before package/service changes and the executor repeats it before reservation. +If exact endpoint mounts or the reviewed mount-switch adapter are unavailable, +the case is recorded as pre-deployment `BLOCKED` without creating either +the blank or foreign-format fixture. + +1. REPLACE 建没有 format 的空逻辑盘；记录空目录证据，启动并观察候选支持的安全初始化或明确拒绝。 +2. 第二实例使用同 run 的第二个隔离小集群生成外来 format，将其按 FORMAT_BAD 手册装入替换根，保留原始副本和所有 UUID。 +3. 外来身份不能被静默覆盖或挂到错误 set；若拒绝，等待有界 deadline 并确认其他正常盘数据不变。 +4. 停止本服务、撤回替换根、恢复原身份后 VERIFY；空盘正常接纳实例还须逐对象重建证明。 + +Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [REPLACE](validation-heal-operations.md#REPLACE) → [FORMAT_BAD](validation-heal-operations.md#FORMAT_BAD) → [BACKGROUND](validation-heal-operations.md#BACKGROUND) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [RECOVER](validation-heal-operations.md#RECOVER) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 引入空未格式化测试盘，并与带异集群格式的盘分别比较 +- 按候选支持的初始化契约安全接纳或明确拒绝；不得覆盖外来身份或将未就绪盘标为可写健康。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## B05: heal.multiple-shard-loss + +Topologies: H1,H2. Branches: loss-1-data, loss-1-parity, loss-2-through-m-data-parity. + +Before seeding, before each fault and after each faulted restart, the executor +observes signed read-only Admin info from every H1/H2 endpoint for at most +90 seconds. All 4/16 slots must match the frozen format UUIDs and report online; +read readiness alone does not prove this prerequisite. Every observation is +retained. Incomplete or foreign identities fail immediately; persistent offline +members expire as `NOT_OBSERVED` without retrying the Heal task. The original +`38200772548` unknown-version receipts remain a retained failure. This startup +check alone does not establish its underlying cause or a new live PASS. + +1. 从候选/实际元数据确定 M=N-R，枚举 1..M 的合法缺片数及 data/parity 组合，写入本批展开参数表；不得只运行 loss=1。 +2. 每实例用相同版本的新对象，将指定成员上的 part 分别 FAULT missing；多份元数据也要证明指向同 version/data_dir。 +3. 在满足修复提交条件时 START deep/WAIT；先检查全部目标片与角色基线相同，再 VERIFY。 +4. 每次恢复隔离备份后换新对象再做下一组合；低于 R 的组合另由 B07 计负例，不合并成绩。 + +Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [SEED_DV](validation-heal-operations.md#SEED_DV) → [FORMAT](validation-heal-operations.md#FORMAT) → [BASELINE](validation-heal-operations.md#BASELINE) → [FAULT](validation-heal-operations.md#FAULT) → [START](validation-heal-operations.md#START) → [WAIT](validation-heal-operations.md#WAIT) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 按实际同版本 EC 布局依次缺失 1..M 个 shard，覆盖 data/parity 角色组合 +- 每个参数实例单独满足读取与写入 quorum；满足修复条件时全部目标重建，不把跨版本或跨 set 的 shard 合并。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## B06: heal.read-quorum-boundary + +Topologies: H1,H2. Branches: exact-r-read, exact-r-heal-write-limited, restored-w. + +执行器仅接受经逐文件和锁策略复核的 `072b57b37559fbe626aeda8abf61617fdcbdd926` 与 GET quorum 修复候选 `1c7fc818dd858247c45b39542abd8291b1e6e31b`。记录所选候选的精确锁契约；inspector 必须同时匹配候选及当前 helper 源码摘要，未知候选在部署前拒绝。below-R GET 仍必须为候选冻结的 `503/SlowDownRead`，旧 `500/InternalError` 失败记录保留；支持新候选不等于新的 live 验收。 + +1. 记录读取 R、修复写 W、锁 quorum 的来源；选择同一版本保留恰好 R 个权威片，其他片隔离。 +2. 第一次请求 GET 验证读取边界并记录 read-repair 干扰；用第二个相同形态新对象执行只手工 deep heal。 +3. 如果只有 R 但写/锁 quorum 不足，要求拒绝持久化成功；若拓扑中该组合不可实现，在运行前声明精确原因并选满足条件的拓扑，不伪造离线数。 +4. 恢复到 W 后重新启动修复并保存独立 token，比较精确目标片与全部内容；成功读不能充修复成功。 + +Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [FORMAT](validation-heal-operations.md#FORMAT) → [FAULT](validation-heal-operations.md#FAULT) → [GET](validation-heal-operations.md#GET) → [START](validation-heal-operations.md#START) → [WAIT](validation-heal-operations.md#WAIT) → [RECOVER](validation-heal-operations.md#RECOVER) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 保留恰好 R 个同版本权威 shard，测试读取及修复提交，再恢复写 quorum +- 读取边界和修复写入边界分别判定；不能因还能 GET 就要求在写 quorum 不足时提交成功。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## B07: heal.below-read-quorum + +Topologies: H1,H2. Branches: r-minus-1-unversioned, r-minus-1-historical. + +1. 选定精确 versionId 保留 R-1 份同版本权威；隔离其余 part，确保没有其他 pool 或旧版本兜底被当作同版本来源。 +2. GET 必须返回冻结的不可恢复错误，不能返回 200 错内容或空对象；deep heal 的 execution/disposition 必须如实失败/未解决。 +3. 在恢复前快照所有好片和 xl.meta，证实负例没有扩大损坏或执行 remove；保留失败任务原 token。 +4. 仅在负例取证完成后 RECOVER 原始片，再显式 heal 和 VERIFY；恢复后的绿不能覆盖故障期间的错误。 + +Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [SEED_DV](validation-heal-operations.md#SEED_DV) → [FAULT](validation-heal-operations.md#FAULT) → [GET](validation-heal-operations.md#GET) → [START](validation-heal-operations.md#START) → [WAIT](validation-heal-operations.md#WAIT) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [RECOVER](validation-heal-operations.md#RECOVER) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 使权威来源小于 R，执行 GET 与 deep heal，再恢复被隔离源 +- 故障期间不得返回伪造的成功内容、空对象或误删记录；明确 unresolved/失败；恢复后原对象和版本完整。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## B08: heal.below-write-quorum + +Topologies: H1,H2. Branches: read-present-write-absent, lock-quorum-absent. + +1. 用 layout.json 分别计算对象写和锁 quorum；DRIVE_GATE/NETWORK 构造仍可读但不能安全提交的测试服务可见性。 +2. 执行唯一 key PUT 与已有缺片对象 heal；保存每个 ACK/HTTP/body，超时写必须查询唯一 key 解析结果，禁止直接重试换 key。 +3. 期间不能 reported repaired 或返回无法兑现的成功写 ACK；从各 peer 收集 quorum 与锁失败日志。 +4. 解除故障后按全部已确认/已解析写入重建 manifest，实际补齐目标并 VERIFY；普通低 quorum 错误不是基础设施错误。 + +Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [FORMAT](validation-heal-operations.md#FORMAT) → [DRIVE_GATE](validation-heal-operations.md#DRIVE_GATE) → [NETWORK](validation-heal-operations.md#NETWORK) → [WORKLOAD](validation-heal-operations.md#WORKLOAD) → [START](validation-heal-operations.md#START) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [RECOVER](validation-heal-operations.md#RECOVER) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 构造读 quorum 尚存而写/锁 quorum 不足，执行 heal 与新 PUT +- 不能虚报持久化修复或成功写入；分别记录对象、元数据、分布式锁 quorum；恢复后核对所有已确认写入。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## B09: heal.node-outage-rejoin + +Topologies: H2. Branches: one-node-stop, rejoin-with-new-versions. + +1. 确认 H2 目标 set 跨节点，统计停一节点实际损失的盘数，不能使用每台独立 pool 的 H5 替代。 +2. PROCESS stop 精确测试 unit，见证 MainPID 消失及端点不可达；保持其他节点服务。 +3. 按计算允许的 quorum 执行降级 PUT/DELETE/GET，保存版本化 ACK；重入同一节点同一数据根。 +4. 观察 MRF/修复责任直到补齐离线期间成功对象；从所有节点 VERIFY 和目标 PHYSICAL，不能只看进程 ready。 + +Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [SEED_DV](validation-heal-operations.md#SEED_DV) → [FORMAT](validation-heal-operations.md#FORMAT) → [PROCESS](validation-heal-operations.md#PROCESS) → [WORKLOAD](validation-heal-operations.md#WORKLOAD) → [BACKGROUND](validation-heal-operations.md#BACKGROUND) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 停止同一跨节点 set 的一个节点，执行允许的降级 I/O，再重入 +- 按实际丢失盘数判定可用性；缺失期间成功提交的对象补齐；节点重新 ready 不等于所有对象已重建。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## B10: heal.disk-and-node-flapping + +Topologies: H2. Branches: drive-flap-3, node-flap-3, interrupt-rebuild-again. + +1. 预注册三个故障周期、每次最长窗口和总 deadline；准备带 nonce 的 ACK ledger，设置只属于本服务的 drive gate 或 unit 操作。 +2. 每轮离线/在线都先见证实际状态；第三轮在 WINDOW 证明部分重建时再次离线，不以固定 sleep 替代。 +3. 记录每轮责任/队列、任务身份、错误率及 CPU/FD；超过预注册重试/资源预算即失败，不继续循环等绿。 +4. 彻底解除故障后验证全部已确认对象、目标分片和持久队列收敛；活动窗口没命中记 NOT_OBSERVED。 + +Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [DRIVE_GATE](validation-heal-operations.md#DRIVE_GATE) → [PROCESS](validation-heal-operations.md#PROCESS) → [WORKLOAD](validation-heal-operations.md#WORKLOAD) → [WINDOW](validation-heal-operations.md#WINDOW) → [BACKGROUND](validation-heal-operations.md#BACKGROUND) → [METRICS](validation-heal-operations.md#METRICS) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 重复短暂离线/在线并在重建途中再次失联 +- 重试与队列有界、身份稳定，无重复任务风暴；最终故障解除后收敛，所有已确认对象保持正确。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## C01: heal.missing-part-file + +Topologies: H1,H2. Branches: data-single-part, parity-single-part, multipart-first, multipart-middle, multipart-last. + +1. DB/DM 写入后用 FORMAT/PHYSICAL 将每个 version/part 映射到真实盘角色；每个 variant 选不同目标，不按盘序号猜 data/parity。 +2. 停住注入目标写入后 FAULT missing 只移除已冻结摘要的 part，xl.meta 和 format 保持；恢复运行后禁止目标 GET。 +3. START deep 并 WAIT；首先验证该角色目标 part 恢复到故障前完整 SHA-256/长度，再 GET/Range 和版本清单。 +4. 任何仅 xl.meta 恢复、重建错误 part 或 control 变更都失败；清理时才允许使用原始备份。 + +Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [SEED_DM](validation-heal-operations.md#SEED_DM) → [FORMAT](validation-heal-operations.md#FORMAT) → [BASELINE](validation-heal-operations.md#BASELINE) → [FAULT](validation-heal-operations.md#FAULT) → [START](validation-heal-operations.md#START) → [WAIT](validation-heal-operations.md#WAIT) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 仅隔离指定对象某个 part/shard，保留 format 与其他元数据，执行 deep heal +- 对应数据或校验分片重新生成，part/版本身份准确；不得只看到 xl.meta 重现就通过。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## C02: heal.data-shard-bitrot + +Topologies: H1,H2. Branches: data-first-byte, data-middle-byte, data-last-byte, multipart-data. + +1. 记录候选解析出的 data shard；用不重复 SHAKE 内容避免数据片/校验片摘要相同，验证健康基线。 +2. FAULT bitflip 在冻结的首/中/尾字节偏移各翻转一位；保存偏移、注入前后 hash 及目标物理角色。 +3. 保持足够正确来源，分别执行本批预注册 deep/read-repair 触发；手工实例不得预读目标。 +4. 目标恢复完整原始片摘要，解码后全文和 Range 正确；只返回正确 GET 但坏片仍在必须失败。 + +Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [SEED_DM](validation-heal-operations.md#SEED_DM) → [FORMAT](validation-heal-operations.md#FORMAT) → [FAULT](validation-heal-operations.md#FAULT) → [START](validation-heal-operations.md#START) → [WAIT](validation-heal-operations.md#WAIT) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 在指定 data shard 的已知偏移翻转字节，包含首/中/尾位置 +- 故障前后摘要证明注入生效；deep 或 read-repair 发现损坏并修复；独立 bitrot/EC 及 S3 全内容验证通过。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## C03: heal.parity-shard-bitrot + +Topologies: H1,H2. Branches: parity-first-byte, parity-middle-byte, parity-last-byte. + +1. 选择经 FORMAT 证明的 parity 片，保留足够 data；普通读取是否需要该片另作控制实例，不能触发后再声称纯 deep。 +2. FAULT bitflip 覆盖首/中/尾位置；注入后仅 PHYSICAL，不 GET/HEAD。 +3. START deep/WAIT，要求 repaired 责任对应正确对象/版本/pool/set；与修复前同角色 SHA-256 比较。 +4. 全文 GET 及版本/metadata 对照通过且错误 parity 已消失才通过；后台提前补片则该触发实例未观测。 + +Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [FORMAT](validation-heal-operations.md#FORMAT) → [FAULT](validation-heal-operations.md#FAULT) → [START](validation-heal-operations.md#START) → [WAIT](validation-heal-operations.md#WAIT) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 损坏 parity shard，使普通 GET 仍可能成功，再执行 deep heal +- 必须识别并重建损坏 parity；只有 GET 正确、无目标分片修复不能通过。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## C04: heal.truncated-and-zeroed-shard + +Topologies: H1,H2. Branches: truncate-half, truncate-near-tail, zero-all, checksum-header. + +1. 保存外部 part 完整备份及正确 metadata quorum；每种损坏使用新的独立对象。 +2. FAULT truncate 默认保留一半；near-tail 按 FAULT_FORMAT 手册只截去最后一字节并记录原始长度；zero 清零全部；header 翻转前 32 字节。 +3. 见证实际长度/摘要差异后 START deep，捕获短读/校验诊断与责任结果。 +4. 目标原始长度和完整片摘要恢复，GET 无截断/补零，控制对象不变；拒绝把一份可解析 metadata 当内容健康。 + +Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [BASELINE](validation-heal-operations.md#BASELINE) → [FAULT](validation-heal-operations.md#FAULT) → [FAULT_FORMAT](validation-heal-operations.md#FAULT_FORMAT) → [START](validation-heal-operations.md#START) → [WAIT](validation-heal-operations.md#WAIT) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 分别截断 shard、置零和破坏校验头，保留正确元数据仲裁来源 +- 短读、全零和校验损坏被准确识别；修复不能接受截断内容或错误长度。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## C05: heal.missing-or-corrupt-xlmeta + +Topologies: H1,H2. Branches: missing-xlmeta, truncated-xlmeta, corrupt-xlmeta, inline-payload, versioned-xlmeta. + +1. DB 小对象与 DV 历史版本先全量读取，FORMAT 解析每盘 xl.meta 的版本/inline/data_dir 关联并保存解析器源码 SHA。 +2. 在少数成员上对精确对象 xl.meta 做 missing/truncate/header；inline 实例仅由匹配候选的解析器定位 payload 后翻位，不能随便偏移后称 inline。 +3. 保留同版本正确来源，START deep/WAIT；恢复后重新解析 xl.meta，核对所有版本、marker 与 data 引用。 +4. 每个 versionId 全 GET 和版本集合一致；可以有合法重编码，但不能新建空 metadata、遗漏历史版本或移动删除语义。 + +Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [SEED_DV](validation-heal-operations.md#SEED_DV) → [FORMAT](validation-heal-operations.md#FORMAT) → [FAULT](validation-heal-operations.md#FAULT) → [FAULT_FORMAT](validation-heal-operations.md#FAULT_FORMAT) → [START](validation-heal-operations.md#START) → [WAIT](validation-heal-operations.md#WAIT) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 在少数成员上移走或损坏 xl.meta/inline 内容，保留足够权威来源 +- 恢复同一对象的全部应存版本、数据引用与删除标记；不以新建空元数据替代真正修复。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## C06: heal.stale-version-rejoin + +Topologies: H1,H2. Branches: unversioned-overwrite, delete-historical-version, delete-marker. + +1. 保存 DV/旧代次，DRIVE_GATE 让一个成员离线且其旧数据原样保留。 +2. 在线 quorum 执行新内容覆盖、精确旧 versionId 删除或创建 marker；串行 ACK 确定因果顺序，并在在线端确认提交。 +3. 恢复携带旧数据的同一盘，核对原 disk UUID 后观察手工/自动修复，并保存旧/新代次权威选择日志。 +4. 最终最新值与全部历史/marker 严格匹配 ACK 清单；已删除版本不复活；目标旧盘同一 key 的 metadata 与内容纠正。 + +Operations: [SEED_DV](validation-heal-operations.md#SEED_DV) → [DRIVE_GATE](validation-heal-operations.md#DRIVE_GATE) → [WORKLOAD](validation-heal-operations.md#WORKLOAD) → [BACKGROUND](validation-heal-operations.md#BACKGROUND) → [START](validation-heal-operations.md#START) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 盘离线后执行覆盖/版本删除，再让携带旧元数据的盘重入 +- 以已确认的新代次/版本为准；旧当前值与已删版本不复活，旧盘被纠正而非成为新的错误权威。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## C07: heal.conflicting-subquorum-authority + +Topologies: H1,H2. Branches: disjoint-version-quorums, disjoint-incarnations, native-authority-fence. + +1. 同一个本例 key 串行写两个不同版本/代次，保存各自成员副本；按 FORMAT_BAD 在静止服务下组合为两个各小于 R 的集合，原片全部保留。 +2. 解析每个成员版本身份，生成按 version/incarnation 分组的计数表；即使总数达到 R，也证明每一组都没有 quorum。 +3. GET/deep heal 必须安全拒绝，不以较新 mtime 或跨版本片并集提交；检查原本仍可恢复的证据没有被 remove。 +4. 运行匹配源代码的 NATIVE 权威仲裁回归；在真机解除混合现场后恢复原夹具并 VERIFY，两类证据分开计数。 + +Operations: [SEED_DV](validation-heal-operations.md#SEED_DV) → [FORMAT](validation-heal-operations.md#FORMAT) → [FORMAT_BAD](validation-heal-operations.md#FORMAT_BAD) → [GET](validation-heal-operations.md#GET) → [START](validation-heal-operations.md#START) → [WAIT](validation-heal-operations.md#WAIT) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [RECOVER](validation-heal-operations.md#RECOVER) → [VERIFY](validation-heal-operations.md#VERIFY) → [NATIVE](validation-heal-operations.md#NATIVE) + +Source assertions: + +- 用受控原生夹具构造互不构成 quorum 的版本/代次集合，再做对应真机反例 +- 禁止把不同版本的 shard 数量并集当成权威；无法证明时安全拒绝且保留可恢复证据，不猜测较新时间戳。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## C08: heal.metadata-payload-mismatch + +Topologies: H1,H2. Branches: length-mismatch, payload-mismatch, insufficient-authority. + +1. 先保存版本化外部 part 的全套 metadata 与 hash；从本例另一种长度/内容的版本取得替换 payload。 +2. 停服务按 FAULT_FORMAT 只更换少数 part 的内容或截断，保留原 xl.meta 声称的长度/摘要，不改健康成员。 +3. deep heal 对比 metadata 与实际内容；充分权威实例应重建，不充分权威实例应拒绝且保留好的副本。 +4. 取证后恢复，验证精确版本无跨代内容；保留实际解析结果，不把任意无效 header 注入当作长度字段覆盖。 + +Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [SEED_DV](validation-heal-operations.md#SEED_DV) → [FORMAT](validation-heal-operations.md#FORMAT) → [FAULT_FORMAT](validation-heal-operations.md#FAULT_FORMAT) → [START](validation-heal-operations.md#START) → [WAIT](validation-heal-operations.md#WAIT) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [RECOVER](validation-heal-operations.md#RECOVER) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 构造元数据声称某长度/摘要但实际 part 不匹配的少数副本 +- 元数据与内容联合判断；不把可解析的 xl.meta 当健康；权威不足时拒绝覆盖正确副本。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## C09: heal.foreign-format-and-unknown-schema + +Topologies: H1,H2. Branches: foreign-disk, foreign-set, future-format, future-object-schema. + +1. 用第二个隔离测试集群生成真实外来 disk/set format；当前实例只有一个替换逻辑根可被修改。 +2. FORMAT_BAD 在服务停止时分别装入外来 format 或按候选解析器支持的版本头写入未来 schema；记录原始 bytes/digest，不碰共享盘 format。 +3. 启动、GET/deep heal，核对明确隔离/unsupported，不得把未知格式当 missing 初始化或写成新集群。 +4. 停止服务收集原记录仍存在的物理证据；撤回坏根、恢复原身份后完整 VERIFY；无法确认格式字段不能冒充已注入。 + +Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [FORMAT](validation-heal-operations.md#FORMAT) → [REPLACE](validation-heal-operations.md#REPLACE) → [FORMAT_BAD](validation-heal-operations.md#FORMAT_BAD) → [PROCESS](validation-heal-operations.md#PROCESS) → [START](validation-heal-operations.md#START) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [RECOVER](validation-heal-operations.md#RECOVER) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 提供外来 disk/set 身份、未来 schema 或不支持的记录格式 +- 明确隔离/拒绝且不破坏原记录，不把未知当 missing 后直接重建；解除故障后原数据可用。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## C10: heal.dangling-and-orphan-safety + +Topologies: H1,H2. Branches: remove-false, remove-true-before-grace, remove-true-after-grace, shared-version-reference, pending-mpu. + +1. 创建 DV/DM 并合法覆盖/删除形成候选孤儿；以匹配格式解析器列出所有 version→data_dir 引用和 pending UploadId。 +2. 只在没有活引用的本次测试目录形成 dangling；保存 orphan 成因与真实年龄，宽限期实例等待配置窗口，不改全局时间。 +3. 分别 START remove=false/true；宽限期前、共享历史引用、pending MPU 必须保留，过期且有充分删除证明的孤儿才允许清理。 +4. 比较物理引用集合，所有应存版本及未完成 upload list_parts 完整；remove 误删任何共享引用即失败，救援只在取证后。 + +Operations: [SEED_DV](validation-heal-operations.md#SEED_DV) → [SEED_DM](validation-heal-operations.md#SEED_DM) → [FORMAT](validation-heal-operations.md#FORMAT) → [FAULT_FORMAT](validation-heal-operations.md#FAULT_FORMAT) → [START](validation-heal-operations.md#START) → [WAIT](validation-heal-operations.md#WAIT) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 构造 dangling part、旧 data_dir 与共享历史引用，分别 remove=false/true、宽限期前后检查 +- 默认不得破坏仍可恢复数据；显式 remove 只清除具有充分删除证明且过期的本次孤儿，不删共享版本或 MPU。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## C11: heal.bucket-incarnation-recreation + +Topologies: H1,H2. Branches: old-metadata-rejoin, old-checkpoint-replay, old-mrf-replay, native-incarnation-fence. + +Execution policy: `evidence-blocked`; disabled by default: `false`. + +Blocked reason (`EVIDENCE_INCARNATION_LIVE_ORCHESTRATION`): 已实现 H1/H2 旧元数据重返、真实 checkpoint/resume 回放、ACK 绑定的 committed MRF 回放和精确 native 证据门禁；尚缺全部现场分支的独立审计，诊断执行保持 FAIL/BLOCKED，原生前置通过不能替代 canonical 验收。 + +1. 建旧 DV 桶并让一盘离线；按 S3 正常列举/删除全部版本和 marker，确认 delete_bucket 成功，禁止直接内部清空绕过删除。 +2. 同名 create_bucket 后写新内容，保存新 incarnation；恢复旧盘并让旧任务/持久 MRF 重放。 +3. 对比任务身份和代次守卫，旧责任不得删除/覆盖新对象，不得把旧 marker 或数据复活到新桶。 +4. 按 NATIVE 验证精确 incarnation 提交屏障；真机清单及新桶配置始终匹配成功 ACK，旧/新物理范围独立留证。 + +Operations: [SEED_DV](validation-heal-operations.md#SEED_DV) → [DRIVE_GATE](validation-heal-operations.md#DRIVE_GATE) → [S3](validation-heal-operations.md#S3) → [BACKGROUND](validation-heal-operations.md#BACKGROUND) → [START](validation-heal-operations.md#START) → [FORMAT](validation-heal-operations.md#FORMAT) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) → [NATIVE](validation-heal-operations.md#NATIVE) + +Source assertions: + +- 盘离线时完成旧桶合法清空/删除，再同名建桶写入新对象，恢复旧盘与旧任务 +- 旧 bucket incarnation 的 checkpoint/MRF/元数据不得污染新桶；旧任务不得删除或恢复新 incarnation 的对象。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## D01: heal.object-size-boundaries + +Topologies: H1,H2. Branches: zero, one, 4kib-minus-exact-plus, inline-minus-exact-plus, stripe-minus-exact-plus, large-external. + +1. SEED_DB 显式包含 0、1、4095、4096、4097；从固定候选配置/FORMAT 获得 inline 与 EC block 阈值，新增阈值±1 和一个多 block 外部对象。 +2. 每个大小先按物理记录确认实际形态；零字节/inline 没有外部 part 时改用合法 metadata/inline 故障，不能虚构 part。 +3. data/parity 上依次 missing 与 bitflip，START deep/WAIT；每个形态独立 variant 计数。 +4. 核对全体长度/摘要/ETag/HEAD/LIST；跨边界及末字节 Range 为正确 206，空对象范围按冻结契约拒绝。 + +Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [FORMAT](validation-heal-operations.md#FORMAT) → [FAULT](validation-heal-operations.md#FAULT) → [FAULT_FORMAT](validation-heal-operations.md#FAULT_FORMAT) → [START](validation-heal-operations.md#START) → [WAIT](validation-heal-operations.md#WAIT) → [RANGE](validation-heal-operations.md#RANGE) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 覆盖 0/1 字节、页/inline/EC block 边界±1以及大对象 +- 每个实际存储形态均确认后注入；长度、SHA-256、ETag、HEAD/LIST 与边界 Range 正确，无假设阈值。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## D02: heal.completed-multipart-object + +Topologies: H1,H2. Branches: first-part, middle-part, last-part, cross-part-range. + +1. SEED_DM 建至少三 part 且非末 part≥5 MiB，非等长内容和独立 nonce；保存 UploadId、每个 part ETag、Complete ACK/最终版本。 +2. 每实例损坏第一、中间、末尾 part 对应的一个物理片；按 FORMAT 映射实际 part，不依赖文件名猜逻辑位置。 +3. START deep/WAIT 后先验证对应目标片，再完整 GET；最终 ETag 应与基线一致，但不用 multipart ETag 计算整对象 MD5。 +4. RANGE 覆盖两个 part 边界及尾部，比较逐字节/摘要、metadata 和版本；其他未完成 MPU 仍可 list_parts。 + +Operations: [SEED_DM](validation-heal-operations.md#SEED_DM) → [FORMAT](validation-heal-operations.md#FORMAT) → [FAULT](validation-heal-operations.md#FAULT) → [START](validation-heal-operations.md#START) → [WAIT](validation-heal-operations.md#WAIT) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [RANGE](validation-heal-operations.md#RANGE) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 修复已完成的不等长多 part 对象，分别损坏首/中/末 part +- 完整内容、part 顺序、原版本/ETag/元数据和跨 part Range 不变；multipart ETag 不能当整对象 MD5。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## D03: heal.historical-versions + +Topologies: H1,H2. Branches: old-version-part, current-version-part, recursive-all-versions, multiple-version-targets. + +1. SEED_DV 每 key 三个不同 payload 版本，逐一保存 versionId/ETag；可额外写 marker，清单不按当前 GET 代替历史读取。 +2. 为不同历史/当前版本选不同盘和 part，metadata 故障实例保留同版本 quorum。 +3. 对 object 和 recursive bucket 分别 START/WAIT；物理解析必须显示所有应存 version/data_dir 而不是只最新对象。 +4. 对每个原 versionId 全 GET，LIST versions 无新增/遗漏，isLatest 身份与 marker 顺序保持。 + +Operations: [SEED_DV](validation-heal-operations.md#SEED_DV) → [FORMAT](validation-heal-operations.md#FORMAT) → [FAULT](validation-heal-operations.md#FAULT) → [START](validation-heal-operations.md#START) → [WAIT](validation-heal-operations.md#WAIT) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 不同版本分布不同损坏位置，按对象和递归范围修复 +- 所有原 versionId 可寻址且内容正确，最新版本身份不漂移，无多余版本或遗漏历史版本。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## D04: heal.delete-marker-semantics + +Topologies: H1,H2. Branches: current-marker, historical-marker, missing-marker-replica, stale-disk-marker. + +1. 创建三个数据版本与当前/历史 marker，记录 delete_object 返回的 VersionId/DeleteMarker。 +2. 缺少 marker 的实例使用 xl.meta 少数副本缺失；陈旧实例通过成员离线后合法创建/删除 marker 形成，不能直接改任意 metadata。 +3. heal 后 current GET/HEAD 对当前 marker 应为 404 且正确 marker 身份头；指定 marker versionId 的 GET 按 S3 契约为 405 并保留版本头。 +4. LIST versions 精确集合与最新标记一致；显式删除当前 marker 后露出的应是预期历史对象，不能产生额外数据版本。 + +Operations: [SEED_DV](validation-heal-operations.md#SEED_DV) → [S3](validation-heal-operations.md#S3) → [FAULT](validation-heal-operations.md#FAULT) → [DRIVE_GATE](validation-heal-operations.md#DRIVE_GATE) → [START](validation-heal-operations.md#START) → [WAIT](validation-heal-operations.md#WAIT) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 缺失/陈旧 marker 副本后 heal，并按当前值与精确 marker versionId 查询 +- 当前 GET/HEAD 与指定 marker 的状态码及身份头符合契约；marker 不复活数据，LIST versions 无重复。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## D05: heal.versioning-suspended-null + +Topologies: H1,H2. Branches: enabled-to-suspended, null-overwrite, null-delete-marker. + +1. 启用版本写三代，再 put_bucket_versioning Status=Suspended，连续写两次 null 版本并创建/删除 null marker。 +2. 基线同时保存非 null 历史和当前 null 身份；对目标外部 part 或 xl.meta 制造少数缺失。 +3. START deep/WAIT；记录 null 不能被转换成随机 versionId，也不能与启用期历史版本混片。 +4. VERIFY 全部原 versionId、null 内容和 marker；再次开启版本后写新对象，验证旧历史仍可寻址。 + +Operations: [SEED_DV](validation-heal-operations.md#SEED_DV) → [S3](validation-heal-operations.md#S3) → [FORMAT](validation-heal-operations.md#FORMAT) → [FAULT](validation-heal-operations.md#FAULT) → [START](validation-heal-operations.md#START) → [WAIT](validation-heal-operations.md#WAIT) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 在版本开启、暂停、null 版本覆盖之间建夹具再修复 +- null 和非 null 版本不混淆，不新增假版本；暂停前历史对象仍准确可读。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## D06: heal.object-metadata-and-tags + +Topologies: H1,H2. Branches: user-metadata, content-headers, versioned-tags. + +1. 用 DS 设置中文值可编码的 metadata、Content-Type、Content-Encoding、Content-Disposition、Cache-Control 与两组 tags；内容本身使用已声明编码。 +2. 先 GET/HEAD/get_object_tagging 精确记录；对外部 part 和少数 xl.meta 分别故障。 +3. START deep/WAIT，比较所有对外属性和逐版本 tags，修复不新增版本、不丢用户 key 大小写规范化结果。 +4. 允许内部 metadata 合法重写，但必须通过 FORMAT 引用和物理内容证明；不把 Content-Encoding 当服务端压缩已启用。 + +Operations: [SEED_DS](validation-heal-operations.md#SEED_DS) → [BASELINE](validation-heal-operations.md#BASELINE) → [FAULT](validation-heal-operations.md#FAULT) → [START](validation-heal-operations.md#START) → [WAIT](validation-heal-operations.md#WAIT) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 包含用户 metadata、Content-Type/Encoding/Disposition、tags 的对象修复 +- 对外 metadata、tags 和相关响应头逐项相同；不要求无关内部元数据字节完全一致。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## D07: heal.range-head-and-conditions + +Topologies: H1,H2. Branches: head, range-valid, range-outside, if-match, if-none-match, if-modified-since, if-unmodified-since. + +1. DB/DM 基线保存 ETag/LastModified 和跨 part 内容；先执行所有条件请求，冻结预期 200/206/304/412/416。 +2. 对精确 part 制造缺失/坏校验并 repair；同一组 HEAD、Range、IfMatch/IfNoneMatch、时间条件再次请求。 +3. 304/412/416 验证响应无错误成功内容；合法范围比较准确长度、ContentRange 和原片段摘要。 +4. 完整 VERIFY 版本和 metadata；条件请求必须用 API 参数传递，不由本地短路模拟状态码。 + +Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [SEED_DM](validation-heal-operations.md#SEED_DM) → [RANGE](validation-heal-operations.md#RANGE) → [FAULT](validation-heal-operations.md#FAULT) → [START](validation-heal-operations.md#START) → [WAIT](validation-heal-operations.md#WAIT) → [RANGE](validation-heal-operations.md#RANGE) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 修复前后验证 HEAD、合法/越界 Range、If-Match/If-None-Match 及时间条件 +- 200/206/304/412/416 等按逐请求契约判断，绝不返回旧内容、错误范围或错误 ETag。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## D08: heal.encrypted-objects + +Topologies: H1,H2. Branches: sse-s3, sse-kms, sse-c, encrypted-versioned. + +1. DEPENDENCY 先验证候选支持的每种 SSE 与专用 KMS/key 就绪；不支持的产品能力按明确错误契约另记，不能删除该参数实例。 +2. 用 S3 ServerSideEncryption/SSEKMSKeyId 或 SSECustomerAlgorithm/SSECustomerKey 创建对象；密钥从环境注入到内存，不放命令行/清单。 +3. 在密文物理层选择 part 做 missing/bitflip；deep heal 后比较同角色原密文片或匹配候选的重编码验证。 +4. 用正确授权逐版本解密 GET 校验原文，用缺少/错误密钥请求验证拒绝；不得写出明文替代或把密钥错误判成不存在。 + +Operations: [DEPENDENCY](validation-heal-operations.md#DEPENDENCY) → [SEED_DE](validation-heal-operations.md#SEED_DE) → [FORMAT](validation-heal-operations.md#FORMAT) → [FAULT](validation-heal-operations.md#FAULT) → [START](validation-heal-operations.md#START) → [WAIT](validation-heal-operations.md#WAIT) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [S3](validation-heal-operations.md#S3) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 按候选支持的 SSE-S3/SSE-KMS/SSE-C 建加密对象并修复 +- 授权解密后原内容及版本一致，物理验证使用正确编码/密文层；密钥不可用时不误删、不降级明文，证据脱敏。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## D09: heal.object-lock-retention + +Topologies: H1,H2. Branches: governance-retention, compliance-retention, legal-hold, historical-protected-version. + +1. 创建启用 ObjectLock 的专用桶，SEED_DO 使用短且真实未到期的 retention 或 ON legal hold；记录 UTC 截止和独占资源到期安排。 +2. 在无 bypass 权限角色下对受保护 versionId DELETE 必须 AccessDenied，确认保护本来有效。 +3. 对受保护版本缺片/xl.meta 缺失后 repair，核对内容及 get_object_retention/legal_hold；保留期内再次删除仍拒绝。 +4. legal hold 仅正常 API 解除；Compliance 等到真实到期后清理，禁止改时钟或内部删文件；未到期现场标 RETAINED，不能交还空环境。 + +Operations: [SEED_DO](validation-heal-operations.md#SEED_DO) → [S3](validation-heal-operations.md#S3) → [FAULT](validation-heal-operations.md#FAULT) → [START](validation-heal-operations.md#START) → [WAIT](validation-heal-operations.md#WAIT) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) → [RECOVER](validation-heal-operations.md#RECOVER) + +Source assertions: + +- 对开启 retention/legal hold 的受保护版本修复，期间测试非法删除 +- heal 不绕过保留/hold，也不丢失其元数据；拒绝删除与合法修复均有明确证据。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## D10: heal.special-keys-and-pagination + +Topologies: H1,H2. Branches: unicode-space-percent, encoded-slash, adjacent-prefixes, multi-page-list. + +1. DS 创建中文、空格、%2F 字面量、真实 / 和多层前缀键；至少 257 个对象并设置 List MaxKeys=17，确保跨页。 +2. FORMAT 建逻辑 key→物理路径映射，确认 %2F 与 / 不混淆；文件故障只用受限相对路径工具，不拼 shell。 +3. 对选定 key 分别 missing part/xl.meta，使用 URL percent-encode 后的 Admin scope，deep heal。 +4. 分页 LIST/versions 与 manifest 无遗漏/重复；控制前缀不变；对每个特殊 key 精确 GET 和 metadata 校验。 + +Operations: [SEED_DS](validation-heal-operations.md#SEED_DS) → [FORMAT](validation-heal-operations.md#FORMAT) → [FAULT](validation-heal-operations.md#FAULT) → [START](validation-heal-operations.md#START) → [WAIT](validation-heal-operations.md#WAIT) → [S3](validation-heal-operations.md#S3) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 覆盖中文、空格、百分号、编码斜线、多层前缀和多页对象列表 +- 逻辑 key 与物理路径映射准确，不跨桶/前缀；分页无遗漏重复，不能把用户 key 当可执行 shell 或裸文件路径。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## D11: heal.bucket-metadata-integrity + +Topologies: H1,H2. Branches: versioning-config, bucket-policy-tags, lock-config, unknown-record-negative. + +1. DS/DV 创建隔离桶配置：版本、只授权本桶的策略、bucket tags、可到期锁配置；通过 S3 GET 配置存基线。 +2. FORMAT_BAD 白名单仅包含该桶对应的系统 bucket metadata 文件；在匹配格式/可恢复契约已核实后少数副本缺失/截断。 +3. 不能使用通用 heal_fault 修改 .rustfs.sys；按专用系统记录手册在停止服务、精确备份和审批清单下操作，未知格式实例只要求安全拒绝。 +4. repair 后逐 API 读取配置比对，业务对象版本完整；确认 IAM/凭据配置及其他桶系统记录完全未变。 + +Operations: [SEED_DS](validation-heal-operations.md#SEED_DS) → [SEED_DV](validation-heal-operations.md#SEED_DV) → [S3](validation-heal-operations.md#S3) → [FORMAT](validation-heal-operations.md#FORMAT) → [FAULT_FORMAT](validation-heal-operations.md#FAULT_FORMAT) → [START](validation-heal-operations.md#START) → [WAIT](validation-heal-operations.md#WAIT) → [VERIFY](validation-heal-operations.md#VERIFY) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) + +Source assertions: + +- 在隔离桶白名单内测试可支持的桶元数据恢复，并核对外部 bucket 配置 +- 版本设置、策略/标签/保留配置按契约保持；不得遍历凭据配置或把未知内部记录当可重建数据。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## E01: heal.manual-normal-attribution + +Topologies: H1,H2. Branches: normal-missing-data, normal-missing-parity. + +1. 部署 manual 配置：关闭 scanner、auto-heal、MRF；保留 format，记录服务实际环境与 PID，不只保存拟用 env 文件。 +2. FAULT missing 后不能 GET/HEAD 目标；先确认缺失仍在，再 START scanMode=1。 +3. WAIT 精确 token/outcome，先 PHYSICAL 证明目标新生成，再 VERIFY；源归属需结合配置、时间线和任务对象。 +4. 正常扫描未发现静默 bitrot 不能因此判 E01 失败；那是 E02/C02/C03 的 deep 实例。 + +Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [CONFIG](validation-heal-operations.md#CONFIG) → [FAULT](validation-heal-operations.md#FAULT) → [START](validation-heal-operations.md#START) → [WAIT](validation-heal-operations.md#WAIT) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 禁止预读目标，保留 format，隔离背景干扰后显式 normal heal +- 证明本次 Admin token 实际覆盖目标并修复可由 normal 检出的缺失；不要求 normal 发现所有静默 bitrot。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## E02: heal.manual-deep-attribution + +Topologies: H1,H2. Branches: deep-parity-bitrot, deep-data-bitrot. + +1. manual 配置排除其他后台来源；FORMAT 确认分片角色，注入 bitflip 并保存坏片仍存在证据。 +2. 不先读目标，START scanMode=2；保留请求 body、clientToken、incarnation/pool/set 与最终对象 disposition。 +3. 先验证坏片完整恢复，再读取原文；如果任务开始前已有恢复，记录 ATTRIBUTION_NOT_OBSERVED。 +4. 检查健康 control 没被意外修改，source 不能仅由最后一次发出的命令推测。 + +Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [CONFIG](validation-heal-operations.md#CONFIG) → [FORMAT](validation-heal-operations.md#FORMAT) → [FAULT](validation-heal-operations.md#FAULT) → [START](validation-heal-operations.md#START) → [WAIT](validation-heal-operations.md#WAIT) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 在普通读取仍可成功的 parity 损坏上显式 deep heal +- 按 token/对象/版本及物理变更证明 deep 检查和修复，不把后台提前修复算给手工任务。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## E03: heal.read-repair-attribution + +Topologies: H1,H2. Branches: get-missing, range-bitrot, historical-version-read. + +1. 关闭 scanner/auto-heal，按候选 read-repair 实现配置其所需 MRF 队列，记录为何开启；不发送 Admin heal。 +2. FAULT 后物理证明损坏尚存；只对一个精确 key/version 发一次带请求 ID 的 GET 或 Range。 +3. GET 内容先验正确还不够：只观察 PHYSICAL/后台状态/日志，直到该片重建；异步路径需关联 read-repair 入队和责任完成。 +4. 完成补片后才全量 VERIFY；如果只能看到读取成功但找不到 source 证据，该路径仍 NOT_OBSERVED，不能计 PASS。 + +Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [SEED_DV](validation-heal-operations.md#SEED_DV) → [CONFIG](validation-heal-operations.md#CONFIG) → [FAULT](validation-heal-operations.md#FAULT) → [GET](validation-heal-operations.md#GET) → [RANGE](validation-heal-operations.md#RANGE) → [BACKGROUND](validation-heal-operations.md#BACKGROUND) → [LOGS](validation-heal-operations.md#LOGS) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 不发 Admin heal，对已确认的特定损坏对象发一次可追踪 GET/Range +- 读取正确且后续真实补片；若走异步队列需证实 source=read-repair，不能只有 GET 成功。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## E04: heal.scanner-cold-object-repair + +Topologies: H1,H2. Branches: cold-missing, cold-bitrot, scanner-cycle-boundary. + +1. 启用 scanner 和所需自动修复，预注册 cycle/start delay/bitrot cycle 与预算；先用健康冷对象证明扫描器实际巡检。 +2. 注入时用 PROCESS 短停以保持静止窗口；重启后不发目标 GET/HEAD、不发 Admin heal。 +3. 只用磁盘快照、scanner 遍历日志和后台责任观察，证明当前周期检查目标，记录发现→入队→补片。 +4. PHYSICAL 完成后再全量 VERIFY；单纯周期计数增长、全局 idle 或其他自动 replacement 先修不能记 scanner 通过。 + +Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [CONFIG](validation-heal-operations.md#CONFIG) → [PROCESS](validation-heal-operations.md#PROCESS) → [FAULT](validation-heal-operations.md#FAULT) → [BACKGROUND](validation-heal-operations.md#BACKGROUND) → [LOGS](validation-heal-operations.md#LOGS) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 不 GET/HEAD 损坏目标、不发手工 heal，仅等待已启用的 scanner 路径 +- 冷对象被发现并修复，来源和遍历窗口明确；scanner 周期结束但目标没检查不能通过。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## E05: heal.mrf-offline-write-replay + +Topologies: H2. Branches: offline-put, offline-version-delete, offline-marker. + +1. 在 H2 开启 durable MRF，先验证目标节点离线时剩余同 set 满足提交条件；DRIVE_GATE 或 PROCESS 只影响目标成员。 +2. 在线节点执行唯一 key PUT、精确版本 DELETE、marker 操作；逐个保存 ACK 和责任 identity。 +3. 在目标仍离线时采集 journal/状态，证明责任已持久化；恢复同一盘身份，不发手工 heal。 +4. 观察 MRF 重放和退休理由；只有实际 repaired/verified_healthy/authoritatively_absent 后退休；VERIFY 全部 ACK 和目标片，不能以队列空独自证明。 + +Operations: [SEED_DV](validation-heal-operations.md#SEED_DV) → [CONFIG](validation-heal-operations.md#CONFIG) → [DRIVE_GATE](validation-heal-operations.md#DRIVE_GATE) → [WORKLOAD](validation-heal-operations.md#WORKLOAD) → [CHECKPOINT](validation-heal-operations.md#CHECKPOINT) → [BACKGROUND](validation-heal-operations.md#BACKGROUND) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 成员离线时完成有 quorum 的 PUT/DELETE，确认待修责任后重入 +- durable MRF 责任持久且身份精确；只在 repaired/verified healthy/authoritatively absent 等已证明结果后释放，成功提交不丢失。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## E06: heal.replacement-auto-rebuild + +Topologies: H1,H2. Branches: replacement-unversioned, replacement-versioned-marker. + +1. 启用 replacement auto-heal；REPLACE 注入空逻辑盘，旧盘备份对服务不可达；保存新 UUID/slot。 +2. 重启/接纳后不发 Admin heal、不读目标；仅查询 BACKGROUND/REPLACEMENT、日志及 PHYSICAL。 +3. 从 replacement generation、ready 与责任完成状态关联所有 manifest 对象/版本的补片时间线。 +4. 全部目标分片实际重建后 VERIFY；服务 ready、计数重置或队列空都不能提前结束。 + +Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [SEED_DV](validation-heal-operations.md#SEED_DV) → [CONFIG](validation-heal-operations.md#CONFIG) → [REPLACE](validation-heal-operations.md#REPLACE) → [FORMAT](validation-heal-operations.md#FORMAT) → [REPLACEMENT](validation-heal-operations.md#REPLACEMENT) → [BACKGROUND](validation-heal-operations.md#BACKGROUND) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 替换盘接纳后不发 Admin/目标 GET，观察自动 replacement heal +- 替换代次、ready/completion 与盘上全部目标版本一致；不能用服务 ready、队列空或计数重置代替完成。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## F01: heal.root-graceful-coordinator-restart + +Topologies: H1,H2. Branches: root-partial-graceful, same-record-disk-resume. + +1. DX root heal 启动前保存原协调者、拓扑、记录盘和预算；所有 root checkpoint 通过 CHECKPOINT 只读白名单采集。 +2. WINDOW 证明 0<已修目标数<总数 且任务 running，再 PROCESS graceful-stop 原协调者；记录退出信号和持久记录。 +3. 用原 unit/env/数据根/候选重启原协调者，不换记录盘；观察原任务 ID/剩余责任恢复，不能将新扫描当原任务续跑。 +4. 最终全部对象/片正确，root 活动记录退休，重复重启无重复任务；未观察部分进度保留 NOT_OBSERVED。 + +Operations: [SEED_DX](validation-heal-operations.md#SEED_DX) → [FAULT](validation-heal-operations.md#FAULT) → [START](validation-heal-operations.md#START) → [WINDOW](validation-heal-operations.md#WINDOW) → [CHECKPOINT](validation-heal-operations.md#CHECKPOINT) → [PROCESS](validation-heal-operations.md#PROCESS) → [STATUS](validation-heal-operations.md#STATUS) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- root heal 非零进度时优雅重启原协调者，保留原拓扑与记录盘 +- 在明确支持的边界内恢复原 task ID 和剩余责任/预算，完成后无遗留活动记录；不扩张为协调盘丢失后的故障转移。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## F02: heal.target-graceful-restart + +Topologies: H2. Branches: target-partial-graceful, versioned-target-partial. + +1. REPLACE 目标逻辑盘并启用自动重建，使用 DX/DV 使目标总量可观测；协调者不是重启目标。 +2. PHYSICAL 与 REPLACEMENT 双证实目标只完成部分，再 PROCESS 对目标 unit 优雅停止；保存 MainPID、退出码、前后 boot_id。 +3. 保留同一新 disk UUID/代次，正常启动原目标服务；不能换新目录重测。 +4. 观察余下目标继续构建且任务责任一致，全部版本/marker/片完整；备份不得参与重建。 + +Operations: [SEED_DX](validation-heal-operations.md#SEED_DX) → [SEED_DV](validation-heal-operations.md#SEED_DV) → [REPLACE](validation-heal-operations.md#REPLACE) → [REPLACEMENT](validation-heal-operations.md#REPLACEMENT) → [WINDOW](validation-heal-operations.md#WINDOW) → [PROCESS](validation-heal-operations.md#PROCESS) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 替换目标仅完成部分重建时优雅重启目标节点 +- 故障确实发生于部分完成窗口；重启后续建同一目标盘/代次，最终 manifest 与全部目标分片正确。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## F03: heal.target-process-crash + +Topologies: H2. Branches: target-partial-sigkill, target-partial-sigkill-versioned. + +1. 与 F02 同样准备替换盘和可观察部分进度，先冻结将被杀掉的 unit MainPID/进程启动时间。 +2. WINDOW 内 PROCESS kill，精确 systemctl kill --signal=SIGKILL --kill-whom=main；记录 signal=9 与非优雅退出，明确不是物理断电。 +3. 正常启动同 unit/根/新 disk UUID；记录恢复 responsibility/代次，不允许丢弃尚未完成对象。 +4. 全量 VERIFY/PHYSICAL，检查临时文件和持久责任无泄漏；最终通过不抹掉这次真实中断的证据。 + +Operations: [SEED_DX](validation-heal-operations.md#SEED_DX) → [SEED_DV](validation-heal-operations.md#SEED_DV) → [REPLACE](validation-heal-operations.md#REPLACE) → [WINDOW](validation-heal-operations.md#WINDOW) → [PROCESS](validation-heal-operations.md#PROCESS) → [CHECKPOINT](validation-heal-operations.md#CHECKPOINT) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 部分重建时 SIGKILL 测试进程，再正常启动 +- unclean shutdown 与修复进度有真实证据；已提交对象不丢失，未完成目标不会被当完整；明确不是断电测试。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## F04: heal.coordinator-process-crash + +Topologies: H2. Branches: root-coordinator-kill, mrf-coordinator-kill, replacement-coordinator-kill. + +1. 三个机制使用独立场景和冻结的持久化支持表；在协调者非零且未完成进度时采集 CHECKPOINT。 +2. PROCESS SIGKILL 协调者，目标节点保持；恢复原根/原拓扑。 +3. 分别核对机制承诺：持久任务必须续责；易失 token 丢失不能算已完成，应显示未完成并允许明确的新扫描补偿。 +4. 原任务与补偿任务分别保存；最终 VERIFY 所有 ACK、版本和片，不能把安全重扫成功写成原 token 自动恢复。 + +Operations: [SEED_DX](validation-heal-operations.md#SEED_DX) → [SEED_DB](validation-heal-operations.md#SEED_DB) → [START](validation-heal-operations.md#START) → [WINDOW](validation-heal-operations.md#WINDOW) → [CHECKPOINT](validation-heal-operations.md#CHECKPOINT) → [PROCESS](validation-heal-operations.md#PROCESS) → [BACKGROUND](validation-heal-operations.md#BACKGROUND) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 协调者非优雅退出并恢复，分别观察 root、MRF、replacement +- 按各机制已核实的持久化边界判断；不得把丢失的易失 token 当成功，需显式记录剩余责任并可安全重扫恢复。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## F05: heal.full-cluster-restart + +Topologies: H2. Branches: all-graceful-restart, all-restart-with-mpu, repeat-full-restart. + +1. 准备 DB/DV/DM、pending UploadId 与未完成持久修复责任；冻结全部节点/盘/slot。 +2. WINDOW 命中后按固定顺序只停止本例全部 units；保存每节点退出码、checkpoint/MRF 与完整时序。 +3. 按原拓扑同时或预注册顺序重启，等待全部 peer，不能用自动重新格式化或新数据根恢复。 +4. 完成后全版本/marker/MPU 和物理片校验，再重复一次重启验证幂等，任何丢失的已承诺责任计失败。 + +Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [SEED_DV](validation-heal-operations.md#SEED_DV) → [SEED_DM](validation-heal-operations.md#SEED_DM) → [FAULT](validation-heal-operations.md#FAULT) → [START](validation-heal-operations.md#START) → [WINDOW](validation-heal-operations.md#WINDOW) → [PROCESS](validation-heal-operations.md#PROCESS) → [CHECKPOINT](validation-heal-operations.md#CHECKPOINT) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 有未完成修复责任时重启全部测试节点，固定原拓扑/磁盘身份 +- 有持久化保证的责任不遗失；全部端点恢复一致，旧版本/marker/MPU 与目标分片完整，重复重启不扩散错误。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## F06: heal.cancelled-task-restart + +Topologies: H1,H2. Branches: cancel-restart, old-token-after-restart, force-new-after-cancel. + +1. DX 缺片任务在 WINDOW 中 STOP；STATUS 与磁盘静默窗口证明取消和在途退出完成。 +2. 保存旧 token/checkpoint，PROCESS 优雅重启；确认已取消任务不会自行复活或报 repaired。 +3. 新 START/forceStart 获取独立 token，向旧 token 发 stop/status；不得影响新任务。 +4. 新任务完成后 VERIFY/PHYSICAL；取消记录/新任务责任不串代。 + +Operations: [SEED_DX](validation-heal-operations.md#SEED_DX) → [FAULT](validation-heal-operations.md#FAULT) → [START](validation-heal-operations.md#START) → [WINDOW](validation-heal-operations.md#WINDOW) → [STOP](validation-heal-operations.md#STOP) → [CHECKPOINT](validation-heal-operations.md#CHECKPOINT) → [PROCESS](validation-heal-operations.md#PROCESS) → [STATUS](validation-heal-operations.md#STATUS) → [VERIFY](validation-heal-operations.md#VERIFY) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) + +Source assertions: + +- 完成取消及在途退出后重启，随后 forceStart/新任务正常修复 +- 已取消的任务不自动复活；旧 checkpoint/token 不能取消或覆盖新任务，旧责任不会误标为修复完成。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## F07: heal.checkpoint-and-mrf-corruption + +Topologies: H1,H2. Branches: checkpoint-truncate, mrf-checksum, future-schema, stale-generation, journal-write-failure, native-persist-fence. + +实时读取遇到原子替换后链接数为零的旧文件句柄时，先核对文件类型、大小与挂载边界，再拒绝旧句柄字节并返回读取变化。只有原有有界观察循环可以重试；停止后的读取和故障注入仍直接失败，硬链接、非普通文件、超限和挂载逃逸仍被拒绝。原 pidfd、60 秒 advance 预算和完整 checkpoint/resume/root 校验不变。Linux 原子替换组件复现仅证明读取竞争，不证明旧 live 失败的唯一原因，也不替代 RustFS 持久化屏障证据。 + +Execution policy: `evidence-blocked`; disabled by default: `false`. + +Blocked reason (`EVIDENCE_DURABLE_JOURNAL_ORCHESTRATION`): 已实现 H1/H2 checkpoint 截断、未来 schema、真实较早代次回放、committed MRF CRC 故障、私有限额卷的真实 journal ENOSPC 和独立安全重扫；10 项原生回归仅为前置证据。仍缺全部所需 fsync/commit 临界点审核和完整现场分支的独立 live 审计，诊断执行不能提升 canonical 验收。 + +Journal ENOSPC recovery preserves the first failed checkpoint/terminal publication +attempt before restarting the original durable owner after filler release. A new +rescan is permitted only after that same responsibility either completes with +full physical and durable terminal proof or retains an exact failed terminal +without a runnable owner. A still-pending scope remains a failure. Successful +original replay must repair the complete remaining manifest before any target +GET; the later distinct rescan must report zero repairs and preserve original +completion bytes. No forceStart, owner deletion or old-record restoration is +used to bypass overlap admission. This flow still requires fresh live evidence. + +1. 先制造并证明真实 durable root/MRF 责任，CHECKPOINT 定位精确本 run task/record，其他系统记录禁止触碰。 +2. 停止测试服务，按 FAULT_FORMAT 对这些已审核记录分别截断、坏校验、未来 schema、旧代次；写失败实例用独占限额故障层，不塞满真实共享盘。 +3. 重启后核对解析/错误状态：未知或损坏记录保留、可诊断，不能静默删除并 ACK；责任仍能安全重扫处理。 +4. 精确 fsync/commit 崩溃窗口执行 NATIVE 回归，保存屏障命中证据；真机记录损坏与原生临界点不要混算。 + +Operations: [SEED_DV](validation-heal-operations.md#SEED_DV) → [START](validation-heal-operations.md#START) → [CHECKPOINT](validation-heal-operations.md#CHECKPOINT) → [PROCESS](validation-heal-operations.md#PROCESS) → [FAULT_FORMAT](validation-heal-operations.md#FAULT_FORMAT) → [IO_FAULT](validation-heal-operations.md#IO_FAULT) → [BACKGROUND](validation-heal-operations.md#BACKGROUND) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) → [NATIVE](validation-heal-operations.md#NATIVE) + +Source assertions: + +- 受控构造 checkpoint/MRF 截断、坏校验、未来 schema、陈旧代次及写入失败 +- 已确认持久化责任不得静默删除或伪 ACK；未知记录保留且可诊断；精确崩溃窗口由原生屏障证明，不伪称真机命中。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## F08: heal.mixed-version-upgrade-rollback + +Topologies: H2. Branches: rolling-old-new, mixed-peer-status, rollback-new-old, mixed-restart. + +Execution policy: `environment-gated`; disabled by default: `true`. + +Blocked reason (`ENV_IMMUTABLE_RELEASE_PAIR`): 需要两个已固定且相互兼容的 RustFS release、混合版本 peer 状态和经过审核的回滚兼容矩阵；当前 auto-testing Heal 节点没有这组不可变双版本资源。 + +1. 固定两个 release optimized 包的 commit/tree/binary/格式兼容说明；UPGRADE 先验证支持回滚的记录版本，不能使用任意旧包。 +2. 有持久修复责任时逐节点升级，保留数据根和 unit；混合阶段通过各 endpoint STATUS/BACKGROUND 并持续唯一版本写入。 +3. 缺少能力的 peer 必须显示 partial/unsupported，不能 aggregate 成 complete；新记录不被老节点误退休。 +4. 按兼容方案回滚并重启，验证所有 ACK 和责任；不兼容格式只测明确拒绝/保护，禁止冒险降级覆盖。 + +Operations: [SEED_DV](validation-heal-operations.md#SEED_DV) → [START](validation-heal-operations.md#START) → [CHECKPOINT](validation-heal-operations.md#CHECKPOINT) → [UPGRADE](validation-heal-operations.md#UPGRADE) → [WORKLOAD](validation-heal-operations.md#WORKLOAD) → [STATUS](validation-heal-operations.md#STATUS) → [BACKGROUND](validation-heal-operations.md#BACKGROUND) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 固定前后两个明确兼容候选，覆盖滚动升级、混合 peer、回滚与重启 +- 能力不足必须显示 partial/unsupported 而非完整成功；按格式兼容契约保留责任，旧版本不能误退休新版本记录。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## F09: heal.dedicated-host-forced-restart + +Topologies: H6. Branches: dedicated-vm-forced-stop, dedicated-host-power-cycle. + +Execution policy: `environment-gated`; disabled by default: `true`. + +Blocked reason (`ENV_DEDICATED_H6_HOST`): 需要 H6 专用主机、平台强制停启权限、带外恢复和 boot_id/平台事件证据；共享 Heal 节点不能证明真实电源循环。 + +1. 先取得 H6 专项资源 ID、独占证明和带外恢复批准；普通 fixed fleet 禁止执行本例。 +2. 持续限量写入并在 replacement 部分重建窗口，记录已成功 ACK/结果未知写集合与最近持久化证据。 +3. HOST_POWER 使用批准的平台命令强制停启，保留平台活动 ID、时间线和 boot_id；VM stop 与真实电源断电分别命名实例。 +4. 恢复后按原盘/根重启并解析未知写；全版本/片和持久责任验收，不把 SIGKILL 或软件 reboot 当作此项证据。 + +Operations: [SEED_DX](validation-heal-operations.md#SEED_DX) → [SEED_DV](validation-heal-operations.md#SEED_DV) → [WORKLOAD](validation-heal-operations.md#WORKLOAD) → [REPLACE](validation-heal-operations.md#REPLACE) → [WINDOW](validation-heal-operations.md#WINDOW) → [HOST_POWER](validation-heal-operations.md#HOST_POWER) → [CHECKPOINT](validation-heal-operations.md#CHECKPOINT) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 经专项授权在部分修复/持续写入时强制重启独占测试 VM/主机 +- 保留平台事件与 ACK ledger；验证落盘和恢复，不把 SIGKILL 当断电；VM 强制停止也不等同于物理控制器断电。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## G01: heal.race-unversioned-overwrite + +Topologies: H1,H2. Branches: overwrite-vs-deep, overwrite-vs-read-repair, native-commit-cleanup. + +1. 同 key 串行 nonce 的覆盖客户端与 heal 客户端独立；WORKLOAD 每次记录 request-start/end、ACK、payload hash，不凭跨机墙钟判最后版本。 +2. 旧对象缺片/bitflip 后两端同时运行，WINDOW 证明实际交叠；每个 worker 的同 key PUT 必须串行，避免期望值本身无序。 +3. 停止写入后解析全部未知 ACK，按最后可证明提交核对最新 GET 与 data_dir；不能回滚旧内容。 +4. 运行 NATIVE commit/cleanup 屏障实例；真实竞争没命中精确窗口只算黑盒实例，记录 no-lock/锁重验日志。 + +Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [FAULT](validation-heal-operations.md#FAULT) → [WORKLOAD](validation-heal-operations.md#WORKLOAD) → [START](validation-heal-operations.md#START) → [WINDOW](validation-heal-operations.md#WINDOW) → [VERIFY](validation-heal-operations.md#VERIFY) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [NATIVE](validation-heal-operations.md#NATIVE) + +Source assertions: + +- 同 key 连续覆盖与 heal 同时推进，包含提交尾部清理窗口 +- 按可证明的提交顺序验证最终最后一次成功写入；不能回滚到旧 data_dir，未确认超时写入单独解析。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## G02: heal.race-versioned-put + +Topologies: H1,H2. Branches: new-version-vs-historical-heal, read-all-versions-during-heal, native-version-fence. + +1. DV 历史版本缺片时，前台串行写入独立 payload 的新版本，保存每个成功 versionId。 +2. START deep 与 WORKLOAD 交叠；读取端始终指定 versionId 验证新旧内容，禁止用 current GET 代替。 +3. 静默后按完整 ACK ledger 重建期望版本集合，最新值根据串行提交顺序判断；全量 PHYSICAL 检查不同 data_dir 无混片。 +4. 精确版本提交屏障运行 NATIVE；任何未解析超时写/缺失 ACK 阻止最终通过。 + +Operations: [SEED_DV](validation-heal-operations.md#SEED_DV) → [FAULT](validation-heal-operations.md#FAULT) → [WORKLOAD](validation-heal-operations.md#WORKLOAD) → [START](validation-heal-operations.md#START) → [WINDOW](validation-heal-operations.md#WINDOW) → [VERIFY](validation-heal-operations.md#VERIFY) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [NATIVE](validation-heal-operations.md#NATIVE) + +Source assertions: + +- 修复历史版本时创建新版本并重复读取各 versionId +- 新旧版本不混片、不串内容；已返回成功的每个版本都保留，最新值按确定的先后关系验收。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## G03: heal.race-version-delete + +Topologies: H1,H2. Branches: delete-old-version-vs-deep, delete-current-version-vs-deep, native-delete-fence. + +1. 选 DV 的精确 versionId 制造可恢复缺片；前台删除请求明确带该 ID，不是只创建 marker。 +2. WINDOW 内并发 delete_object 与 deep heal，保存 DELETE 成功回执和 heal scope/identity。 +3. 终态按精确 version GET 应不存在，LIST versions 不得复活；其他所有版本内容保持。 +4. 运行 NATIVE 删除/修复提交交错点；清理旧片仅作为最终物理证据，不能先用备份复活已删版本。 + +Operations: [SEED_DV](validation-heal-operations.md#SEED_DV) → [FAULT](validation-heal-operations.md#FAULT) → [START](validation-heal-operations.md#START) → [WORKLOAD](validation-heal-operations.md#WORKLOAD) → [WINDOW](validation-heal-operations.md#WINDOW) → [S3](validation-heal-operations.md#S3) → [VERIFY](validation-heal-operations.md#VERIFY) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [NATIVE](validation-heal-operations.md#NATIVE) + +Source assertions: + +- 对损坏的精确历史 versionId 发 DELETE，与 deep heal 竞争 +- 已确认删除的版本不能被复活，其他历史版本不受影响；原生临界点测试与真机竞争分别留证。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## G04: heal.race-markers-and-batch-delete + +Topologies: H1,H2. Branches: create-marker, delete-marker, batch-version-delete, unversioned-delete, native-marker-fence. + +1. 分别建立版本化和非版本化桶；marker创建/删除、批量带versionId删除和无版本DELETE分开运行，不混淆接口语义。 +2. 在 xl.meta 少数缺失下启动 heal，同时提交冻结的删除列表；delete_objects 必须逐项检查 Deleted/Errors，HTTP200不代表全成功。 +3. 按逐项 ACK 计算最终数据版本、marker 和最新值集合；精确 GET验证不存在项和存留项。 +4. native-marker-fence 独立取证；恢复故障不得把已确认删除的旧 metadata 覆盖回来。 + +Operations: [SEED_DV](validation-heal-operations.md#SEED_DV) → [SEED_DB](validation-heal-operations.md#SEED_DB) → [FAULT](validation-heal-operations.md#FAULT) → [START](validation-heal-operations.md#START) → [WORKLOAD](validation-heal-operations.md#WORKLOAD) → [S3](validation-heal-operations.md#S3) → [VERIFY](validation-heal-operations.md#VERIFY) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [NATIVE](validation-heal-operations.md#NATIVE) + +Source assertions: + +- 与创建/删除 marker、批量版本删除和无版本删除分别交错 +- 按逐项 DELETE 回执核对版本/marker 集合，无复活、误删或遗漏批量错误。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## G05: heal.race-multipart-complete-abort + +Topologies: H1,H2. Branches: upload-continue, complete, abort, native-mpu-fence. + +1. DM 同时准备已提交对象与两个可识别的 pending UploadId，各自 part/ETag 独立。 +2. 已提交对象缺片并 heal；并发给 upload-A 续传并 Complete，upload-B Abort，WINDOW 保存实际交叠。 +3. Complete 返回版本只出现一次，完整内容为按PartNumber排序的ACK part；Abort 的 UploadId 不再可list_parts，不能误伤另一个upload或已提交对象。 +4. 核对所有 MPU 列表和物理目录；原生提交/取消屏障另用 NATIVE，不允许仅发出并发请求即声称命中。 + +Operations: [SEED_DM](validation-heal-operations.md#SEED_DM) → [FAULT](validation-heal-operations.md#FAULT) → [START](validation-heal-operations.md#START) → [WORKLOAD](validation-heal-operations.md#WORKLOAD) → [WINDOW](validation-heal-operations.md#WINDOW) → [S3](validation-heal-operations.md#S3) → [VERIFY](validation-heal-operations.md#VERIFY) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [NATIVE](validation-heal-operations.md#NATIVE) + +Source assertions: + +- heal 期间对不同已知 UploadId 分别续传、Complete、Abort +- 成功 Complete 内容只出现一次；Abort 不破坏已提交对象或别的 upload；未完成 part 不被伪造为完整对象。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## G06: heal.race-copy-and-metadata-update + +Topologies: H1,H2. Branches: copy-source, copy-destination, replace-metadata, update-tags, native-copy-fence. + +1. DS 源/目标使用不同内容和 metadata，copy 请求 CopySource 使用结构化 Bucket/Key/VersionId，不拼路径字符串。 +2. 源或目标缺片并 heal，同窗口 COPY、MetadataDirective=REPLACE 的自拷贝、put_object_tagging 分别运行。 +3. 保存对应提交ETag/versionId及更新顺序，验证最终内容与该提交 metadata/tags 组合一致；旧 read-repair 必须重验代次。 +4. 检查源目标和control均无交叉覆盖；NATIVE 记录锁升级/陈旧提交拒绝的精确窗口。 + +Operations: [SEED_DS](validation-heal-operations.md#SEED_DS) → [FAULT](validation-heal-operations.md#FAULT) → [START](validation-heal-operations.md#START) → [WORKLOAD](validation-heal-operations.md#WORKLOAD) → [WINDOW](validation-heal-operations.md#WINDOW) → [S3](validation-heal-operations.md#S3) → [VERIFY](validation-heal-operations.md#VERIFY) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [NATIVE](validation-heal-operations.md#NATIVE) + +Source assertions: + +- COPY 源/目标和标签/元数据更新与修复交错 +- 最终对象内容与对应提交的 metadata 一致，读修复发现新代次应重验/放弃陈旧提交而非覆盖。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## G07: heal.race-ilm-expiry-and-transition + +Topologies: H2. Branches: current-expiry, noncurrent-expiry, transition, restore, native-ilm-fence. + +Execution policy: `environment-gated`; disabled by default: `true`. + +Blocked reason (`ENV_DEDICATED_ILM_TIER_WINDOW`): 需要隔离 ILM/tier 租户、真实 Expire/Transition/Restore 窗口和远端版本映射；共享 ILM 环境不能安全制造交叠窗口。 + +1. 用现有 ILM 专用远端 tier 能力建立本例桶和规则；记录真实到期窗口/远端version映射，不在共享桶上改规则。 +2. 在实际出现 Expire/Transition/Restore 进度时启动缺片 heal，WINDOW 必须从日志/状态证明交叠；提前完成只能记 NOT_OBSERVED。 +3. 过期 ACK/ILM 删除证明决定哪些版本应不存在，禁止 heal 复活；迁移对象通过真实远端 GET/restore 验证原版本内容。 +4. 本地物理与远端引用双重校验；源端 GET 成功不足，清理只删除本例规则/远端数据。 + +Operations: [SEED_DV](validation-heal-operations.md#SEED_DV) → [SEED_DE](validation-heal-operations.md#SEED_DE) → [DEPENDENCY](validation-heal-operations.md#DEPENDENCY) → [ILM](validation-heal-operations.md#ILM) → [FAULT](validation-heal-operations.md#FAULT) → [START](validation-heal-operations.md#START) → [WINDOW](validation-heal-operations.md#WINDOW) → [S3](validation-heal-operations.md#S3) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) → [NATIVE](validation-heal-operations.md#NATIVE) + +Source assertions: + +- 分别与当前/历史版本过期、远端 transition/restore 同窗口执行 +- 已授权过期不被恢复；迁移状态与远端版本引用正确；只有实际观察到交叠才计并发覆盖，tier 未配置需明确阻断对应实例。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## G08: heal.race-replication-resync + +Topologies: H2. Branches: replicate-put, replicate-delete, resync, native-replication-fence. + +Execution policy: `environment-gated`; disabled by default: `true`. + +Blocked reason (`ENV_ISOLATED_REPLICATION_PAIR`): 需要隔离的复制两端、resync 控制、跨端版本映射和 Heal/resync 实际交叠窗口；当前固定 fleet 没有独占复制对。 + +1. 准备两个隔离复制端点/本例桶、明确源目标版本映射和同步规则；先跑一份健康复制控制。 +2. 源端缺片 heal 同时提交 PUT/DELETE 并启动指定 resync；保存两侧日志与跨端版本映射，证明操作时间交叠。 +3. 分别从两端全量列举版本、marker、内容并比对复制契约；源端正确但远端缺失/无限重试必须失败。 +4. 恢复临时规则/凭据，确认无无限循环/重复传播；精确交错仅由匹配 NATIVE 证据补充。 + +Operations: [SEED_DV](validation-heal-operations.md#SEED_DV) → [DEPENDENCY](validation-heal-operations.md#DEPENDENCY) → [REPLICATION](validation-heal-operations.md#REPLICATION) → [FAULT](validation-heal-operations.md#FAULT) → [START](validation-heal-operations.md#START) → [WORKLOAD](validation-heal-operations.md#WORKLOAD) → [WINDOW](validation-heal-operations.md#WINDOW) → [VERIFY](validation-heal-operations.md#VERIFY) → [LOGS](validation-heal-operations.md#LOGS) → [NATIVE](validation-heal-operations.md#NATIVE) + +Source assertions: + +- 在隔离复制两端与 heal 同时运行新增/删除/重同步 +- 对端版本、删除语义及源数据最终一致，无无限回环或重复传播；不得依赖仅源端 GET 宣称复制正确。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## H01: heal.with-expansion-rebalance + +Topologies: H5. Branches: expand-while-data-heal, expand-while-parity-heal, versioned-rebalance. + +Execution policy: `environment-gated`; disabled by default: `true`. + +Blocked reason (`ENV_H5_EXPANSION_REBALANCE`): 需要 H5 pool 扩容、rebalance 和 Heal 同时活动的完整拓扑，并逐对象保存 pool/set 迁移证据；当前 auto-testing 未绑定扩容前后布局与专用操作窗口。 + +1. 使用已批准 H5，先在源 pool 写 DB/DV 并以 FORMAT/PHYSICAL 证明位置；新 pool 为空且未纳入旧选择器。 +2. 目标 data/parity 缺失或bitflip后按 POOL 手册扩容重启，确认新拓扑；启动 rebalance 与 heal。 +3. WINDOW 保存两任务活动证据，逐对象跟踪源/目标 pool/set/版本身份，不能只看全局数量。 +4. 两任务均正常完成后物理证明目标权威副本，VERIFY全版本，保留scanner backlog的持久提交/退休证据。 + +Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [SEED_DV](validation-heal-operations.md#SEED_DV) → [FORMAT](validation-heal-operations.md#FORMAT) → [FAULT](validation-heal-operations.md#FAULT) → [POOL](validation-heal-operations.md#POOL) → [START](validation-heal-operations.md#START) → [WINDOW](validation-heal-operations.md#WINDOW) → [BACKGROUND](validation-heal-operations.md#BACKGROUND) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 先证明源 pool 持有夹具，再扩容并与修复同时 rebalance +- 修复目标不得指向错误 pool/set；新旧数据完整、移动终态正确，保留 scanner backlog 提交权威检查。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## H02: heal.with-rebalance-stop-resume + +Topologies: H5. Branches: rebalance-stop, rebalance-resume, rebalance-service-restart. + +Execution policy: `environment-gated`; disabled by default: `true`. + +Blocked reason (`ENV_H5_REBALANCE_LIFECYCLE`): 需要可观测的 H5 rebalance、停止/恢复/服务重启控制及独立 task 身份；当前 auto-testing 未绑定这两类任务的独立身份和操作窗口。 + +1. H5 中使用足够对象使移动可观测，缺片 heal 与 rebalance 同时活动；记录两类独立 task ID。 +2. 只停止 rebalance，观察 heal 不被误取消；然后恢复 rebalance，重启实例仅重启本例 unit。 +3. 持续唯一 key 写入并保存 ACK；停止/恢复前后物理和任务状态要连续，不用新数据根替代恢复。 +4. 最终两类任务各自满足完成契约、对象/版本及移动目标正确，重复停止不影响新代次。 + +Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [POOL](validation-heal-operations.md#POOL) → [FAULT](validation-heal-operations.md#FAULT) → [START](validation-heal-operations.md#START) → [WINDOW](validation-heal-operations.md#WINDOW) → [PROCESS](validation-heal-operations.md#PROCESS) → [WORKLOAD](validation-heal-operations.md#WORKLOAD) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- heal 活动期间 stop/restart/resume rebalance +- 两类任务状态和身份独立；停止/恢复移动不误终止修复或丢失对象，恢复后两者均满足各自契约。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## H03: heal.with-decommission + +Topologies: H5. Branches: heal-decommission-source, heal-survivor-target, versioned-multipart. + +Execution policy: `environment-gated`; disabled by default: `true`. + +Blocked reason (`ENV_H5_DECOMMISSION`): 需要 H5 pool decommission、survivor 目标、写入限制和移动 ACK/物理权威证据；当前 auto-testing 未绑定本例退役前后拓扑和恢复方案。 + +1. 先证实夹具位于待退役源，另写 survivor control；分别在源和目标制造在quorum内可恢复故障。 +2. 按既有 Pool 行为启动退役与 heal；退役期间按当前与 MinIO 一致的写入限制检查拒绝，不能把拒绝写当数据丢失。 +3. 源数据只能在目标提交权威后删除；记录移动 ACK 与 target physical，而不是临时副本数量。 +4. 退役完成后停止/移除源，只访问 survivor 全版本/MPU/内容；若还依赖源兜底必失败。 + +Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [SEED_DV](validation-heal-operations.md#SEED_DV) → [SEED_DM](validation-heal-operations.md#SEED_DM) → [POOL](validation-heal-operations.md#POOL) → [FAULT](validation-heal-operations.md#FAULT) → [START](validation-heal-operations.md#START) → [WINDOW](validation-heal-operations.md#WINDOW) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 分别在待退役源和存活目标制造可恢复缺片，再执行退役 +- 不得提前删源或以临时副本冒充迁移成功；按 Pool 既定写入限制验收；只剩 survivor 后仍完整可读且有物理证明。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## H04: heal.with-decommission-cancel-clear + +Topologies: H5. Branches: cancel-during-heal, clear-after-inflight-exit, resume-new-generation, capacity-lock-retry-window. + +Execution policy: `environment-gated`; disabled by default: `true`. + +Blocked reason (`ENV_H5_DECOMMISSION_CANCEL`): 需要退役取消/清理/新代次恢复以及 #2423 所需容量锁重试窗口；普通取消结果不能替代这个临界点。 + +1. 在移动和 heal 都有部分进度时取消退役；保存取消持久化记录、通知和在途退出顺序。 +2. 确认已退出后 Clear，再恢复/新建退役；旧 token/代次不得误取消新任务，持续 ACK ledger 不能丢。 +3. 容量锁超时实例需要 LOCK_WINDOW 或匹配 NATIVE 屏障，实际见证 #2423 所涉重试路径；普通取消成功不能代替。 +4. 两流程收敛后 VERIFY/PHYSICAL；没有命中锁窗口单列 NOT_OBSERVED，不声称覆盖 #2423 的临界点。 + +Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [POOL](validation-heal-operations.md#POOL) → [FAULT](validation-heal-operations.md#FAULT) → [START](validation-heal-operations.md#START) → [WINDOW](validation-heal-operations.md#WINDOW) → [WORKLOAD](validation-heal-operations.md#WORKLOAD) → [LOCK_WINDOW](validation-heal-operations.md#LOCK_WINDOW) → [CHECKPOINT](validation-heal-operations.md#CHECKPOINT) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) → [NATIVE](validation-heal-operations.md#NATIVE) + +Source assertions: + +- heal/移动争用中取消退役、Clear、恢复，并覆盖容量锁超时恢复 +- 保持持久化后通知及在途退出顺序；不误取消新代次；对 #2423 的重试窗口单独注明真机是否实际触发，不只看最终绿色。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## H05: heal.after-retired-pool-removal + +Topologies: H5. Branches: survivor-missing-part, survivor-missing-metadata, retired-source-offline. + +Execution policy: `environment-gated`; disabled by default: `true`. + +Blocked reason (`ENV_H5_RETIRED_POOL_REMOVAL`): 需要完整迁移后移除 retired pool、源保持离线和 survivor-only 访问；当前 auto-testing 未提供本例源持续离线和 survivor-only 的绑定证据。 + +1. 依既有 Pool 全量迁移并完成退休配置移除，停止源服务，证明入口仅绑定 survivor；保留源不可达证据。 +2. 只在 survivor 的选定对象/版本制造缺片或metadata缺失；START root或正确pool/set deep。 +3. 记录 source authority 选择，不能向已退休 pool 写入，也不能无权威覆盖/错误回退（#2422）。 +4. 检查survivor实际补片、全部版本/marker和源根未变化；在源持续停机情况下全量 VERIFY。 + +Operations: [SEED_DV](validation-heal-operations.md#SEED_DV) → [POOL](validation-heal-operations.md#POOL) → [PROCESS](validation-heal-operations.md#PROCESS) → [FAULT](validation-heal-operations.md#FAULT) → [START](validation-heal-operations.md#START) → [WAIT](validation-heal-operations.md#WAIT) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) → [LOGS](validation-heal-operations.md#LOGS) + +Source assertions: + +- 源 pool 完全退役移除并停机后，只在 survivor 上触发修复 +- 不再向已移除源写入或依赖其兜底；来源权威和版本正确；#2422 路径不能出现无权威覆盖或错误回退。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## H06: heal.multi-pool-target-isolation + +Topologies: H5. Branches: selected-pool, selected-set, unselected-control-fault. + +1. 在两个以上 pool/set分别放不同对象，用实际layout确认归属；健康control与控制故障分开登记。 +2. 只选一个pool/set START；其他pool保留缺片控制，不先GET它们。 +3. PHYSICAL先确认选中目标补齐，未选故障原样保留；任务、容量、配额与drive身份都指向正确目标。 +4. 取证后才恢复控制故障和VERIFY；对象在其他pool可读不能证明所选盘完成修复。 + +Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [SEED_DV](validation-heal-operations.md#SEED_DV) → [FORMAT](validation-heal-operations.md#FORMAT) → [FAULT](validation-heal-operations.md#FAULT) → [START](validation-heal-operations.md#START) → [WAIT](validation-heal-operations.md#WAIT) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [RECOVER](validation-heal-operations.md#RECOVER) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 仅破坏选定 pool/set，并在另一 pool 保留不同对象与控制故障 +- 选择器、任务、容量/配额和目标 drive 一致；未选 pool 不受错误修复，跨 pool 存在不等于该目标已修复。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## H07: heal.ec84-multi-set-restart + +Topologies: H3,H4. Branches: ec84-one-set-restart, ec84-one-set-crash, ec84-two-sets-restart, ec84-two-sets-crash. + +Execution policy: `environment-gated`; disabled by default: `true`. + +Blocked reason (`ENV_EC84_MULTI_SET_TOPOLOGY`): 需要 H3 的 12 个独立盘和 H4 的至少 24 个独立 endpoint，并按 set 分别记录重启/崩溃窗口；当前 H4 资源不可用。 + +1. 分别绑定 H3 12盘和 H4 至少24盘，FORMAT必须证实每set为8data+4parity；同一物理设备多个目录不能算独立故障盘。 +2. 每个set都准备DB/DV/DX并证明对象归属，REPLACE选定目标，在目标仅部分重建时优雅重启或SIGKILL。 +3. 保存实际中断窗口、盘代次、每set已修/未修对象，不能跨set合并片数。 +4. 每个set单独完成物理与全版本校验；四盘pilot不进入该项分子，native EC8+4回归作为独立补证。 + +Operations: [SEED_DX](validation-heal-operations.md#SEED_DX) → [SEED_DV](validation-heal-operations.md#SEED_DV) → [FORMAT](validation-heal-operations.md#FORMAT) → [REPLACE](validation-heal-operations.md#REPLACE) → [WINDOW](validation-heal-operations.md#WINDOW) → [PROCESS](validation-heal-operations.md#PROCESS) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) → [NATIVE](validation-heal-operations.md#NATIVE) + +Source assertions: + +- 固定 EC8+4 单 set 与至少两 set，部分重建时重启/崩溃目标 +- 每个 set 的故障覆盖和重建分别证明，不跨 set 拼 shard；必须满足真实盘数及映射，不能用 4 盘用例替代 EC8+4。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## I01: heal.slow-io-and-eio + +Topologies: H1,H2. Branches: bounded-latency, short-read, eio. + +Execution policy: `environment-gated`; disabled by default: `true`. + +Blocked reason (`ENV_DEDICATED_FAULT_IO`): 需要专用 faultfs 或故障设备、可恢复的 slow/short-read/EIO 注入和 namespace 隔离；共享盘模拟会污染其他运行。 + +1. IO_FAULT 只在预分配专用故障设备或受限faultfs，冻结延迟/错误率、IO范围、最长窗口和原状恢复命令。 +2. 在服务相同namespace/UID先证明迟延/短读/EIO真实生效，再启动目标缺片heal；不得让dd填满共享盘。 +3. 错误窗口记录重试、CPU、队列和物理片，不能提前reported healthy；暂态与永久错误分开记录。 +4. 撤销故障后重新观察责任完成并VERIFY；故障层证据缺失算INFRA_ERROR，不把模拟EIO说成真实介质损坏。 + +Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [SEED_DX](validation-heal-operations.md#SEED_DX) → [IO_FAULT](validation-heal-operations.md#IO_FAULT) → [FAULT](validation-heal-operations.md#FAULT) → [START](validation-heal-operations.md#START) → [BACKGROUND](validation-heal-operations.md#BACKGROUND) → [METRICS](validation-heal-operations.md#METRICS) → [RECOVER](validation-heal-operations.md#RECOVER) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 在独占故障层注入有界延迟、短读、EIO，解除后观察恢复 +- 暂态与永久错误分开，重试有界且无忙循环；解除故障后实际修复，期间不得提前标健康。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## I02: heal.enospc-inodes-and-quota + +Topologies: H1,H2. Branches: bounded-volume-enospc, bounded-inode-exhaustion, bucket-quota. + +1. IO_FAULT使用有容量上限的独立loop/fault卷，预注册字节/inode限制及本例桶配额，不改变共享XFS全局配额。 +2. 先证明服务实际写返回ENOSPC或quota错误；准备缺片目标和DM写入，再触发heal/PUT，逐次保留ACK和错误。 +3. 不可写期间不得损坏健康来源、虚报补片或释放仍需容量；元数据/part双重快照。 +4. 释放本例占用或提高已批准限额后续修，VERIFY全内容和目标片；共享盘可用空间保持安全余量。 + +Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [SEED_DM](validation-heal-operations.md#SEED_DM) → [IO_FAULT](validation-heal-operations.md#IO_FAULT) → [QUOTA](validation-heal-operations.md#QUOTA) → [FAULT](validation-heal-operations.md#FAULT) → [START](validation-heal-operations.md#START) → [WORKLOAD](validation-heal-operations.md#WORKLOAD) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [RECOVER](validation-heal-operations.md#RECOVER) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 限制专用测试卷空间/inode，并参数化本次桶配额 +- 仅填专用受限卷、不耗尽共享真实盘；失败不破坏好副本或释放尚需容量，释放空间后正确续修。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## I03: heal.network-partition-and-lock-quorum + +Topologies: H2. Branches: minority-partition, read-boundary, write-boundary, lock-boundary, rejoin. + +候选支持与 B06 相同，但 storageclass 与锁策略分别按精确候选绑定。`1c7fc81` 的锁客户端范围、quorum 阈值及 EC:8 配置已复核；其 RPC 控制通道工厂有变化，必须重新取得真实双向网络探针、admission、partial coverage 和 rejoin 证据，不能由源码等价或 B07 通过推定本例通过。B08 的候选范围保持独立。 + +The adapter additionally binds exact source `62ff96774fa4139d45f14d70b70519aae8070ac0` and its typed RPC `Unavailable` response mapping to `503/ServiceUnavailable`. Every referenced lock, EC, multipart and delete-recovery source file remains independently checked against that committed source; neighboring or abbreviated revisions are rejected. This source requires its matching immutable package and source-bound inspector before live execution. All five original branches, complete DB/DV/DL fixtures, refused-mutation accounting, physical-before-GET proofs, task responsibility and cleanup requirements remain mandatory; the source admission alone does not establish live acceptance. + +1. H2 先冻结实际节点/slot/对象R/W和锁quorum，NETWORK规则只匹配本例服务端口并带run ID、超时恢复。 +2. 构造少数派及边界分区，双端探针证实隔离；SSH、跳板机和其他服务端口保持连通。 +3. 双方执行带nonce的PUT/GET/heal，少数派不能承诺不安全提交；BACKGROUND必须暴露missing peer/partial coverage。 +4. 解除规则并检查规则计数归零，按ACK因果对账全版本/内容及目标片，无分叉、旧代覆盖或丢失责任。 + +Operations: [SEED_DV](validation-heal-operations.md#SEED_DV) → [FORMAT](validation-heal-operations.md#FORMAT) → [NETWORK](validation-heal-operations.md#NETWORK) → [WORKLOAD](validation-heal-operations.md#WORKLOAD) → [START](validation-heal-operations.md#START) → [BACKGROUND](validation-heal-operations.md#BACKGROUND) → [METRICS](validation-heal-operations.md#METRICS) → [RECOVER](validation-heal-operations.md#RECOVER) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 仅隔离测试服务节点间流量，覆盖少数派、读/写/锁 quorum 边界及恢复 +- 少数派不得承诺无安全性的修复或写入；cluster coverage 不完整不能显示全局健康；恢复无分叉、跨代覆盖。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## I04: heal.queue-pressure-and-admission + +Topologies: H1,H2. Branches: full-queue, dropped-admission, merged-duplicates, cross-bucket-fairness. + +1. 专用服务冻结队列容量、并发、最大请求数和时限；DX与多桶夹具通过WORKLOAD施加有上限的Admin/read-repair请求。 +2. 分别记录Full/Dropped/Merged的真实返回及任务ID，重复token/合并不按独立修复数计分。 +3. 持久责任不能因入队失败被退休；记录journal、队列深度、等待时延和资源曲线，超过预算立即停注入。 +4. 停止负载后在deadline内收敛，所有可修目标真实补齐；不能只以CLI exit0或全局idle通过。 + +Operations: [SEED_DX](validation-heal-operations.md#SEED_DX) → [CONFIG](validation-heal-operations.md#CONFIG) → [WORKLOAD](validation-heal-operations.md#WORKLOAD) → [START](validation-heal-operations.md#START) → [BACKGROUND](validation-heal-operations.md#BACKGROUND) → [CHECKPOINT](validation-heal-operations.md#CHECKPOINT) → [METRICS](validation-heal-operations.md#METRICS) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 多来源请求冲击有限队列，混合重复任务、不同优先级和多个桶 +- Full/Dropped/Merged 语义与实际任务一致；丢弃不能丢 durable 责任；在预先声明负载窗口内无无限堆积或健康误报。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## I05: heal.lock-timeout-and-caller-abort + +Topologies: H1,H2. Branches: object-lock-timeout, caller-disconnect, stale-read-repair, native-lock-owner-fence. + +Execution policy: `evidence-blocked`; disabled by default: `false`. + +Blocked reason (`NATIVE_LOCK_OWNER_ABORT_BARRIER`): 已有同候选源码的 lock owner、取消物理 unlink 与 stale read-repair 原生子集；H1/H2 timeout 与 queued-successor 诊断均已通过独立 live 审计（39767336741/1、39817606025/1），包括原始 ACK、历史代次、真实锁事件、各 peer inventory、停机后新分片完整性和有界 FD 收尾。仍缺实际断连边界的精确所有权、完整 native timeout/stale 屏障、stale-generation controller，以及 H1/H2 DL/DV 完整对象/ACK/物理/锁/FD 闭环。精确源码 cfg(test) 原生证据独立保留；诊断子集和原生子集不能充作完整 canonical PASS。 + +1. LOCK_WINDOW用本例慢IO/受控原生锁屏障持有目标对象锁，先证明真实争用超过预注册预算。 +2. 同时heal/覆盖或删除；调用方断开实例仅断本次TCP连接，不杀服务器或关闭跳板机；记录请求开始/断开/服务在途结束。 +3. 锁释放后观察取消/超时不提前释放所有权、不重复提交，陈旧read-repair必须重验版本。 +4. 核对所有ACK和最终代次，检查锁/FD无泄漏；黑盒不能证明锁所有权时以NATIVE窗口独立补证，不能猜命中。 + +Operations: [SEED_DV](validation-heal-operations.md#SEED_DV) → [IO_FAULT](validation-heal-operations.md#IO_FAULT) → [LOCK_WINDOW](validation-heal-operations.md#LOCK_WINDOW) → [WORKLOAD](validation-heal-operations.md#WORKLOAD) → [START](validation-heal-operations.md#START) → [WINDOW](validation-heal-operations.md#WINDOW) → [METRICS](validation-heal-operations.md#METRICS) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) → [NATIVE](validation-heal-operations.md#NATIVE) + +Source assertions: + +- 真实对象锁争用超过预算及调用方断开，再释放锁恢复 +- 取消/超时不得提前释放在途所有权或重复提交；陈旧 read-repair 必须重验代次；恢复后无泄漏和错误覆盖。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## I06: heal.kms-and-tier-outage + +Topologies: H2. Branches: kms-unavailable, kms-denied, tier-unavailable, tier-version-unavailable. + +Execution policy: `environment-gated`; disabled by default: `true`. + +Blocked reason (`ENV_DEDICATED_KMS_TIER_TENANT`): 需要专用 KMS/tier 租户、依赖网络隔离和安全的 key/tier 版本证据；不能在共享依赖上全局断网。 + +1. 先在专用KMS/tier租户验证DE/DV夹具可正常解密/restore，保存非敏感key/tier版本标识。 +2. DEPENDENCY仅断本测试客户端/服务到该依赖的专用namespace路由，或撤销本例租户权限；不全局停共享KMS。 +3. 发GET/restore/deep heal并观察责任，依赖错误不能判对象不存在、可remove、已修好；密钥不进入错误日志。 +4. 恢复依赖和权限后原版本可解密/恢复，物理/责任完整；远端对象不存在与远端暂不可达分别判定。 + +Operations: [SEED_DE](validation-heal-operations.md#SEED_DE) → [SEED_DV](validation-heal-operations.md#SEED_DV) → [DEPENDENCY](validation-heal-operations.md#DEPENDENCY) → [NETWORK](validation-heal-operations.md#NETWORK) → [S3](validation-heal-operations.md#S3) → [START](validation-heal-operations.md#START) → [BACKGROUND](validation-heal-operations.md#BACKGROUND) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 仅让已分配测试 KMS/远端 tier 暂不可用，持续查询与修复 +- 依赖故障不得解释为对象不存在/可删/已修复，不暴露密钥；恢复后原版本可解密或恢复，责任未丢失。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## J01: heal.status-partial-disabled-unknown + +Topologies: H1,H2. Branches: enabled-idle, disabled, uninitialized, missing-peer, recovered-complete. + +1. 在独立服务配置分别观察 enabled无任务、HEAL禁用、初始化未完成，保留实际启动配置及时间线。 +2. H2 PROCESS停一个peer并跨节点查询BACKGROUND/REPLACEMENT；完整coverage必须含expected/responded/unknown明细。 +3. disabled/uninitialized/partial/degraded不能被rc或聚合器解释成全局完成；收集rc原输出与raw Admin逐字段对照。 +4. 恢复所有peer后才允许完整idle，依然用PHYSICAL/manifest核对无遗留责任；未观测初始化窗口单独记NOT_OBSERVED。 + +Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [CONFIG](validation-heal-operations.md#CONFIG) → [PROCESS](validation-heal-operations.md#PROCESS) → [BACKGROUND](validation-heal-operations.md#BACKGROUND) → [REPLACEMENT](validation-heal-operations.md#REPLACEMENT) → [RC_STATUS](validation-heal-operations.md#RC_STATUS) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 分别检查 enabled/disabled/uninitialized、全体可达及缺失 peer 的状态 +- idle 只在覆盖完整且真实无活动时成立；partial/unknown/degraded/disabled 不被客户端转换为全部完成。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## J02: heal.outcome-counters-and-error-truth + +Topologies: H1,H2. Branches: mixed-healthy-repaired-failed, dry-run, deferred, readonly-terminal-failure, unknown-negative. + +Execution policy: `evidence-blocked`; disabled by default: `false`. + +Blocked reason (`EVIDENCE_MIXED_OUTCOME_ORCHESTRATION`): 已有 mixed healthy/repaired/failed、独立 dry-run、依赖 deferral、真实 SDK legacy Unknown、原始全副本损坏元数据 Unknown、只读永久失败/新 token 补偿及版本级 grace Deferred/新 token 清理执行流程，均按逐对象 manifest/attemptFailures 对账；仍需完整串行 live 证据及独立验收，故保持禁止默认 dispatch 和 canonical admission；本地检查不能作为完整 PASS。 + +1. 同批独立key建立健康、可恢复缺片、低于R的不可恢复对象；deferred由冻结的锁/依赖故障产生，dry-run另建任务。 +2. 保存逐对象物理基线和期望disposition，对结果的execution/coverage/counters/object identities做逐项对账。 +3. unknown/skipped/completed_with_errors不算修复成功，legacy objectsHealed与canonical healed分开解释；attemptFailures不得被隐藏。 +4. 使用独立完整manifest弥补有界outcome窗口，保持不可修对象的正确失败记录；恢复后另任务修复不覆盖原结果。 +5. grace 分支先用 SDK PUT 建立两个真实版本并验证完整分片；保存一个成员的原对象树后，SDK DELETE 精确历史版本，逐盘验证仅当前版本存在且其逻辑元数据、数据片及控制对象未变。只向原成员回放原对象树；实际候选进程固定 grace=120 秒，不修改元数据年龄或时钟。原任务须在窗口内完成，精确报告旧版 deferred、当前版 verified_healthy 及 retryNotBefore；自然等待过期后原 token 仍不变，新 token 才能清理旧版。清理后先核对全部物理字节等于 DELETE 后权威现场，再跨节点 GET/404 和复查旧 token，最后才做 fixture finalization。错过窗口保留 NOT_OBSERVED；异常保留现场并标 DIRTY；救援仅恢复 DELETE 后权威树，不复活已删除版本。 +6. Manual diagnostic entry: explicitly select heal.outcome-counters-and-error-truth through Functional Chain with the frozen candidate and candidate-bound manager/inspector tools. The default catalog policy stays evidence-blocked; even all passing diagnostic branches retain complete=false, canonicalCasePassClaimed=false and a blocking gate verdict until independent acceptance. +7. For a pre-replay public-manifest refusal, recover only after the complete SDK request/ACK ledger proves all writes acknowledged and no DELETE or replay occurred. Bind all four original physical members and the reserved owner, stop the exact candidate, then preserve all roots and state using durable same-filesystem non-overwriting rename. Partial archive retries require the same durable inventory; missing roots or drift retain DIRTY. Isolation is not product repair or restoration and never rewrites the failed case. +8. Manual J02 candidate admission is limited to the original 072b57b37559fbe626aeda8abf61617fdcbdd926 and corrected 1c7fc818dd858247c45b39542abd8291b1e6e31b sources. Each execution requires the matching immutable package and independently audited inspector/manager bundle. A corrected-candidate run is new evidence; it cannot replace an original failure. The full exact-version, physical, timing, retry and responsibility oracles remain unchanged, and all passing diagnostics still require independent canonical acceptance. +9. The deferred source branch follows Batch.deferred in frozen validation script 524eb5b32c3236497fdd8a9271df0494fa3d9974: complete the exact-version grace, natural-expiry and new-token cleanup procedure, then finalize this branch. An additional R-1 metadata existence-check diagnostic is not part of that source procedure and does not gate its verdict. Its historical failures and strict replay assertions remain retained; original source failures, missed-window NOT_OBSERVED and uncertain-mutation DIRTY are unchanged. + +Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [SEED_DV](validation-heal-operations.md#SEED_DV) → [FAULT](validation-heal-operations.md#FAULT) → [LOCK_WINDOW](validation-heal-operations.md#LOCK_WINDOW) → [START](validation-heal-operations.md#START) → [WAIT](validation-heal-operations.md#WAIT) → [BACKGROUND](validation-heal-operations.md#BACKGROUND) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [RECOVER](validation-heal-operations.md#RECOVER) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 同批次混合健康、可修、不可修、deferred、dry-run 对象 +- execution/coverage/disposition/计数和逐对象物理结果相符；unknown、skipped 或 completed_with_errors 不能算修复成功。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## J03: heal.incremental-cursor-and-truncation + +Topologies: H1,H2. Branches: within-window-cursor, past-retention-gap, cross-node-cursor, terminal-truncation. + +1. DX至少257个目标，超过128诊断窗口；边启动recursive heal边以sinceSeq查询，每次请求/nextSeq/minSeq/items全部保存。 +2. 单端连续读取应无重复倒退；故意使用过旧cursor应明示truncated/gap；跨节点查询按任务/epoch而非全局数字比较。 +3. 任务终态也读取旧cursor，验证有界列表不被当完整清单；所有错误与缺口原样进入报告。 +4. 最终用全量独立S3/physical manifest与canonical counters验收覆盖；若任务太快仍可做终态截断，但不能声称观察了运行中游标。 + +Operations: [SEED_DX](validation-heal-operations.md#SEED_DX) → [FAULT](validation-heal-operations.md#FAULT) → [START](validation-heal-operations.md#START) → [CURSOR](validation-heal-operations.md#CURSOR) → [STATUS](validation-heal-operations.md#STATUS) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 生成超过结果保留上限的任务，分页轮询 sinceSeq，并跨节点/终态读取 +- nextSeq/minSeq 单调且 gap/truncation 明示；有界 items 不是完整责任清单，最终须用独立 manifest/计数补证，不静默丢错误。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## J04: heal.repeat-idempotency + +Topologies: H1,H2. Branches: normal-repeat, deep-repeat, repeat-after-restart, versioned-repeat. + +1. DB/DV缺片先修复；versioned 的整份 xl.meta 缺失允许所有 identity repaired，或至少一个 repaired 加其余正健康态；marker 只能为 repaired/metadata_healthy，counters.healed 与 repaired 数量一致、counters.unchanged 与其余数量一致，unknown/skipped/failed=0。完整保存 post-heal S3/physical 作为第二轮基线，不使用最初故障前 xl.meta 要求内部字节完全相同。 +2. 再次 normal/deep：normal 只接受 metadata_healthy，deep 才要求 verified_healthy；两者均 healed=0、unknown/skipped/failed=0，不得新增 repaired，所有原版本/marker 身份与数量一致。 +3. PROCESS原unit重启后再重复；检查新任务有独立token但不产生新对象版本、无限责任或重复修复计数。 +4. 允许经格式证据证明的内部维护记录变化，用户内容/引用不变，目标片摘要稳定。 + +Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [SEED_DV](validation-heal-operations.md#SEED_DV) → [FAULT](validation-heal-operations.md#FAULT) → [START](validation-heal-operations.md#START) → [WAIT](validation-heal-operations.md#WAIT) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) → [PROCESS](validation-heal-operations.md#PROCESS) + +Source assertions: + +- 第一次修复后重复 normal/deep、重启再检查 +- versioned 首次允许所有 identity repaired 或至少一个 repaired 加其余正健康态，marker 仅 repaired/metadata_healthy，计数必须与 receipt 集合一致；normal repeat 接受 `metadata_healthy`，deep repeat 要求 `verified_healthy`，repeat marker 为 `metadata_healthy` 且 healed=0；unknown/skipped/failed 不得成为 PASS。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## J05: heal.long-soak + +Topologies: H2,H5. Branches: 24h, 72h. + +Execution policy: `environment-gated`; disabled by default: `true`. + +Blocked reason (`ENV_DEDICATED_LONG_SOAK_LANE`): 必须在独占 H2/H5 lane 真实运行 24 小时和 72 小时，包含限速、监控、到期回收和完整资源预算；短跑不能替代 soak 证据。 + +1. 只在专门分配的H2/H5独占lane，冻结完整24h或72h墙钟窗口、循环故障日程、并发/限速/存储上限和到期回收安排。 +2. WORKLOAD持续记录逐请求ACK，周期性故障仅使用本run DRIVE_GATE/PROCESS/依赖规则；每轮见证与恢复，不无限加数据。 +3. 持续METRICS、日志、队列/MRF/checkpoint取样；资源增长、panic、未解释责任达到预注册阈值立即保留失败证据。 +4. 到期停止写入/注入，解除全部故障，解析未知ACK后全量VERIFY和PHYSICAL；短跑不能充24h/72h结果。 + +Operations: [SEED_DX](validation-heal-operations.md#SEED_DX) → [SEED_DV](validation-heal-operations.md#SEED_DV) → [WORKLOAD](validation-heal-operations.md#WORKLOAD) → [DRIVE_GATE](validation-heal-operations.md#DRIVE_GATE) → [PROCESS](validation-heal-operations.md#PROCESS) → [BACKGROUND](validation-heal-operations.md#BACKGROUND) → [CHECKPOINT](validation-heal-operations.md#CHECKPOINT) → [METRICS](validation-heal-operations.md#METRICS) → [LOGS](validation-heal-operations.md#LOGS) → [RECOVER](validation-heal-operations.md#RECOVER) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 独占环境运行预注册 24h/72h 窗口，混合修复、掉线、写入和版本删除 +- 每次 ACK 可追踪；最终全量 manifest 正确，队列/内存/FD/临时空间无无界增长，无后台 panic 或未解释持久责任。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## J06: heal.performance-abba + +Topologies: H2. Branches: abba-repeat-1, abba-repeat-2, abba-repeat-3. + +1. 同硬件/候选/rc预注册预热、每轮时长、并发、读写比例、数据量/缓存策略、P95/P99与错误率阈值；禁止结果出来再定门槛。 +2. 按A无heal、B有heal、B有heal、A无heal顺序，每轮从相同逻辑内容和队列基线开始；完成至少三组独立ABBA。 +3. WORKLOAD记录每请求延迟/成功/错误/超时，METRICS记录CPU/IO/FD及实际修复字节；错误不从总请求分母删除。 +4. 报告每轮及配对差异、修复吞吐和GET/PUT各自P95/P99；功能VERIFY必须通过，性能阈值也必须独立满足。 + +Operations: [SEED_DX](validation-heal-operations.md#SEED_DX) → [PERFORMANCE](validation-heal-operations.md#PERFORMANCE) → [WORKLOAD](validation-heal-operations.md#WORKLOAD) → [FAULT](validation-heal-operations.md#FAULT) → [START](validation-heal-operations.md#START) → [METRICS](validation-heal-operations.md#METRICS) → [LOGS](validation-heal-operations.md#LOGS) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 同硬件同候选按 ABBA 运行无 heal 基线和有 heal 负载，重复并固定限速 +- 分别报告修复吞吐、GET/PUT P95/P99、错误率和 CPU/IO；阈值事先冻结，不能事后挑轮次或把功能 PASS 充性能验收。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 diff --git a/docs/validation-heal-contract.md b/docs/validation-heal-contract.md new file mode 100644 index 00000000..63b216c7 --- /dev/null +++ b/docs/validation-heal-contract.md @@ -0,0 +1,309 @@ +# Heal 真机验证设计与验收契约 + +## 状态与范围 + +本规范保留源仓库的 84 个 Heal canonical 用例、302 个操作分支及 534 个适用拓扑分支模板。资源要求、公共通过判据和逐项附加断言必须一起满足。 + +入口为 `scripts/validation_heal/matrix.py`、[逐例操作卡](validation-heal-cases.md) 和 [执行手册](validation-heal-operations.md)。运行环境是当前 auto-testing 的 Heal 节点；候选及 rc 使用当前 validation lane 的不可变包和实际二进制摘要。源仓库历史 pilot 结果不能计入本次通过数量。 + +全部 84 个 Heal ID 已有调度路由，包括 67 个常规入口、13 个默认关闭的环境场景入口，以及 C11/F07/I05/J02 的四个诊断入口。测试实现完整性与产品 live 结果独立统计：C11/F07/I05/J02 的源分支已有执行流程；F07 组合相同候选的 manager 中断与目录 fsync 证据，I05 总控串行执行 H1/H2 超时和断连控制器，并独立绑定完整内容/物理代次及 owner 原生证明，见[实现审查](validation-migration-implementation-review.md)。四个诊断入口的 canonical 验收门禁继续关闭；注册入口或 native 子集通过均不能充作完整 PASS。A09/G01–G06 的精确 native 与候选准入按各自执行器核对，G06 已有完整独立 live 验收。产品 FAIL 不自动等于测试未迁移。分片执行器核对实际盘角色、停止服务后的文件故障、原始同角色分片摘要与逐块 HighwayHash。A03 通过原路径/token 比较 H2 协调者、独立 peer 和固定节点入口的任务身份、终态及完整 outcome；候选每响应生成的 startTime 不作为任务 epoch。A04 保留四个 DB 参数组合及两个 DV 分支，要求完整 dry_run_observed 集合且对象没有变化。A05 使用独立部署执行全部八个 H1/H2 分支，冻结实际 merge 配置、真实非零 running 窗口、精确冲突与 token 策略及旧/新任务责任，完成后在服务运行状态下证明所有 peer 退出和物理静默。A06/F06 证明取消后的物理静默、旧 token 隔离；F06 还保留取消任务原始持久化记录，重启后不得复活或覆盖新任务。A08 保留五种主体各自 start/status/stop 操作、真实过期签名、仅授权 admin:Heal 的临时用户及清理后凭据失效。B09 使用明确启用 MRF 的专用服务，记录节点离线时的降级 ACK，并先验逐盘补片再执行目标 S3 请求。J03 必须实际超过 1024 项保留窗口，截断的诊断清单仍需独立全量物理/S3 补证。13 个环境场景保留源操作分支、未满足条件和显式启用入口；其完整 live 验证仍需资源适配器与源断言审核。每项实际状态见 [迁移映射](validation-migration-map.md)；本地检查、操作卡和归档不能代替 live PASS。 + +B03/B04 replacement runner 先执行只读 mount admission preflight，再允许停止服务、下载/安装候选包或创建远端 run namespace；阻断结果明确记录 `execution_stage=pre-deployment`、候选/rc 来源和 `binary_sha256=null`，表示尚未观察到安装候选二进制。通过预检后，部署阶段仍会在 reservation 前再次读取同一组 mount/设备身份，避免预检与实际变更之间的环境漂移。 + +H07 的单 set 两个分支只属于 H3，双 set 两个分支只属于 H4；四个不匹配组合记为 `nonApplicableCombinations`。缺少机器属于资源阻塞。 + +目标不是只证明“坏盘后仍能 GET”，而是分别证明： + +1. 故障真实发生在指定对象、版本、set 和目标盘。 +2. 客户可见控制接口及期望触发源处理了这份修复责任。 +3. 可恢复时确实修好了目标分片；不可恢复时明确失败且不扩大损坏。 +4. 对象、历史版本、删除语义、并发成功写入和持久化责任正确。 +5. 原始证据完整，临时故障被解除，环境可安全交还。 + +## 版本与已核实的产品边界 + +接口参考来自源设计记录的 RustFS 与 rc 提交，保留下面的源码链接用于追溯。每批执行仍以当前 candidate manifest 的精确源码、包、已安装二进制摘要和 auto-testing/rc 摘要为准，不由历史版本的验证结果推断本次候选已通过。 + +| 已核实的接口/行为 | 对设计的影响 | +| --- | --- | +| 签名 Admin POST `/rustfs/admin/v3/heal/`、`/{bucket}`、`/{bucket}/{prefix}`，token 查询与 forceStop/forceStart | 启动回执仅表示受理；必须查询任务并结合内容与物理证据验收。 | +| POST `/rustfs/admin/v3/background-heal/status`，GET `/rustfs/admin/v4/heal/replacement-recovery` | 后台状态、完整 peer coverage、replacement readiness 与具体任务终态分开判断。 | +| outcome 含 execution、coverage、counters、逐对象 disposition；objects/items 是有界诊断窗口 | finished、队列为 0、HTTP 200 都不是“所有对象已修好”的充分条件。 | +| query 支持 sinceSeq；结果含 nextSeq/minSeq/truncated | 需要游标、截断和缺口负例；被截断的 items 不能用作完整清单。 | +| Admin 保留 nolock wire 字段，但实际强制走正常对象锁路径 | A09 验证其不能绕过锁；不沿用历史架构文档中允许 Admin 绕锁的旧描述。 | +| root heal 优雅重启恢复限于原协调者、原拓扑且记录盘仍可读 | 不预设跨协调者故障转移、记录盘丢失恢复或任意降级兼容；MRF/replacement/root 各按各自责任契约验收。 | +| 参考 rc 提供 heal start/status/stop 及 bucket/prefix/token、normal/deep、dry-run、remove/recreate | pool/set、forceStart、sinceSeq 等未在该 CLI 版本暴露的功能通过签名 Admin 测，不虚构 rc 参数；新批次先检查实际 help/schema。 | + +源码：[Admin heal](https://github.com/rustfs/rustfs/blob/940e221988694da33fe005054bb86086bae88cd6/rustfs/src/admin/handlers/heal.rs)、[HealOpts](https://github.com/rustfs/rustfs/blob/940e221988694da33fe005054bb86086bae88cd6/crates/heal-contracts/src/heal_channel.rs)、[outcome/责任回执](https://github.com/rustfs/rustfs/blob/940e221988694da33fe005054bb86086bae88cd6/crates/heal/src/heal/outcome.rs)、[rc 命令](https://github.com/rustfs/cli/blob/204eb5ad85b46a007afd4da538cbeb1cadd867b6/crates/cli/src/commands/admin/heal.rs)。 + +## 拓扑与故障域 + +下表是**需求模型，不是已经分配的可用机器清单**。所有真机动作必须经跳板机和固定 fleet lease，不能私自创建/删除 Azure VM、磁盘或网络。已有 example profile 不是运行许可；实现时必须绑定节点、端口、服务、mount UUID、盘数、pool/set/slot 与独占范围，读回实际 EC 元数据确认布局。 + +| 编号 | 最低形状/用途 | 当前边界 | +| --- | --- | --- | +| H1 | 单节点四独立盘、一个 EC set；基础盘修复 | 可参考 single-4disk；使用独立 heal 服务和每盘 heal-validation 根。需要先绑定空闲资源，不复用现有 Pool 数据目录。 | +| H2 | 四节点各四盘、同一分布式 pool，明确固定 set 划分；跨节点故障/锁/状态 | 可参考 cluster-1pool；必须证明目标 set 跨节点，不能只看全局节点总数。 | +| H3 | 精确 EC8+4 单 set，参考主仓库的 3 节点×4 盘测试形状 | 是测试参数，不是生产部署推荐；旧两节点 Pool 环境不满足此拓扑。 | +| H4 | 精确 EC8+4 至少两个 set，参考 3 节点×8 盘、同 pool | 至少 24 个独立盘端点及正确 set 映射；不得用 4 盘或一个 set 替代。固定资源未分配时为阻塞，不是通过。 | +| H5 | 两个以上 pool；分开登记两节点各自四盘的生命周期形状和跨节点 pool 形状 | 现 lifecycle-a 每台一个独立 pool，无节点级冗余；可验证盘级修复及移动交互，不能证明节点故障容错。对齐主仓库多 pool EC8+4 基准时，另登记其三单机×12 盘/三个 pool 布局，不混用计数。 | +| H6 | 独占 VM/主机与专用故障盘，可做强制重启 | 需要额外运维授权、带外恢复和事件证据；普通故障 lane 不允许自动进入。 | + +生产拓扑取舍可参考 [RustFS Linux 部署说明](https://docs.rustfs.com/en/installation/linux)：单机多盘的盘冗余不等于整机故障冗余。本设计的结论只对实际验证的拓扑成立。 + +每个对象/版本预先记录 N、data/parity、分片索引与 set/盘映射，再从候选实现与元数据确定读取 quorum R、修复写 quorum W 和锁 quorum。测试 R-1/R/R+1 与 W-1/W/W+1 的**合法组合**，不以全局在线盘数或固定“坏一半仍能写”代替计算。跨版本、跨 set 的片不能相加；R 与 W 不一定相同。N=1 无 EC 的能力/不可恢复负例属于额外参数实例，不得宣传可自动找回完全丢失的数据。 + +## 故障后端与安全门禁 + +所有注入必须有 machine-readable fault-plan 和 fault-receipt；不存在“命令成功即注入成功”。默认每次只隔离本用例一个已解析目标，再按预注册组合扩展。使用以下分类，报告必须记录实际 faultBackend，不能互相冒充。 + +| 编号 | 故障类别 | 注入有效性与限制 | +| --- | --- | --- | +| F0 | 无故障、纯接口请求 | 记录稳定基线；仅用于无副作用/负请求验证。 | +| F1 | 指定 part/shard 缺失 | 先校验精确路径和摘要，以同盘隔离备份/原子移位保存，再证明目标缺失；不是物理掉盘。 | +| F2 | 指定字节翻转、截断、校验头损坏 | 在隔离测试对象副本上操作，保存偏移、长度、前后摘要及原始备份；必须有足够未损坏权威。 | +| F3 | 指定 xl.meta/inline/身份/桶元数据缺失或损坏 | 需要与候选格式匹配的只读解析器和允许列表；不得操作 IAM/凭据配置、共享 format 或无关对象。 | +| F4 | 原逻辑盘/服务卷短暂不可达、只读 | 采用仅影响测试服务的受限挂载命名空间或故障层，并实际验证访问失败。不得卸载含其他服务数据的共享 XFS mount；chmod 不能未经验证当作有效故障。 | +| F5 | 慢 I/O、EIO、短读、ENOSPC/inode 故障 | 只在预分配独占测试设备或有容量上限的故障层；不得塞满共享数据盘。明确模拟故障与真实设备故障的区别。 | +| F6 | 测试进程优雅停止、SIGKILL、重启 | 限定已验证服务和 MainPID，证明在部分进度窗口发生；记录 boot/process 身份。SIGKILL 不是断电。 | +| F7 | 节点间测试端口网络故障 | 仅对本服务地址/端口、受控 namespace 或独占环境，保护 SSH/跳板机/管理平面；规则带唯一 ID、超时和幂等恢复。 | +| F8 | 空替换逻辑盘、新 disk UUID | 验证原/新盘和 pool/set/slot；不把移动回备份当重建。真实设备 detach/attach 另需独占资源授权，不修改共享盘。 | +| F9 | checkpoint/commit/锁窗口的确定性故障 | 优先原生 runtime-fence/E2E 屏障；Release 黑盒无法证明精确窗口时，单独列原生证据与真机覆盖缺口，禁止给优化包塞入隐藏测试 hook 冒充同一候选。 | +| F10 | 独占 VM/主机强制重启 | 单独审批的破坏性 lane；有带外恢复和平台时间线。VM forced stop 不证明物理控制器缓存断电语义。 | +| F11 | 队列/请求压力、测试 KMS/tier/配额依赖故障 | 只作用于已分配测试租户/进程，限量限时，禁止全局断开共享 KMS 或远端服务。 | + +fault-plan 至少包含 run/case/variant ID、拓扑指纹、对象/版本/incarnation、pool/set/slot、盘 UUID、精确文件允许列表、设备/根目录身份、注入前条件、变更预算、备份摘要、恢复动作和自动解除故障截止时间。路径按目录描述符解析，拒绝链接逃逸、设备漂移、未知文件和自由 shell 插值。 + +故障解除与数据清理是两件事：网络/只读/进程等临时环境损伤要安全解除；失败数据、分片、checkpoint 和证据应保留，环境标记 DIRTY/QUARANTINED，不能继续下一例。捕获证据后可按已批准流程恢复业务可达性，但不覆盖失败记录或把恢复重跑算首次通过。备份只可用于失败后的救援/重置，不参与修复成功判定。 + +## 数据夹具与参数维度 + +| 编号 | 内容 | +| --- | --- | +| DB | 确定性基础 manifest：默认 128 个对象，包含 0/1、4095/4096/4097 字节以及实际 inline、EC stripe/block、multipart 边界±1；按磁盘证据确认形态，不硬编码内部阈值。 | +| DM | 已完成的多 part 对象、非等长 part、跨 part Range；另外创建可追踪 UploadId 的 pending/Complete/Abort 夹具，遵守 S3 最小非末 part 大小限制。 | +| DV | 每 key 至少三个不同内容版本、当前/历史 delete marker、versioning suspended/null、明确删除回执；记录 versionId、Last-Modified、ETag 和版本列表。 | +| DS | 多桶、相邻前缀、多页 key、中文/空格/百分号等编码，metadata/tags/content headers 与隔离桶配置。 | +| DE | 候选明确支持且依赖就绪的 SSE-S3/KMS/C、远端 tier 版本；密钥只由受控凭据渠道提供，禁止进入日志/manifest。 | +| DO | object-lock、retention 和 legal hold 的受保护版本；使用可到期测试策略，防止成功用例因无法清理永久占用环境。 | +| DL | 并发 PUT/GET/HEAD/LIST/COPY/DELETE、版本及 MPU 操作的逐请求 ACK ledger；每 key 用递增 nonce 与确定性内容，保存请求/响应和不确定结果。 | +| DX | 保证可观察部分进度、大于状态窗口（>128 项）的夹具；默认上限在运行计划中冻结，长稳/性能单独配额，不靠无限加数据等到偶然命中。 | + +按 case/variant 参数化：normal/deep、data/parity shard、inline/外置 part、单片/多片、原盘/新盘、对象/前缀/root/set、版本/加密、协调者/目标节点、单节点/多节点/多 set/pool。使用预先冻结的 pairwise 组合控制规模；损坏+quorum、旧代次+并发删除、目标重建+进程中断等高风险组合必须显式三维覆盖，不由 pairwise 替代。 + +候选声明支持的非默认 parity/storage class、服务端压缩 on/off 也应加入 DB/DM/DV 参数；Content-Encoding 响应头不是服务端压缩已被执行的证据。每个实例都从实际元数据确认编码形态。Object Lock 的拒绝删除必须发生于真实保留期内；清理不能绕过 Compliance、修改主机时间或以内部删文件规避限制。Legal hold 仅由本次授权测试角色正常解除；保留期超出 lane 窗口的实例必须预先分配独占 stateful 环境及到期处理方案，未清理环境不能交还 READY。 + +重试不能隐藏 ACK：PUT/Complete 超时属于结果未知，必须按唯一 key/version/UploadId 查询解析；不能把未收到成功的一次写当丢失，也不能把重试新版本当同一次成功。并发时只根据可证明的 happens-before/提交回执判断“最后一次”，不凭跨节点墙钟排序。 + +## 公共通过判据（每例均适用） + +每行用例的完整规范 = 本节 + 对应拓扑/夹具/故障契约 + 行内操作与附加断言。参数实例独立留证、计分。 + +1. **预检和基线。** 全部预期 peer/磁盘、candidate/rc/harness 身份一致；预先 GET/HEAD/LIST 完整基线并保存必要的盘上记录。记录扫描/自动 heal 配置与任务归属。 +2. **注入和见证。** 以只读物理证据确认故障确实发生，未影响其他对象/服务；注入后不要先 GET 目标而意外触发 read-repair。受理过快或已被其他来源修好，不能冒充本例触发成功。 +3. **控制面真相。** 收集每次 rc exitCode、签名 Admin HTTP/body、token、source、scope、outcome 和 peer coverage。区分 repaired、verified_healthy、metadata_healthy、authoritatively_absent、deferred、failed、unknown、cancelled、dry_run_observed；deferred、failed、unknown、cancelled、dry_run_observed 不能作为“修复完成”。`metadata_healthy` 只证明 metadata/identity 健康，不证明 payload integrity；Normal scan 不得冒充 `verified_healthy`，deep payload 证明继续要求 `verified_healthy`。实际 wire 枚举以固定候选为准。 +4. **对象正确。** 所有必须存在的对象/版本全量 GET 验证长度/SHA-256/ETag/metadata/tags；HEAD、分页 LIST、历史版本与 marker 语义、已确认删除和并发 ACK 全部核对。允许修复产生合法内部元数据变化，不要求整个 xl.meta 字节不变。 +5. **实际补片。** 在静默校验窗口解析目标物理数据与元数据，核对版本/代次/data_dir、part 长度、bitrot 校验与 EC 一致性。对 data/parity 各有独立重建或重新编码对照，或用包含目标片的固定合法解码集合佐证；不能只统计文件、看 mtime 或依赖 GET 自动纠删码兜底。密文/压缩存储层不能直接拿明文摘要比较。格式不支持/HighwayHash 未验证时必须标明缺口，不能声称已完成深度物理验收。 +6. **持久化与幂等。** 需重启的用例验证重启后责任和数据；重复 heal 不引入新版本、复活删除或无限增加任务。状态计数按进程/任务/epoch 区分，不能把重置后的全局计数简单判为回退。 +7. **完成条件。** 正向修复要有相关责任完成、范围 coverage 完整、对象与物理证据一致。无关的全局 idle、列表项被截断、缺失 peer、unknown/残留待修工作都不能代替证明。预期不可恢复/权限/取消/dry-run 用例按预注册负向契约通过，明确它不是修复成功。 +8. **恢复和证据。** 解除临时故障，保存 before/fault/after/restart 四阶段证据，校验全局结果/JUnit/计划/对象数量/证据引用及清理。任何取证缺失、目标误命中、恢复失败或结果结构损坏均阻断该 lane。 + +针对精确时间窗口，必须观察活动/部分进度或受控屏障，再注入故障；普通重建允许快速完成但必须有实际补片证据。禁止用固定 sleep 或“重跑直到绿”代替见证。建议起始测试预算：接口就绪 90 秒、普通修复 15 分钟、重启恢复 30 分钟、同 epoch 静默窗口 5 秒；这些是待校准的**测试预算，不是产品 SLO**，所有最终预算须在运行前冻结。预期暂态错误只按明确 code 有界重试，每次都留证。 + +判定类型：产品安全/内容/终态错误为 fail；环境注入/取证工具失败为 infra_error；缺少尚未分配拓扑或辅助能力在部署前阻断。不得把产品 SlowDown 或低 quorum 下的预期错误泛化成 infra_error。必选能力缺失、未知选项、缺失参数实例不能用 unsupported/skipped 充 PASS；不适用项只能在运行前的 applicability manifest 中说明，环境尚未准备好不是“不适用”。 + +## 84 个基础用例 + +编号用于设计评审；`heal.*` 为稳定 catalog ID。topology、fixture、fault 中逗号表示必须展开记录的适用实例/后端集合，不表示同时向全部对象注入所有故障。对应的组合、先后和最小 quorum 必须在编译后的 variant plan 中明确，未证明的一项不得计为通过。 + +| 分类 | 设计条目数 | +| --- | ---: | +| A 控制面、任务与权限契约 | 10 | +| B 磁盘故障、替换与 quorum 边界 | 10 | +| C 分片损坏、元数据与权威选择 | 11 | +| D 对象类型与 S3 语义 | 11 | +| E 触发路径与修复责任 | 6 | +| F 中断、持久化、恢复与升级 | 9 | +| G 与前台和后台操作并发 | 8 | +| H 多 Pool、移动与多 set 隔离 | 7 | +| I 资源、依赖故障与调度 | 6 | +| J 可观测性、幂等、长稳与性能 | 6 | +| 合计 | 84 | + + + +### A. 控制面、任务与权限契约 + +| 编号 / 稳定 ID | P | 拓扑 / 夹具 / 故障 | 操作与触发 | 附加通过判据 | +| --- | --- | --- | --- | --- | +| A01 · `heal.admin-healthy-noop` | P0 | H1,H2 / DB / F0 | 对健康对象分别执行 normal/deep，再重复查询 | 不修改用户数据或版本；Normal 只接受 `metadata_healthy`，Deep 才要求 `verified_healthy`，healthy 与 repaired 分开计数，不能把检查盘数当修复盘数。 | +| A02 · `heal.admin-scope-isolation` | P0 | H1,H2 / DB,DS / F1 | 分别以 root、bucket、prefix、object、pool/set 选择器启动 | 目标范围实际遍历；相邻前缀、其他桶及未选 set 的故障保持原状；按对象清单而非目录数量验收。 | +| A03 · `heal.admin-token-cross-node-status` | P0 | H2 / DB / F1 | 从节点甲启动，经乙及稳定入口按原 token 查询 | 任务身份、scope、终态及核心计数有界收敛；状态缺失/未知不能解释为完成。 | +| A04 · `heal.admin-dry-run` | P0 | H1,H2 / DB,DV / F1,F2 | 对缺片与损坏对象执行 dryRun，参数化 remove/recreate | 有明确 dry_run_observed 诊断；用户对象分片、xl.meta、版本与删除标记不变；后台修复干扰须排除。 | +| A05 · `heal.admin-overlap-and-force-start` | P1 | H1,H2 / DB,DS / F1 | 活动中重发同范围、父子前缀和不相交请求，再 forceStart | 按已固定 overlap policy 验证合并返回原 token 或明确冲突；forceStart 不能泄漏旧任务责任或并发破坏对象。 | +| A06 · `heal.admin-stop-and-stale-token` | P0 | H1,H2 / DB / F1 | 非零修复进度时 stop，稳定后新建任务，再发旧 token 的查询/停止 | 停止需等在途写结束并稳定；新任务身份独立，旧请求不得终止新任务；重复 stop 按公开契约处理。 | +| A07 · `heal.admin-invalid-request` | P0 | H1,H2 / DB / F0 | 发送非法 selector、重复 query、非法布尔/scanMode、冲突 flags、超长 body | 服务端返回已核实的结构化错误；无误启动、范围扩大或持久化副作用；CLI 参数错误不能替代服务端负例。 | +| A08 · `heal.admin-authorization` | P0 | H1,H2 / DB / F0 | 匿名、坏签名、过期签名、无 HealAdminAction 用户与最小授权用户分别请求 | 未授权 start/status/stop 均不能越权或泄漏对象信息；授权用户可完成规定操作，凭据不得写入证据。 | +| A09 · `heal.admin-nolock-safety` | P0 | H1,H2 / DL,DV / F1,F9 | 通过签名 API 传 nolock=true，并与同对象提交制造确定性争用 | 兼容字段不得跳过存储层对象锁；最后成功写入和已确认删除保持正确，原生屏障与真机竞争证据分开。 | +| A10 · `heal.admin-option-contract` | P1 | H1,H2 / DB,DV / F1,F2 | 参数化 recursive、recreate、remove、updateParity、readRepair 与合法/非法 pool/set 组合 | 仅测试已核实的候选语义；normal/deep 与危险 remove 的副作用明确；API 独有选项不得伪称 rc 已覆盖。 | + + + +### B. 磁盘故障、替换与 quorum 边界 + +| 编号 / 稳定 ID | P | 拓扑 / 夹具 / 故障 | 操作与触发 | 附加通过判据 | +| --- | --- | --- | --- | --- | +| B01 · `heal.disk-loss-and-repair` | P0 | H1,H2 / DB / F4 | 让一个已确认 set 成员盘对 heal 服务不可用，持续读取，恢复原盘 | 记录真实故障窗口和源/目标盘身份；在可恢复条件下最终补齐目标分片及完整 manifest，不只验证降级 GET。 | +| B02 · `heal.disk-readonly-and-permission` | P1 | H1,H2 / DB / F4,F5 | 仅对独占测试卷制造只读/拒绝访问，恢复后重试 | 不可写目标不得 reported repaired；错误及待修责任可见；恢复后真实写入目标且不落到系统盘或其他目录。 | +| B03 · `heal.disk-replacement-formatted` | P0 | H1,H2 / DB,DV / F8 | 换入空的新逻辑盘，确认新 disk UUID/正确格式后触发重建 | 新盘与旧盘身份可区分且属于正确 pool/set/slot；逐版本重建完成，不能从隔离备份复制回来冒充 heal。 | +| B04 · `heal.disk-replacement-unformatted` | P1 | H1,H2 / DB / F8 | 引入空未格式化测试盘，并与带异集群格式的盘分别比较 | 按候选支持的初始化契约安全接纳或明确拒绝；不得覆盖外来身份或将未就绪盘标为可写健康。 | +| B05 · `heal.multiple-shard-loss` | P0 | H1,H2 / DB,DV / F1 | 按实际同版本 EC 布局依次缺失 1..M 个 shard，覆盖 data/parity 角色组合 | 每个参数实例单独满足读取与写入 quorum；满足修复条件时全部目标重建，不把跨版本或跨 set 的 shard 合并。 | +| B06 · `heal.read-quorum-boundary` | P0 | H1,H2 / DB / F1 | 保留恰好 R 个同版本权威 shard，测试读取及修复提交，再恢复写 quorum | 读取边界和修复写入边界分别判定；不能因还能 GET 就要求在写 quorum 不足时提交成功。 | +| B07 · `heal.below-read-quorum` | P0 | H1,H2 / DB,DV / F1 | 使权威来源小于 R，执行 GET 与 deep heal，再恢复被隔离源 | 故障期间不得返回伪造的成功内容、空对象或误删记录；明确 unresolved/失败；恢复后原对象和版本完整。 | +| B08 · `heal.below-write-quorum` | P0 | H1,H2 / DB,DL / F4 | 构造读 quorum 尚存而写/锁 quorum 不足，执行 heal 与新 PUT | 不能虚报持久化修复或成功写入；分别记录对象、元数据、分布式锁 quorum；恢复后核对所有已确认写入。 | +| B09 · `heal.node-outage-rejoin` | P0 | H2 / DB,DV,DL / F6 | 停止同一跨节点 set 的一个节点，执行允许的降级 I/O，再重入 | 按实际丢失盘数判定可用性；缺失期间成功提交的对象补齐；节点重新 ready 不等于所有对象已重建。 | +| B10 · `heal.disk-and-node-flapping` | P1 | H2 / DB,DL / F4,F6 | 重复短暂离线/在线并在重建途中再次失联 | 重试与队列有界、身份稳定，无重复任务风暴；最终故障解除后收敛，所有已确认对象保持正确。 | + + + +### C. 分片损坏、元数据与权威选择 + +| 编号 / 稳定 ID | P | 拓扑 / 夹具 / 故障 | 操作与触发 | 附加通过判据 | +| --- | --- | --- | --- | --- | +| C01 · `heal.missing-part-file` | P0 | H1,H2 / DB,DM / F1 | 仅隔离指定对象某个 part/shard，保留 format 与其他元数据，执行 deep heal | 对应数据或校验分片重新生成，part/版本身份准确；不得只看到 xl.meta 重现就通过。 | +| C02 · `heal.data-shard-bitrot` | P0 | H1,H2 / DB,DM / F2 | 在指定 data shard 的已知偏移翻转字节，包含首/中/尾位置 | 故障前后摘要证明注入生效；deep 或 read-repair 发现损坏并修复；独立 bitrot/EC 及 S3 全内容验证通过。 | +| C03 · `heal.parity-shard-bitrot` | P0 | H1,H2 / DB,DM / F2 | 损坏 parity shard，使普通 GET 仍可能成功，再执行 deep heal | 必须识别并重建损坏 parity；只有 GET 正确、无目标分片修复不能通过。 | +| C04 · `heal.truncated-and-zeroed-shard` | P0 | H1,H2 / DB,DM / F2 | 分别截断 shard、置零和破坏校验头，保留正确元数据仲裁来源 | 短读、全零和校验损坏被准确识别；修复不能接受截断内容或错误长度。 | +| C05 · `heal.missing-or-corrupt-xlmeta` | P0 | H1,H2 / DB,DV / F3 | 在少数成员上移走或损坏 xl.meta/inline 内容，保留足够权威来源 | 恢复同一对象的全部应存版本、数据引用与删除标记；不以新建空元数据替代真正修复。 | +| C06 · `heal.stale-version-rejoin` | P0 | H1,H2 / DV,DL / F4 | 盘离线后执行覆盖/版本删除，再让携带旧元数据的盘重入 | 以已确认的新代次/版本为准；旧当前值与已删版本不复活，旧盘被纠正而非成为新的错误权威。 | +| C07 · `heal.conflicting-subquorum-authority` | P0 | H1,H2 / DV / F3,F9 | 用受控原生夹具构造互不构成 quorum 的版本/代次集合，再做对应真机反例 | 禁止把不同版本的 shard 数量并集当成权威；无法证明时安全拒绝且保留可恢复证据，不猜测较新时间戳。 | +| C08 · `heal.metadata-payload-mismatch` | P1 | H1,H2 / DB,DV / F2,F3 | 构造元数据声称某长度/摘要但实际 part 不匹配的少数副本 | 元数据与内容联合判断；不把可解析的 xl.meta 当健康；权威不足时拒绝覆盖正确副本。 | +| C09 · `heal.foreign-format-and-unknown-schema` | P0 | H1,H2 / DB / F3,F8 | 提供外来 disk/set 身份、未来 schema 或不支持的记录格式 | 明确隔离/拒绝且不破坏原记录，不把未知当 missing 后直接重建；解除故障后原数据可用。 | +| C10 · `heal.dangling-and-orphan-safety` | P1 | H1,H2 / DV,DM / F1,F3 | 构造 dangling part、旧 data_dir 与共享历史引用，分别 remove=false/true、宽限期前后检查 | 默认不得破坏仍可恢复数据；显式 remove 只清除具有充分删除证明且过期的本次孤儿，不删共享版本或 MPU。 | +| C11 · `heal.bucket-incarnation-recreation` | P0 | H1,H2 / DV / F4,F9 | 盘离线时完成旧桶合法清空/删除，再同名建桶写入新对象，恢复旧盘与旧任务 | 旧 bucket incarnation 的 checkpoint/MRF/元数据不得污染新桶；旧任务不得删除或恢复新 incarnation 的对象。 | + + + +### D. 对象类型与 S3 语义 + +| 编号 / 稳定 ID | P | 拓扑 / 夹具 / 故障 | 操作与触发 | 附加通过判据 | +| --- | --- | --- | --- | --- | +| D01 · `heal.object-size-boundaries` | P0 | H1,H2 / DB / F1,F2 | 覆盖 0/1 字节、页/inline/EC block 边界±1以及大对象 | 每个实际存储形态均确认后注入；长度、SHA-256、ETag、HEAD/LIST 与边界 Range 正确，无假设阈值。 | +| D02 · `heal.completed-multipart-object` | P0 | H1,H2 / DM / F1,F2 | 修复已完成的不等长多 part 对象，分别损坏首/中/末 part | 完整内容、part 顺序、原版本/ETag/元数据和跨 part Range 不变；multipart ETag 不能当整对象 MD5。 | +| D03 · `heal.historical-versions` | P0 | H1,H2 / DV / F1,F3 | 不同版本分布不同损坏位置，按对象和递归范围修复 | 所有原 versionId 可寻址且内容正确，最新版本身份不漂移，无多余版本或遗漏历史版本。 | +| D04 · `heal.delete-marker-semantics` | P0 | H1,H2 / DV / F3,F4 | 缺失/陈旧 marker 副本后 heal，并按当前值与精确 marker versionId 查询 | 当前 GET/HEAD 与指定 marker 的状态码及身份头符合契约；marker 不复活数据，LIST versions 无重复。 | +| D05 · `heal.versioning-suspended-null` | P1 | H1,H2 / DV / F1,F3 | 在版本开启、暂停、null 版本覆盖之间建夹具再修复 | null 和非 null 版本不混淆，不新增假版本；暂停前历史对象仍准确可读。 | +| D06 · `heal.object-metadata-and-tags` | P1 | H1,H2 / DS / F1,F3 | 包含用户 metadata、Content-Type/Encoding/Disposition、tags 的对象修复 | 对外 metadata、tags 和相关响应头逐项相同；不要求无关内部元数据字节完全一致。 | +| D07 · `heal.range-head-and-conditions` | P1 | H1,H2 / DB,DM / F1,F2 | 修复前后验证 HEAD、合法/越界 Range、If-Match/If-None-Match 及时间条件 | 200/206/304/412/416 等按逐请求契约判断，绝不返回旧内容、错误范围或错误 ETag。 | +| D08 · `heal.encrypted-objects` | P0 | H1,H2 / DE / F1,F2 | 按候选支持的 SSE-S3/SSE-KMS/SSE-C 建加密对象并修复 | 授权解密后原内容及版本一致，物理验证使用正确编码/密文层；密钥不可用时不误删、不降级明文，证据脱敏。 | +| D09 · `heal.object-lock-retention` | P0 | H1,H2 / DO,DV / F1,F3 | 对开启 retention/legal hold 的受保护版本修复，期间测试非法删除 | heal 不绕过保留/hold，也不丢失其元数据；拒绝删除与合法修复均有明确证据。 | +| D10 · `heal.special-keys-and-pagination` | P1 | H1,H2 / DS / F1,F3 | 覆盖中文、空格、百分号、编码斜线、多层前缀和多页对象列表 | 逻辑 key 与物理路径映射准确，不跨桶/前缀；分页无遗漏重复，不能把用户 key 当可执行 shell 或裸文件路径。 | +| D11 · `heal.bucket-metadata-integrity` | P0 | H1,H2 / DS,DV / F3 | 在隔离桶白名单内测试可支持的桶元数据恢复，并核对外部 bucket 配置 | 版本设置、策略/标签/保留配置按契约保持；不得遍历凭据配置或把未知内部记录当可重建数据。 | + + + +### E. 触发路径与修复责任 + +| 编号 / 稳定 ID | P | 拓扑 / 夹具 / 故障 | 操作与触发 | 附加通过判据 | +| --- | --- | --- | --- | --- | +| E01 · `heal.manual-normal-attribution` | P1 | H1,H2 / DB / F1 | 禁止预读目标，保留 format，隔离背景干扰后显式 normal heal | 证明本次 Admin token 实际覆盖目标并修复可由 normal 检出的缺失；不要求 normal 发现所有静默 bitrot。 | +| E02 · `heal.manual-deep-attribution` | P0 | H1,H2 / DB / F2 | 在普通读取仍可成功的 parity 损坏上显式 deep heal | 按 token/对象/版本及物理变更证明 deep 检查和修复，不把后台提前修复算给手工任务。 | +| E03 · `heal.read-repair-attribution` | P0 | H1,H2 / DB,DV / F1,F2 | 不发 Admin heal，对已确认的特定损坏对象发一次可追踪 GET/Range | 读取正确且后续真实补片；若走异步队列需证实 source=read-repair，不能只有 GET 成功。 | +| E04 · `heal.scanner-cold-object-repair` | P0 | H1,H2 / DB / F1,F2 | 不 GET/HEAD 损坏目标、不发手工 heal，仅等待已启用的 scanner 路径 | 冷对象被发现并修复，来源和遍历窗口明确；scanner 周期结束但目标没检查不能通过。 | +| E05 · `heal.mrf-offline-write-replay` | P0 | H2 / DL,DV / F4 | 成员离线时完成有 quorum 的 PUT/DELETE，确认待修责任后重入 | durable MRF 责任持久且身份精确；只在 repaired/verified healthy/authoritatively absent 等已证明结果后释放，成功提交不丢失。 | +| E06 · `heal.replacement-auto-rebuild` | P0 | H1,H2 / DB,DV / F8 | 替换盘接纳后不发 Admin/目标 GET，观察自动 replacement heal | 替换代次、ready/completion 与盘上全部目标版本一致；不能用服务 ready、队列空或计数重置代替完成。 | + + + +### F. 中断、持久化、恢复与升级 + +| 编号 / 稳定 ID | P | 拓扑 / 夹具 / 故障 | 操作与触发 | 附加通过判据 | +| --- | --- | --- | --- | --- | +| F01 · `heal.root-graceful-coordinator-restart` | P0 | H1,H2 / DX / F6 | root heal 非零进度时优雅重启原协调者，保留原拓扑与记录盘 | 在明确支持的边界内恢复原 task ID 和剩余责任/预算，完成后无遗留活动记录；不扩张为协调盘丢失后的故障转移。 | +| F02 · `heal.target-graceful-restart` | P0 | H2 / DB,DV,DX / F6,F8 | 替换目标仅完成部分重建时优雅重启目标节点 | 故障确实发生于部分完成窗口；重启后续建同一目标盘/代次，最终 manifest 与全部目标分片正确。 | +| F03 · `heal.target-process-crash` | P0 | H2 / DB,DV,DX / F6,F8 | 部分重建时 SIGKILL 测试进程，再正常启动 | unclean shutdown 与修复进度有真实证据；已提交对象不丢失，未完成目标不会被当完整；明确不是断电测试。 | +| F04 · `heal.coordinator-process-crash` | P0 | H2 / DB,DX / F6 | 协调者非优雅退出并恢复，分别观察 root、MRF、replacement | 按各机制已核实的持久化边界判断；不得把丢失的易失 token 当成功，需显式记录剩余责任并可安全重扫恢复。 | +| F05 · `heal.full-cluster-restart` | P0 | H2 / DB,DV,DM / F6 | 有未完成修复责任时重启全部测试节点，固定原拓扑/磁盘身份 | 有持久化保证的责任不遗失；全部端点恢复一致，旧版本/marker/MPU 与目标分片完整，重复重启不扩散错误。 | +| F06 · `heal.cancelled-task-restart` | P0 | H1,H2 / DB,DX / F6 | 完成取消及在途退出后重启，随后 forceStart/新任务正常修复 | 已取消的任务不自动复活；旧 checkpoint/token 不能取消或覆盖新任务，旧责任不会误标为修复完成。 | +| F07 · `heal.checkpoint-and-mrf-corruption` | P0 | H1,H2 / DB,DV / F9 | 受控构造 checkpoint/MRF 截断、坏校验、未来 schema、陈旧代次及写入失败 | 已确认持久化责任不得静默删除或伪 ACK；未知记录保留且可诊断；精确崩溃窗口由原生屏障证明，不伪称真机命中。 | +| F08 · `heal.mixed-version-upgrade-rollback` | P1 | H2 / DB,DV / F6,F9 | 固定前后两个明确兼容候选，覆盖滚动升级、混合 peer、回滚与重启 | 能力不足必须显示 partial/unsupported 而非完整成功；按格式兼容契约保留责任，旧版本不能误退休新版本记录。 | +| F09 · `heal.dedicated-host-forced-restart` | P1 | H6 / DB,DV,DX / F10 | 经专项授权在部分修复/持续写入时强制重启独占测试 VM/主机 | 保留平台事件与 ACK ledger；验证落盘和恢复，不把 SIGKILL 当断电；VM 强制停止也不等同于物理控制器断电。 | + + + +### G. 与前台和后台操作并发 + +| 编号 / 稳定 ID | P | 拓扑 / 夹具 / 故障 | 操作与触发 | 附加通过判据 | +| --- | --- | --- | --- | --- | +| G01 · `heal.race-unversioned-overwrite` | P0 | H1,H2 / DL / F1,F2,F9 | 同 key 连续覆盖与 heal 同时推进，包含提交尾部清理窗口 | 按可证明的提交顺序验证最终最后一次成功写入；不能回滚到旧 data_dir，未确认超时写入单独解析。 | +| G02 · `heal.race-versioned-put` | P0 | H1,H2 / DV,DL / F1,F9 | 修复历史版本时创建新版本并重复读取各 versionId | 新旧版本不混片、不串内容；已返回成功的每个版本都保留，最新值按确定的先后关系验收。 | +| G03 · `heal.race-version-delete` | P0 | H1,H2 / DV / F1,F2,F9 | 对损坏的精确历史 versionId 发 DELETE，与 deep heal 竞争 | 已确认删除的版本不能被复活，其他历史版本不受影响；原生临界点测试与真机竞争分别留证。 | +| G04 · `heal.race-markers-and-batch-delete` | P0 | H1,H2 / DV / F3,F9 | 与创建/删除 marker、批量版本删除和无版本删除分别交错 | 按逐项 DELETE 回执核对版本/marker 集合，无复活、误删或遗漏批量错误。 | +| G05 · `heal.race-multipart-complete-abort` | P0 | H1,H2 / DM,DL / F1,F9 | heal 期间对不同已知 UploadId 分别续传、Complete、Abort | 成功 Complete 内容只出现一次；Abort 不破坏已提交对象或别的 upload；未完成 part 不被伪造为完整对象。 | +| G06 · `heal.race-copy-and-metadata-update` | P1 | H1,H2 / DS,DL / F1,F9 | COPY 源/目标和标签/元数据更新与修复交错 | 最终对象内容与对应提交的 metadata 一致，读修复发现新代次应重验/放弃陈旧提交而非覆盖。 | +| G07 · `heal.race-ilm-expiry-and-transition` | P0 | H2 / DV,DE / F1,F9,F11 | 分别与当前/历史版本过期、远端 transition/restore 同窗口执行 | 已授权过期不被恢复；迁移状态与远端版本引用正确；只有实际观察到交叠才计并发覆盖，tier 未配置需明确阻断对应实例。 | +| G08 · `heal.race-replication-resync` | P1 | H2 / DV,DL / F1,F9,F11 | 在隔离复制两端与 heal 同时运行新增/删除/重同步 | 对端版本、删除语义及源数据最终一致，无无限回环或重复传播；不得依赖仅源端 GET 宣称复制正确。 | + + + +### H. 多 Pool、移动与多 set 隔离 + +| 编号 / 稳定 ID | P | 拓扑 / 夹具 / 故障 | 操作与触发 | 附加通过判据 | +| --- | --- | --- | --- | --- | +| H01 · `heal.with-expansion-rebalance` | P0 | H5 / DB,DV / F1,F2 | 先证明源 pool 持有夹具，再扩容并与修复同时 rebalance | 修复目标不得指向错误 pool/set；新旧数据完整、移动终态正确，保留 scanner backlog 提交权威检查。 | +| H02 · `heal.with-rebalance-stop-resume` | P1 | H5 / DB,DL / F1,F6 | heal 活动期间 stop/restart/resume rebalance | 两类任务状态和身份独立；停止/恢复移动不误终止修复或丢失对象，恢复后两者均满足各自契约。 | +| H03 · `heal.with-decommission` | P0 | H5 / DB,DV,DM / F1,F2 | 分别在待退役源和存活目标制造可恢复缺片，再执行退役 | 不得提前删源或以临时副本冒充迁移成功；按 Pool 既定写入限制验收；只剩 survivor 后仍完整可读且有物理证明。 | +| H04 · `heal.with-decommission-cancel-clear` | P0 | H5 / DB,DL / F1,F9 | heal/移动争用中取消退役、Clear、恢复，并覆盖容量锁超时恢复 | 保持持久化后通知及在途退出顺序；不误取消新代次；对 #2423 的重试窗口单独注明真机是否实际触发，不只看最终绿色。 | +| H05 · `heal.after-retired-pool-removal` | P0 | H5 / DB,DV / F1,F3 | 源 pool 完全退役移除并停机后，只在 survivor 上触发修复 | 不再向已移除源写入或依赖其兜底；来源权威和版本正确；#2422 路径不能出现无权威覆盖或错误回退。 | +| H06 · `heal.multi-pool-target-isolation` | P0 | H5 / DB,DV / F1,F4 | 仅破坏选定 pool/set，并在另一 pool 保留不同对象与控制故障 | 选择器、任务、容量/配额和目标 drive 一致；未选 pool 不受错误修复，跨 pool 存在不等于该目标已修复。 | +| H07 · `heal.ec84-multi-set-restart` | P0 | H3,H4 / DB,DV,DX / F6,F8 | 固定 EC8+4 单 set 与至少两 set，部分重建时重启/崩溃目标 | 每个 set 的故障覆盖和重建分别证明，不跨 set 拼 shard；必须满足真实盘数及映射，不能用 4 盘用例替代 EC8+4。 | + + + +### I. 资源、依赖故障与调度 + +| 编号 / 稳定 ID | P | 拓扑 / 夹具 / 故障 | 操作与触发 | 附加通过判据 | +| --- | --- | --- | --- | --- | +| I01 · `heal.slow-io-and-eio` | P1 | H1,H2 / DB,DX / F5 | 在独占故障层注入有界延迟、短读、EIO，解除后观察恢复 | 暂态与永久错误分开，重试有界且无忙循环；解除故障后实际修复，期间不得提前标健康。 | +| I02 · `heal.enospc-inodes-and-quota` | P0 | H1,H2 / DB,DM / F5,F11 | 限制专用测试卷空间/inode，并参数化本次桶配额 | 仅填专用受限卷、不耗尽共享真实盘；失败不破坏好副本或释放尚需容量，释放空间后正确续修。 | +| I03 · `heal.network-partition-and-lock-quorum` | P0 | H2 / DB,DV,DL / F7 | 仅隔离测试服务节点间流量，覆盖少数派、读/写/锁 quorum 边界及恢复 | 少数派不得承诺无安全性的修复或写入；cluster coverage 不完整不能显示全局健康；恢复无分叉、跨代覆盖。 | +| I04 · `heal.queue-pressure-and-admission` | P1 | H1,H2 / DX / F11 | 多来源请求冲击有限队列，混合重复任务、不同优先级和多个桶 | Full/Dropped/Merged 语义与实际任务一致；丢弃不能丢 durable 责任；在预先声明负载窗口内无无限堆积或健康误报。 | +| I05 · `heal.lock-timeout-and-caller-abort` | P0 | H1,H2 / DL,DV / F5,F9 | 真实对象锁争用超过预算及调用方断开，再释放锁恢复 | 取消/超时不得提前释放在途所有权或重复提交；陈旧 read-repair 必须重验代次；恢复后无泄漏和错误覆盖。 | +| I06 · `heal.kms-and-tier-outage` | P1 | H2 / DE,DV / F11 | 仅让已分配测试 KMS/远端 tier 暂不可用，持续查询与修复 | 依赖故障不得解释为对象不存在/可删/已修复，不暴露密钥；恢复后原版本可解密或恢复，责任未丢失。 | + + + +### J. 可观测性、幂等、长稳与性能 + +| 编号 / 稳定 ID | P | 拓扑 / 夹具 / 故障 | 操作与触发 | 附加通过判据 | +| --- | --- | --- | --- | --- | +| J01 · `heal.status-partial-disabled-unknown` | P0 | H1,H2 / DB / F0,F6 | 分别检查 enabled/disabled/uninitialized、全体可达及缺失 peer 的状态 | idle 只在覆盖完整且真实无活动时成立；partial/unknown/degraded/disabled 不被客户端转换为全部完成。 | +| J02 · `heal.outcome-counters-and-error-truth` | P0 | H1,H2 / DB,DV / F1,F2 | 同批次混合健康、可修、不可修、deferred、dry-run 对象 | execution/coverage/disposition/计数和逐对象物理结果相符；unknown、skipped 或 completed_with_errors 不能算修复成功。 | +| J03 · `heal.incremental-cursor-and-truncation` | P0 | H1,H2 / DS,DX / F1 | 生成超过结果保留上限的任务，分页轮询 sinceSeq，并跨节点/终态读取 | nextSeq/minSeq 单调且 gap/truncation 明示；有界 items 不是完整责任清单，最终须用独立 manifest/计数补证，不静默丢错误。 | +| J04 · `heal.repeat-idempotency` | P1 | H1,H2 / DB,DV / F1 | 第一次修复后重复 normal/deep、重启再检查 | versioned 首次允许所有 identity repaired 或至少一个 repaired 加其余正健康态，marker 仅 repaired/metadata_healthy，计数必须与 receipt 集合一致；normal repeat 接受 `metadata_healthy`，deep repeat 要求 `verified_healthy`，repeat marker 为 `metadata_healthy` 且 healed=0；unknown/skipped/failed 不得成为 PASS。 | +| J05 · `heal.long-soak` | P2 | H2,H5 / DX,DL,DV / F4,F6,F11 | 独占环境运行预注册 24h/72h 窗口，混合修复、掉线、写入和版本删除 | 每次 ACK 可追踪；最终全量 manifest 正确，队列/内存/FD/临时空间无无界增长，无后台 panic 或未解释持久责任。 | +| J06 · `heal.performance-abba` | P2 | H2 / DX,DL / F1,F11 | 同硬件同候选按 ABBA 运行无 heal 基线和有 heal 负载，重复并固定限速 | 分别报告修复吞吐、GET/PUT P95/P99、错误率和 CPU/IO；阈值事先冻结，不能事后挑轮次或把功能 PASS 充性能验收。 | + +## 与主仓库既有验证的复用关系 + +以相同源 SHA 的实现为准，先读脚本的 scope、环境要求和故障 oracle，再适配到固定 fleet；**不直接在共享机器运行原 E2E 的清盘/建进程脚本**。已有覆盖只是复用线索，不能直接标记本目录 planned 用例为 implemented。 + +| 已有资产 | 可复用内容 / 本矩阵对应 | 不能混淆的边界 | +| --- | --- | --- | +| `crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs` | runtime wiped disk、Admin deep、remote replacement，B/C/E/F | 进程 E2E 与固定真机、同节点目录盘与独立 XFS 物理盘需分别标记；检查其 skip 分支，必选故障能力缺失不能通过。 | +| registry 的 background-target-restart / crash | F02/F03、目标部分进度与 replacement shard 证据 | SIGKILL 不是断电；原夹具无版本覆盖不能自动扩张为 DV 已覆盖。 | +| registry 的 EC8+4 single-set / multi-set / multi-pool 项 | H07/H06 与相应 F02/F03 参数 | 精确盘数、set/pool 分布与同一故障窗口要匹配，4 盘 PASS 不能覆盖 G14。 | +| W13 durable MRF 与 W16 recovery-intent/quota 证据 | E05、F07、I02、J02 | 责任 ACK、恢复意图、配额与终态内容都需要原始实测输入，不只运行 JSON assembler。 | +| G05/G06 status/outcome、G02 checkpoint、G09 upgrade 证据生产脚本 | A03、J01-J03、F07/F08 | schema 自检、descriptor 组装和真实故障执行是三种不同证据，不可互相代替。 | +| ECStore/heal 原生 runtime-fence、版本删除/覆盖竞争回归 | A09、C07、G01-G06、I05 | 原生可控屏障覆盖精确窗口；没有在 fleet 触发该窗口时只记录功能覆盖，不把两类日志混为真机因果证明。 | + +来源：[必选 E2E registry](https://github.com/rustfs/rustfs/blob/940e221988694da33fe005054bb86086bae88cd6/.config/scanner-heal-required-tests.json)、[磁盘重建 E2E](https://github.com/rustfs/rustfs/blob/940e221988694da33fe005054bb86086bae88cd6/crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs)、[Linux evidence plan](https://github.com/rustfs/rustfs/blob/940e221988694da33fe005054bb86086bae88cd6/scripts/run_scanner_heal_linux_evidence_plan.py)、[脚本分类](https://github.com/rustfs/rustfs/blob/940e221988694da33fe005054bb86086bae88cd6/scripts/README.md)。其中 plan/preflight 不产生 measured evidence，assembler 不替代采集。 + +## 当前框架的执行方式 + +Functional Chain 在共享节点并发锁内调用 auto-testing 的 canonical dispatcher。普通 live 执行消费候选 deb 和工具产物，无需 RustFS checkout；只读 metadata inspector 和原生屏障测试须在构建阶段绑定精确候选源码，单独留证。 + +尚未自动化的分支由 `matrix.py plan` 固定全部适用拓扑、源断言、操作顺序、参数绑定与恢复步骤。`matrix.py audit` 校验九类证据的摘要和分支完整性，结果仍须逐项审阅，绝不自行生成 canonical PASS。 + + +正向 H1/H2 夹具显式启用 `RUSTFS_SHARD_INTEGRITY_WRITE=true` 和 `RUSTFS_SHARD_INTEGRITY_FLEET_CONFIRMED=true`。该确认只针对本例私有 endpoint 集合：所有参与节点先安装并校验同一候选二进制，再启动专用服务；配置从真实进程环境读回。候选默认值为 false，默认 legacy 对象的 unknown 回执已单独保留为失败证据，不能改写为 repaired，也不从保护模式推断默认部署已满足这些正向证明契约。 diff --git a/docs/validation-heal-environment-gates.md b/docs/validation-heal-environment-gates.md new file mode 100644 index 00000000..6486284e --- /dev/null +++ b/docs/validation-heal-environment-gates.md @@ -0,0 +1,56 @@ +# Heal 环境门控与原生阻断 + +## Read-only fleet inspection + +The Functional Chain `environment.inspect` entry records the current four Heal hosts' mount capabilities, fixed `rustfs.service` status, loaded validation service names and listening PIDs on port 9000. Service and listener observations must agree across two samples. Process arguments, service descriptions and credential files are excluded. Inspection never stops services or changes retained scenes. + +These observations do not certify a clean environment. Independently audit retained validation scenes and the complete shared-workflow census before dispatching another case. A completed workflow's success or failure alone does not prove recovery. The regular case installer remains responsible for stopping the standard service, and its existing namespace/port reservation guards remain authoritative. + +When the shared runner cannot start `environment.inspect`, the separate `validation_runner_inspect_only=true` entry inspects GitHub registration and queue metadata on a hosted runner without fleet access. Inspection [40256930895/1](https://github.com/rustfs/rustfs/actions/runs/40256930895) observed `rustfs-smoke-testing` (ID 139653) offline at `2026-10-26T02:00:24Z`. This is not a diagnosis of host or disk health. API failures remain incomplete; restoring registration does not certify retained-scene cleanup or release the existing live admission gates. + +第一批 13 个用例保留全部 recipe、分支和 source assertions，并新增 `environment_cases.py` 的控制流程与断言，接入 validation 调度。它们默认关闭，只产生 `BLOCKED` 结果与未满足原因；默认路径不会连接或修改 fleet。真实执行沿用 auto-testing 的 S3/Admin collector、ACK ledger、候选包绑定、物理故障 oracle 和恢复模式，专用资源操作由部署环境的适配器提供。当前没有把适配器协议或未运行代码计为 canonical PASS。 + +设置 `AUTO_TESTING_ENABLE_ENV_GATED_HEAL=1` 并通过 `VALIDATION_HEAL_ENVIRONMENT_CAPABILITIES` 指向 JSON 能力清单后，门控会检查全部能力。缺项仍返回 `BLOCKED`。能力齐全后还要用 `VALIDATION_HEAL_ENVIRONMENT_RUNTIME` 指向 runtime JSON；只有候选/checkout/client 摘要、全部拓扑、当前 Heal SSH 节点和适配器摘要校验通过后才会运行分支。未绑定适配器仍为 `BLOCKED`；执行结果需要 source assertion 审核，当前实现不会自动生成 canonical PASS。 + +能力清单示例： + +```json +{"capabilities": ["dedicated_faultfs_or_device", "slow_io_short_read_eio", "namespace_isolation", "fault_layer_restore"]} +``` + +| Code | 用例 | 当前未满足原因 | 启用条件 | +| --- | --- | --- | --- | +| F08 | `heal.mixed-version-upgrade-rollback` | 没有经过审核的不可变双版本组合 | 两个兼容 release、混合 peer 状态、滚动升级/回滚和格式兼容矩阵 | +| F09 | `heal.dedicated-host-forced-restart` | 共享 Heal 节点不具备 H6 专用主机及平台断电能力 | 专用 H6、平台 power-cycle、带外恢复、boot_id/平台事件证据 | +| G07 | `heal.race-ilm-expiry-and-transition` | 缺隔离 ILM/tier 与真实操作交叠窗口 | 专用租户、Expire/Transition/Restore 窗口、远端版本映射、规则清理 | +| G08 | `heal.race-replication-resync` | 缺独占复制两端与 resync 控制 | 隔离复制对、resync、跨端版本映射、复制/Heal 交叠证据 | +| H01 | `heal.with-expansion-rebalance` | 缺完整 H5 扩容/rebalance 拓扑 | H5 扩容、rebalance task、逐对象 pool/set 映射、scanner 退休证据 | +| H02 | `heal.with-rebalance-stop-resume` | 缺独立 rebalance 停止/恢复/重启控制 | H5 rebalance、独立 task、stop/resume 和 service restart | +| H03 | `heal.with-decommission` | 缺可回滚的退役拓扑 | H5 decommission、survivor pool、写入限制、移动 ACK 与物理权威证据 | +| H04 | `heal.with-decommission-cancel-clear` | 缺退役新代次与容量锁重试窗口 | cancel/clear/resume、#2423 锁窗口、代次绑定 ACK ledger | +| H05 | `heal.after-retired-pool-removal` | 缺 retired pool 移除后的 survivor-only 拓扑 | retired pool 配置移除、源持续离线、survivor 权威、旧 pool 写保护 | +| H07 | `heal.ec84-multi-set-restart` | H4 的至少 24 个独立 endpoint 不可用 | H3 12 独立盘、H4 24 endpoint、逐 set 身份与重启/崩溃证据 | +| I01 | `heal.slow-io-and-eio` | 没有专用 faultfs/故障设备 | 专用故障层、slow/short-read/EIO、namespace 隔离、原状恢复 | +| I06 | `heal.kms-and-tier-outage` | 没有专用 KMS/tier 租户与依赖隔离 | 独占租户、依赖网络隔离、key 脱敏、tier 版本映射 | +| J05 | `heal.long-soak` | 缺可持续 24h/72h 的独占 lane 与资源预算 | 真实墙钟窗口、限速和监控、循环故障、到期回收；短跑不能替代 | + +`matrix.py list` 同时输出 `automaticExecutor`、`dispatchable` 和 `executionPolicy`：67 个 Heal 自动执行器、13 个环境门控、4 个证据/执行器阻断。调度覆盖共 80 个 Heal，连同 74 个非 Heal 为 154/158；这不代表 154 个 live PASS。 + +第二批 4 个用例仍未进入 canonical 调度。C11/F07/J02 已有 candidate-bound 原生回归和显式诊断入口；诊断分支通过仍需完整矩阵和独立审计后才能解除门控。I05 仍缺精确服务端屏障，增加节点不能解决： + +| Code | 用例 | Block 原因 | 解锁所需证据 | +| --- | --- | --- | --- | +| C11 | `heal.bucket-incarnation-recreation` | H1/H2 旧元数据盘重返、原始 checkpoint/resume 与 committed MRF 重放编排已实现；精确候选的 9 项存储原生回归和 3 项 manager incarnation 前置证据已独立核验。完整现场分支尚无独立审计结果 | 旧/新 incarnation、正常 S3 删除与新 ACK、物理范围和原生 artifact 精确绑定；单分支不提升整例 | +| F07 | `heal.checkpoint-and-mrf-corruption` | 已实现 H1/H2 checkpoint 截断、未来 schema、committed MRF CRC 故障及独立安全重扫；10 项原生回归作为前置证据。真实较早代次回放与私有限额卷的 journal ENOSPC 已实现；仍缺完整 native cut 审核，完整 live 尚未审计 | 真实 durable 记录故障与 publication 证据绑定；现有 interruption hooks 不等于每个 fsync 或断电 | +| I05 | `heal.lock-timeout-and-caller-abort` | 同源码 native 子集已保留；H1/H2 timeout 与 queued-successor 诊断已通过独立 live 审计（39767336741/1、39817606025/1），完整用例仍 BLOCKED | 实际断连边界所有权、完整 native timeout/stale 屏障、stale-generation controller 和 H1/H2 DL/DV 完整对象/ACK/物理/锁/FD 闭环 | +| J02 | `heal.outcome-counters-and-error-truth` | 全部夹具与 manifest 对账均已实现。候选 `7951bb629821246868ee3761b321dd07c06db45c` 修复整桶重试覆盖 repaired 记录，210 项 manager 与 5 项独立进程集成测试通过。新实机 `38874900846/1` 完成 20 个分支：16 个报告通过，4 个原始全元数据损坏探针返回永久 Failed，保留 NOT_OBSERVED；四组混合任务与全部 50 条节点清理记录已独立审计。真实 legacy Unknown 已观察，不能代替原始探针 | 原始元数据损坏分支的 Unknown 仍未观察；完整 source-procedure 验收尚未闭合。原始 Failed、NOT_OBSERVED 和完整用例门控保持不变 | + +I05 与 A09、G01–G06 的精确屏障证据允许使用同一不可变候选源码单独构建的 `cfg(test)` 原生测试执行文件。不得给优化服务候选加入隐藏 hook 冒充原包，也不能把缺少 release hook 本身归为外部环境阻塞。原生子集、完整服务调用和现场最终 ACK/物理检查分别验收；已验收 native 证据不能代替尚缺的 controller 或完整 live 结果。第一批 G07/G08 的 native 分支与 H04 容量锁窗口保留各自未完成条件；外部请求重试或最终对象正确均不能证明精确锁或提交窗口。 + +2026-10-26 已独立复核 hosted-only tools [40086026324/1](https://github.com/rustfs/rustfs/actions/runs/40086026324)：候选 `757b751482bb970bc321e92059690a7698554358` 上 10 项精确原生测试和 104 条有序 milestone 全部通过，覆盖 A09、G01–G06 与 I05 的原生子集。固定 helper `002473b` 的 runtime admission 在该候选上仅开放 A09/G05/G06；G01–G04 仍须选择各自审核过的候选及匹配工具，或先完成新的 runtime 源码审核。完整 live 用例和 I05 剩余控制流程均未因此通过，canonical 计数保持 114/158。 + +资源适配器协议使用 stdin/stdout JSON，不调用 shell。runtime 中的 `adapter` 包含绝对可执行路径、SHA-256 和操作 allowlist；`environment_cases.required_operations(case_id)` 给出所需操作。每个请求携带 run ID、原候选 commit、sequence、拓扑/分支 scope、bucket prefix、租约和参数。响应必须回显上述身份、按 `heal_probe.canonical(request)` 的 UTF-8 SHA-256 绑定原请求，并把真实观察写在 `observation`，禁止返回 verdict。资源适配器需独立实现平台 power control、故障层、pool lifecycle 或 KMS/tier 隔离；本 PR 未向当前共享环境部署适配器。 + +runtime 的 `binding` 使用现有 `heal_probe.validate_binding` 契约，另含 `schemaVersion: 1`、`provider: auto-testing`、`sshNodes`、`rcBinary`、`topologies`、`capabilities`、`adapter` 和对应资源字段。F08 需要 `releasePair`，F09 需要 `hostResourceId`，I01 需要 `faultDeviceId/ioLatencyMs`，G07/I06 需要 `kmsKeyId/tierEndpoint/tenantId/dependencyNamespaceId`，G08 需要 `replicaEndpoint`，J05 需要 `soakLimits`。H6 专用主机应由适配器按 `hostResourceId` 独占绑定；SSH fleet 仍记录当前 auto-testing Heal 节点。secret 通过环境传入，适配器诊断会脱敏，`redactionEnvironmentVariables` 可指定额外 secret 环境变量名。 + +J05 的实际控制循环保留固定 86400/259200 秒窗口、精确版本写入/删除 ACK、三类循环故障、恢复、资源阈值和到期全量校验。单元测试用可注入的时钟验证窗口控制，不修改生产时长。G07/G08 精确 native 分支不调用资源适配器。清理失败保留原失败与 DIRTY 原因，停止后续分支。 diff --git a/docs/validation-heal-operations.md b/docs/validation-heal-operations.md new file mode 100644 index 00000000..71247e0e --- /dev/null +++ b/docs/validation-heal-operations.md @@ -0,0 +1,1130 @@ +# Heal 真机执行手册：84 项操作卡的公共命令 + +此手册和 [84 项逐例操作卡](validation-heal-cases.md) 一起构成可执行的**人工操作方案 + 脚本辅助**。不是 84 个无人值守自动测试，也不是通过报告。操作卡列出 302 个分支模板；真实盘角色、阈值、损坏数量和拓扑仍需在每批 `parameterBindings` 展开，不能把模板数量当实测次数。 + +公共验收契约见 [Heal 规范](validation-heal-contract.md)。源仓库历史 pilot 结果不计入当前 canonical 分母。自动入口 `scripts/run_validation_case.sh CASE_ID CANDIDATE_MANIFEST TESTING_COMMIT ARTIFACTS_DIR` 已接入 67 个 Heal 用例，完整清单通过 `matrix.py list` 查看，以 `automaticExecutor` 为准。入口使用当前 Heal 节点、候选 deb 和独立 H1/H2 服务；其余操作卡仍需按本手册操作留证。 + +## 1. 使用入口与交付物 + +```bash +python scripts/validation_heal/matrix.py list +python scripts/validation_heal/matrix.py show C02 +python scripts/validation_heal/matrix.py show F02,F03,H07 +python scripts/validation_heal/matrix.py plan --selection all \ + --run-id manual-heal-20260910-a --out reports/validation-heal-84-plan.json +python scripts/validation_heal/matrix.py plan --selection A01,A04,C01,C02,C04,J04 \ + --topologies H1 --run-id manual-heal-20260910-b --out reports/heal-h1-plan.json +``` + +`plan` 只冻结选择/分支和对应操作，不部署、不评分。`show` 同时给出步骤、命令入口、验收和恢复要求。不同拓扑是不同实例；未选拓扑会保留 `scopeExclusions`，不能据局部结果声称全矩阵通过。 + +自动用例通过 RustFS Functional Chain 的 `validation_case` 和精确 `auto_testing_sha` 分发。先用同一不可变候选的 `build_run_id`/`build_run_attempt`、`validation_tools_only=true` 构建只读 metadata inspector；live 分发提供该成功构建的 `validation_tools_run_id` 与 `validation_tools_run_attempt`。工具的 candidate/helper SHA、Linux 二进制 SHA 和独立 Cargo.lock 摘要均须通过检查，再复制到本例节点。普通 live 执行消费构建产物，不需要 RustFS checkout。 + +| 工具 | 已实现的职责 | 不代表什么 | +| --- | --- | --- | +| `matrix.py` | 84 项/302 分支索引、逐例操作卡、冻结计划、证据完整性审计 | 不自动分配拓扑，不根据文件存在就判 PASS | +| `heal_probe.py` | 带签名/匿名/坏签名的原始 Admin；受限 S3 操作；全版本/标签/内容摘要清单及比较 | HTTP 检查通过不代表 heal 成功；inventory 的 GET 会触发读修复 | +| `heal_workload.py` | DB/DV/DM/DS/DX 真实夹具；有界并发、同key前台修改串行、请求前及ACK后持久落盘 | 不替用户推导竞态后的最终版本；超时写必须单独解析 | +| `heal_fault.py` | Linux 描述符相对寻址、mount ID 保护、单文件 missing/bitflip/truncate/zero/header、快照和恢复 | 文件故障不是物理掉盘；不修改系统 metadata；不盲目恢复新代次 | +| `prepare_heal_meta_inspector.py` | 为精确候选源码准备只读 Rust metadata 解析器工程 | 需在批准的构建机编译；解析 metadata 不是验证 bitrot/EC | +| `shards.py` | C01–C04、D02、D07、E01、E02 全 24 分支以及 A03 的两个 H2 跨节点状态分支、A04 的六个 DB/DV dry-run 分支；静默故障、真实角色、同角色原片与 HighwayHash、先物理后 GET/Range、MPU control | 有自动入口不等于候选通过；任务 unknown、目标只恢复 metadata 或救援备份均不能记 PASS | +| `check_bitrot_tool.py` | 在固定 MinIO/历史 RustFS 产物上检查 24 个健康片和 12 个坏片负例 | 工具验证不等于 live 用例通过；不调用被测 RustFS 服务 | + +## 2. 先绑定环境，禁止直接猜节点 + +live 命令使用当前 auto-testing 的 `smoke-testing` runner、`RUSTFS_NODES` 和 `RUSTFS_SSH_USER`，复用 `rustfs_heal_test.sh` 的 Heal 节点。运行前取得与 Functional Chain 一致的共享节点并发锁。H1–H6 是测试拓扑要求，不是另一组 SSH 主机；不得按源仓库历史主机名或 IP 选择机器。 + +### Runner 无法接单时的只读诊断 + +RustFS Functional Chain 提供 `validation_runner_inspect_only=true` 入口。它在 `ubuntu-latest` 上读取 GitHub 的 runner 注册信息与活动任务，使用独立并发组，不占用或替换测试 fleet 队列。一次请求只能选择该模式，不同时传入 candidate、tools、case 或 recovery 参数；使用包含此入口的已审核 workflow ref： + +```bash +gh workflow run rustfs-functional-chain.yml --repo rustfs/rustfs \ + --ref "$VALIDATION_DRIVER_REF" -f validation_runner_inspect_only=true +``` + +下载该次 `validation-runner-inspection--` artifact，核对 `workflowCommit`、`runId`、`runAttempt`、`observedAt`、`complete` 和 `runnerAvailability`。`offline`、`online-busy`、`online-idle`、`unregistered` 是 GitHub 注册状态的不同观察；API 拒绝、分页超过上限或其他错误保留 `complete=false` 与失败阶段，不能解释为空闲。报告不连接节点、不重启服务、不清理文件、不取消 workflow，也不证明环境已恢复或共享 workflow 全局空闲。runner 服务恢复后，仍需重新执行资源检查及原有现场/并发门控。 + +用 `matrix.py plan` 冻结当前 SSH 节点与所有适用操作分支，再根据本次只读 preflight 填写私有 `lab.json`。计划中的 `verified:false` 不可直接改成资源验证通过。H4 需要至少 24 个已分配独立盘端点；H6 需要独占主机及平台事件/恢复绑定。缺少资源的分支保留为 BLOCKED。 + +填写内容须包含： + +- 本批 run ID、截止时间、独占服务/端口、jump route、实际主机名、每个磁盘的 mount UUID/major:minor、服务 unit/原 PID/数据根； +- 与当前 validation lane 相同的不可变 candidate manifest、源码 commit、包与已安装二进制 SHA-256；当前 rc 路径与二进制 SHA-256；auto-testing commit；保存真实构建清单，不只填摘要字符串； +- H1–H6 实際形状、每个对象 N/data/parity/R/W/lock quorum、pool/set/slot/盘 UUID；H4 至少 24 个实际盘端点； +- 最长修复/重启/故障时限、对象/字节/请求/并发预算、所有危险动作的精确目标及恢复人； +- 包含平台操作、独占故障设备、旧版本、KMS/tier 或原生屏障的实例，先提供能力及授权证据。 + +`binding-check` 只是语法/租约检查，不是资源或包身份的远程证明： + +```bash +python scripts/validation_heal/heal_probe.py --binding lab.json --out reports/binding-check.json binding-check +``` + +先在跳板只读核验 `hostname`、`findmnt -J`、`lsblk -J`、`ss -lnt`、二进制 `sha256sum`/`--version`、实际运行配置与 PID。保存输出为 preflight。租约没有发放、拓扑不符、端口占用、磁盘 UUID 漂移时在这里 BLOCKED，不先注入后跳过。 + +### 独立实验服务的部署 + +H1 使用当前 Heal 节点经核验的四个独立 mount，**另建** `/data/rustfsN/heal-validation//data` 和同级 `quarantine`，root-owned 0700；每个 `` 根写入包含 `runId` 的 root-owned `owner.json`。普通对象故障可以使用这些独立父盘下的 run 私有目录。B03 和 B04 还要求替换 target 和同组兄弟 endpoint 都能取得 candidate 认可的精确 mount lease；替换过程中必须证明旧 mount 与 quarantine 对服务不可读、新空设备挂到原 endpoint 后 candidate 取得新的 mount lease，并前后记录 mount ID、物理设备 ID 和 filesystem identity。独立文件系统或隔离 mount namespace 中的 bind mount 都可能作为实现方式，但仅创建 bind mount 不足以满足要求：adapter 还必须安全切换被替换 endpoint，且通过 candidate 对 root 和 sibling 物理设备身份的检查。Functional Chain 会先在 `stop_existing` 和候选 deb 下载/安装之前调用只读 `replacement-admission-preflight`，覆盖 H1/H2 endpoints；未通过时以 `BLOCKED` 保留 package/client provenance 和完整现场读数，不停服务、不碰已安装 package，也不创建远端 run namespace。`Fleet.deploy()` 在真正 reservation 前再重复该检查。缺少真实 endpoint mount 或已审核 mount-switch adapter 时应 BLOCKED；当前 runner 只把 run 私有子目录作为 endpoint，尚无此 mount-switch adapter，因此它现有的四个独立父盘不能证明 B03/B04 满足资源条件。不得延长等待时间或复制 quarantine 数据。先检查每个根不存在；已存在的根不能覆盖。复用 `rustfs_heal_test.sh` 的 candidate deb 下载、SHA 校验和安装流程，把已安装且摘要已核验的 `/usr/bin/rustfs` 放在本批 state 下；执行用例不需要 RustFS 源码 checkout。凭据文件为 0600。以下专用 unit 仍运行在当前 Heal 节点，使用已分配的 9000 测试端口： + +如果首次 gate 通过后、`Fleet.deploy()` 的 reservation-time 复检发现 mount 漂移，仍记录 `BLOCKED` 并保留已安装候选的 binary provenance；不得把该结果改写成产品 `FAIL`，也不得把它伪装成 pre-deployment 的 `NOT_REQUIRED_NO_MUTATION`。两阶段检查都必须跳过 reservation 和 fixture seed，并完成 cleanup。 + +```bash +# 在目标节点；所有变量来自已审核的 lab binding，不接受任意在线输入。 +sudo systemd-run --unit="$HEAL_UNIT" --property=Type=exec \ + --property=PrivateMounts=yes --property=KillMode=control-group \ + --property=TimeoutStopSec=30 --property=RuntimeMaxSec=3600 \ + --property=MemoryMax=2G --property=CPUQuota=200% \ + --property=WorkingDirectory="$HEAL_STATE" \ + --property=EnvironmentFile="$HEAL_STATE/service.env" \ + "$HEAL_STATE/rustfs" "${HEAL_DATA_ROOTS[@]}" +``` + +需重启的场景保留 unit 定义、env、二进制及原始 endpoint 参数，不使用 `--collect`；RuntimeMaxSec 按该批预算冻结，J05 必须单独分配长时服务。`HEAL_DATA_ROOTS` 是已审核的 Bash 数组，不从未审核字符串 eval。H2/H3/H4/H5 用各节点一致的 endpoint 序列/显式 pool 分组配置；启动后从每盘 `format.json` 读回，不能从启动参数猜拓扑成功。不得直接运行会清理旧 Pool 根的部署/reset 命令。 + +### 命令约定 + +下列函数在**跳板机**执行。`HEAL_BINDING`、`HEAL_REPORT` 是该实例私有绝对路径；`B` 是绑定 prefix 下的一个桶，`K` 为逻辑 key，`V` 为先前 ACK 的精确版本。每条命令的 label 唯一，已有证据拒绝覆盖。凭据只通过受控环境注入 `HEAL_ACCESS_KEY`/`HEAL_SECRET_KEY`/可选 session token，禁用 `set -x`。 + +```bash +set -euo pipefail +umask 077 +hp() { + local label=$1 + shift + python scripts/validation_heal/heal_probe.py --binding "$HEAL_BINDING" \ + --out "$HEAL_REPORT/$label.json" "$@" +} +hw() { + local label=$1 + shift + python scripts/validation_heal/heal_workload.py --binding "$HEAL_BINDING" \ + --out "$HEAL_REPORT/$label" "$@" +} +B=$(jq -er .bucketPrefix "$HEAL_BINDING") +RUN=$(jq -er .runId "$HEAL_BINDING") +``` + +特权故障命令在**经跳板连接的目标节点**执行。`ROOT` 是本例目标盘 `` 根；`NODE` 是已绑定 SSH alias；把该 SHA 的 `heal_fault.py` 放在 root-owned 的本批 state 下并核对摘要。不得从逻辑 key 拼 SSH shell。以下 `hf` 仅在该目标节点的交互 shell 定义： + +```bash +hf() { sudo python3 "$HEAL_STATE/heal_fault.py" --root "$ROOT" --run-id "$RUN" "$@"; } +hf init --bucket-prefix "$B" --lease-expires-at "$LEASE_EXPIRES_AT" +``` + +初始化只增加故障所有权标记，不创建/格式化数据盘。每盘只 init 一次。系统重启导致 mount ID 变化时暂停注入；先重新核对 UUID/major:minor/所有权并人工重新绑定，不绕过检查。 + +## 3. 数据与客户端命令 + + + + + + + +### 确定性夹具 + +每个命令使用新桶，禁止在现有夹具桶上重复 seed 后称同一轮。 + +```bash +hw seed-db seed --fixture DB --bucket "$B-db" +hw seed-dv seed --fixture DV --bucket "$B-dv" +hw seed-dm seed --fixture DM --bucket "$B-dm" +hw seed-ds seed --fixture DS --bucket "$B-ds" +hw seed-dx seed --fixture DX --bucket "$B-dx" +# 替换为候选确认的真实 inline/block 边界；可另外生成一桶： +hw seed-boundary seed --fixture DB --bucket "$B-boundary" \ + --count 128 --sizes "$BOUNDARY_SIZES" +``` + +DB 默认 128 项，包含 0/1/4095/4096/4097、1MiB±1、多MiB，非重复 SHAKE 数据；1MiB 不是预设产品阈值。DV 默认 4 key，每 key 三个版本，交替创建当前 marker。DM 有已完成三 part（前两 part 分别 5MiB+17、5MiB+93，末 part 1MiB+47）及两个独立 pending upload。DS 默认 257 特殊/多前缀 key；DX 默认 257 外部对象。自动 seed 不包括 suspended/null、特殊压缩、SSE/retention 等额外分支，按下列 S3 命令补充并保存 ACK。 + +从 ACK 文件取得真正的 UploadId/VersionId/part ETag；不要在多个上传之间复用猜测的 ID。总字节预算包含所有版本/part，不足时先扩展经批准的预算，不自动跳过对象。 + +后续命令的 `B` 必须改为该实例实际桶，例如 `B="$(jq -er .bucketPrefix "$HEAL_BINDING")-db"`，不要对没有创建的裸 prefix 做 inventory。根目录 owner 仍绑定原始 run prefix，以允许本run的各后缀桶。 + +seed 完成后自动以完整 GET/版本清单对账确定性原文与创建 ACK，保存 `expected-fixture.json`、`baseline.json`；没有通过基线校验就不准进入注入。这个 GET 发生在故障前，不可拿 seed 命令监测 scanner/read-repair 故障窗口。 + + + + +### 单个 S3 操作与精确版本 + +```bash +hp put s3 --operation put_object --size 5243003 --seed "$RUN/new/1" \ + --params "$(jq -nc --arg b "$B" --arg k "$K" '{Bucket:$b,Key:$k,ContentType:"application/octet-stream",Metadata:{nonce:"1"}}')" +hp get s3 --operation get_object \ + --params "$(jq -nc --arg b "$B" --arg k "$K" --arg v "$V" '{Bucket:$b,Key:$k,VersionId:$v}')" +hp suspend s3 --operation put_bucket_versioning \ + --params "$(jq -nc --arg b "$B" '{Bucket:$b,VersioningConfiguration:{Status:"Suspended"}}')" +hp delete-version s3 --operation delete_object --expect-http 204 \ + --params "$(jq -nc --arg b "$B" --arg k "$K" --arg v "$V" '{Bucket:$b,Key:$k,VersionId:$v}')" +``` + +请求复杂时 `--params-file` 使用 JSON。接口参数为 boto3 snake_case 操作名、原生 CamelCase 参数；只允许本 run 桶、受限操作，copy 源也必须在本 run。GET 只保存完整内容的 hash/长度，不把对象内容写入日志。PUT/Complete 超时记 resultUnknown，下一步用相同唯一 key/version/UploadId 查询解析，不静默重试。 + +`--expect-http` 必须使用该请求预注册的精确值。例如 create/put/list 常见 200，delete/abort 常见 204，Range 206；负例按候选契约分别 304/403/404/405/412/416。CLI 返回 0 只说明该次预期 HTTP 匹配，永不宣称整个 case PASS。SSE-C 参数值可使用字面量 `$HEAL_SSE_C_KEY`，脚本只在内存替换，输出脱敏。 + + + + +### 全版本基线和最终核对 + +```bash +hp baseline inventory --bucket "$B" +# 先完成触发归属和 PHYSICAL，再调用下面 GET 型检查： +hp post-heal inventory --bucket "$B" +hp s3-comparison compare --before "$HEAL_REPORT/baseline.json" \ + --after "$HEAL_REPORT/post-heal.json" +``` + +inventory 分页遍历全部版本/marker，按每个 versionId GET 全内容、tags 和公开 metadata，再用分页 current LIST 交叉核对；对象上限 10,000、单对象 64MiB，批次字节预算来自绑定文件。更大对象场景需单独预注册流式工具和预算，不能截断后称全量。 + +并发、生命周期、合法删除场景的预期清单会变化：人工先按逐请求 ACK 因果关系生成 `expected-post.json`，记录每项增删对应的 ACK 文件；再比较实际清单，不能直接拿静态基线判失败或拿实际输出复制成期望。未知写入未解析则不可通过。加密/归档对象先用授权 GET/restore 参数逐版本取证；默认 inventory 不会自动解锁、传 SSE-C 或 restore。恢复型 GET 必须保存密文/明文两层对应关系。 + + + +### Range、HEAD、条件请求 + +```bash +hp head s3 --operation head_object --params-file request-head.json +hp range s3 --operation get_object --expect-http 206 --params-file request-range.json +hp if-none-match s3 --operation get_object --expect-http 304 --params-file request-if-none-match.json +hp if-match-reject s3 --operation get_object --expect-http 412 --params-file request-if-match-bad.json +hp range-outside s3 --operation get_object --expect-http 416 --params-file request-range-outside.json +``` + +JSON 都包含本例 Bucket/Key/可选 VersionId；另外分别添加 `Range:"bytes=5242879-5242882"`、`IfNoneMatch:<原ETag>`、`IfMatch:<错误但语法合法ETag>`、超出对象长度的 Range。时间条件用 `IfModifiedSince`/`IfUnmodifiedSince` 的 ISO 日期；从 HEAD 的真实 LastModified 推导早/晚两个控制值。验证 ContentRange、响应长度和从确定性原文计算的片段 hash，不能只验状态码。 + + + + + +### SSE、Object Lock 和外部依赖 + +```bash +hp sse-s3-put s3 --operation put_object --size 5243003 --seed "$RUN/sse" --params-file sse-s3-put.json +hp sse-kms-put s3 --operation put_object --size 5243003 --seed "$RUN/kms" --params-file sse-kms-put.json +hp sse-c-put s3 --operation put_object --size 5243003 --seed "$RUN/ssec" --params-file sse-c-put.json +hp locked-bucket s3 --operation create_bucket --params-file locked-bucket.json +hp retained-put s3 --operation put_object --size 5243003 --seed "$RUN/lock" --params-file retained-put.json +hp retention s3 --operation get_object_retention --params-file protected-version.json +hp hold s3 --operation get_object_legal_hold --params-file protected-version.json +hp reject-delete s3 --operation delete_object --expect-http 403 --params-file protected-version.json +``` + +SSE JSON 分别加 `ServerSideEncryption:"AES256"`、`ServerSideEncryption:"aws:kms",SSEKMSKeyId:<本例专用key>`、`SSECustomerAlgorithm:"AES256",SSECustomerKey:"$HEAL_SSE_C_KEY"`。SSE-C GET 也提供同一密钥，先用原请求证明可解密。所有外部 key/endpoint 是经过审批的测试租户，不复用生产依赖。 + +锁桶 JSON 为 `{Bucket:...,ObjectLockEnabledForBucket:true}`。受保护 PUT 使用 `ObjectLockMode:"GOVERNANCE"` 或 `"COMPLIANCE"`、实际未来 UTC `ObjectLockRetainUntilDate`，或 `ObjectLockLegalHoldStatus:"ON"`。必须验证无 bypass 角色在保留期间真的不能删除；脚本拒绝 BypassGovernanceRetention=true。Legal hold 解除使用 `put_object_legal_hold` 的 `LegalHold:{Status:"OFF"}`，仅作用于本例版本。Compliance 等真实到期，不改主机时间、不内部删文件。 + +KMS/tier outage 使用 NETWORK 在本例专用 net namespace 内仅阻断对应依赖的 IP/port，或仅撤销本例租户凭据；恢复原规则/权限后重试原 version。tier 缺失一个远端版本可通过远端本例桶 API 删除并保留ACK/原备份，但这是“远端对象丢失”负例，不等同网络中断。 + +## 4. Admin 触发、状态与窗口 + + + +实际服务配置要取证，但**不要转储包含凭据的全环境**。manual 模式记录以下非敏感字段： + +```text +RUSTFS_HEAL_ENABLED=true +RUSTFS_SCANNER_ENABLED=false +RUSTFS_HEAL_AUTO_HEAL_ENABLE=false +RUSTFS_HEAL_MRF_ENABLE=false +RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE=false +``` + +scanner/自动/MRF 场景按操作卡逐项开启相应字段，并冻结 scanner cycle/start delay/bitrot cycle。配置变更只改专用 service.env，重启本例 unit 后证明实际生效。read-repair 所需队列依赖按候选源码核实，不能为了“排除后台”关掉被测路径。 + + + + + + + +### 原始 Admin 命令 + +`SCOPE_PATH` 为编码后的 `/rustfs/admin/v3/heal//`；key 的每个字节按 URL 编码，root 为 `/rustfs/admin/v3/heal/`。JSON 中 pool/set 是实际零基索引，不能在 H5 固定填 0。例： + +```bash +hp start admin --path "$SCOPE_PATH" \ + --body '{"recursive":false,"dryRun":false,"remove":false,"recreate":false,"scanMode":2,"updateParity":false,"nolock":false,"readRepair":false,"pool":0,"set":0}' +TOKEN=$(jq -er '.observation.response.clientToken | select(type=="string" and length>0)' "$HEAL_REPORT/start.json") +hp status-001 admin --path "$SCOPE_PATH" --query "clientToken=$TOKEN" +hp stop admin --path "$SCOPE_PATH" --query "clientToken=$TOKEN" --query forceStop=true +hp force-start admin --path "$SCOPE_PATH" --query forceStart=true --body-file heal-options.json +# 保留重复 query，真正送到服务端验证： +hp invalid-duplicate admin --path "$SCOPE_PATH" --query clientToken=a \ + --query clientToken=b --expect-http 400 +``` + +WAIT 用有界循环保存每一响应，不仅保存最后绿色状态；普通 repair 900 秒、重启后 1800 秒是起始测试预算，须预注册，不是产品 SLO： + +```bash +for i in $(seq 1 900); do + label=$(printf 'wait-%04d' "$i") + hp "$label" admin --path "$SCOPE_PATH" --query "clientToken=$TOKEN" + summary=$(jq -er '.observation.response.summary' "$HEAL_REPORT/$label.json") + if [ "$summary" != running ]; then break; fi + sleep 1 +done +``` + +循环退出后人工/候选 adapter 检查：execution completed、coverage complete；failed/unknown/attemptFailures=0；不溢出；逐对象 disposition 与设计匹配；非 dry-run 的 incarnation/版本/pool/set 正确。dry-run 只允许 observed，healed=0，随后无物理变更。负例以预注册错误/未完成契约判，不套正向 completed。 + +`summary=finished` 或 HTTP200 单独不够；`truncated=false` 可以省略，null/true 不算 false；status.settings 是查询默认值，不代表原任务实际 scanMode。outcome 的 128 项窗口不是完整对象清单。当前 rc 会简化部分字段，以 raw Admin + 独立 manifest 为准。 + + + + + +```bash +hp background admin --path /rustfs/admin/v3/background-heal/status +hp replacement admin --method GET --path /rustfs/admin/v4/heal/replacement-recovery +"$RC_BIN" --json --no-color admin heal status "$RC_ALIAS" +``` + +rc 必须是本批冻结二进制，私有 RC_CONFIG_DIR/alias 只指向本例服务，stdout/stderr/exitCode 都留存。查询其他节点：直接运行 heal_probe 的全局 `--endpoint n1`/`--endpoint stable`；保存实际连接 peer 身份。健康状态需要 expected/responded/unknown 与所有 peer 覆盖，不允许缺 peer 时 idle 假绿。 + + + + +### 部分进度与增量游标 + +```bash +hp cursor-001 admin --path "$SCOPE_PATH" --query "clientToken=$TOKEN" --query sinceSeq=0 +NEXT=$(jq -er '.observation.response.nextSeq - 1' "$HEAL_REPORT/cursor-001.json") +hp cursor-002 admin --path "$SCOPE_PATH" --query "clientToken=$TOKEN" --query "sinceSeq=$NEXT" +``` + +WINDOW 的必要证据是 running 且 `0 + +### 权限和过期签名 + +在专用服务先保存当前 rc 的 `admin user --help`、`admin policy --help`、`admin heal --help`。使用候选 IAM 支持的 Admin Heal action 创建最小策略（从候选 `HealAdminAction` 的实际字符串导出，不猜拼写）；另建明确不含该 action 的角色。示例用户/策略动作按本批 rc help 的参数顺序执行并保存原始回执，凭据用私有输入通道，不开 shell trace。 + +已核对参考候选 action 为 `admin:Heal`，最小策略主体为 `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["admin:Heal"],"Resource":["arn:aws:s3:::*"]}]}`；RustFS policy parser 要求带类型的 ARN，裸 `"*"` 会被拒绝。将这个权限仅赋予本例临时身份，不改共享管理员策略。新候选仍需核对 action 和 rc attach 语法。 + +```bash +hp anonymous-start admin --path "$SCOPE_PATH" --auth anonymous --expect-http 403 --body-file heal-options.json +hp bad-signature admin --path "$SCOPE_PATH" --auth bad-signature --expect-http 403 --body-file heal-options.json +# 无权限/最小权限实例：更换本次调用环境中的 HEAL_ACCESS_KEY/HEAL_SECRET_KEY，执行相同请求。 +``` + +过期签名：保存一次已签请求，在签名有效期外重放到**相同本例地址/路径/请求体**，仅等待客户端签名超期，不改变服务器时钟。先用新签名同请求做控制。时限按候选签名校验窗口预注册；签名本身含敏感身份，原请求存私有0600文件，只归档脱敏结果和请求摘要。对 start/status/stop 都测试；token 不能替代权限。结束时通过正常 IAM API 删除本例用户/策略并验证旧凭据失效。 + +## 5. 物理证据与故障后端 + + + + +### 真实布局、角色和物理快照 + +在目标节点逐盘读取本例 `data/.rustfs.sys/format.json`（只读）及 mount UUID/major:minor，保存 pool/set/this 映射。不能把同一设备的多个目录作为独立盘。逐对象 xl.meta 使用与候选相同源码的只读解析器： + +```bash +python scripts/validation_heal/prepare_heal_meta_inspector.py --source "$CANDIDATE_SOURCE" \ + --commit "$CANDIDATE_COMMIT" --out "$INSPECTOR_BUILD" +# 仅批准的构建机；生成独立 helper 的锁文件后冻结，和服务构建清单分开。 +cargo build --release --manifest-path "$INSPECTOR_BUILD/Cargo.toml" +sha256sum "$INSPECTOR_BUILD/Cargo.lock" "$INSPECTOR_BUILD/target/release/heal-meta-inspect" +"$INSPECTOR_BUILD/target/release/heal-meta-inspect" "$XL_META_EVIDENCE_COPY" "$B" "$K" +``` + +解析器不输出用户metadata或密钥，只输出版本、data_dir、parts、EC参数/分片角色。首先对同批健康样本和损坏样本分别证明成功/拒绝；不支持的版本不得换别的版本解析器后宣称同候选验证。源 checkout 必须为固定 SHA 且无已跟踪修改。 + +```bash +# 目标节点。所有输出重定向到本例证据文件，不能覆盖已有阶段。 +hf inspect --path "$TARGET_RELATIVE_PATH" +hf snapshot --bucket "$B" +``` + +快照对整桶每个文件保存长度/hash，目录描述符逐层拒绝 symlink/hardlink/不同mount ID（含同设备bind mount）；最多10,000文件/8GiB，每文件64MiB。四阶段 before/fault/after/restart 每盘各一份。外部part与**同一个角色、同版本的健康原片**字节完全相同是独立重建对照；数据应不重复且四片摘要不同。不得读取 quarantine 来辅助服务器重建。 + +xl.meta 可合法重写；比较解析后的版本/引用，而非要求所有内部字节不变。允许的 xl.meta.bkp 必须属于同目标data_dir且摘要等于原xl.meta，不能泛化豁免任意额外文件。普通解析模式不验证 payload；`--verify-part XL_META BUCKET KEY PART_FILE VERSION_ID PART_NUMBER` 按候选 metadata 的实际版本、part 和 legacy/current framing 逐块验证 HighwayHash，同时拒绝短读、尾随字节和不支持的格式。分片执行器还逐项要求所有同版本片与故障前原始同角色长度/hash 完全一致，并确认基线全文、真实 set slot 及不重复分片摘要。它不支持加密/压缩、inline 或改变编码形态的物理验收；这些用例仍保留各自工具要求。 + + + +### 单文件故障（已有脚本） + +先停止本例写入并让测试进程静止；格式映射已确认后选择精确物理路径，不能只找第一个part。注入/恢复时 `--service-quiescent` 是操作者确认，工具不会替你停止共享服务。 + +```bash +hf apply --path "$TARGET_RELATIVE_PATH" --fault-id C02-data-middle \ + --kind bitflip --offset "$BYTE_OFFSET" --expected-sha256 "$BASELINE_SHA256" --service-quiescent +# 其他实例各用新故障ID：--kind missing / truncate / zero / header +hf snapshot --bucket "$B" +``` + +`truncate` 保留一半；`header` 翻转前32字节；bitflip 可指定任意有效offset；zero 若不会改变字节则拒绝。先持久化intent/原件，再变更，再见证；重复apply拒绝覆盖。备份在同run隔离目录，绝不在RustFS数据根中。 + +文件故障为 F1/F2/对象级F3，不是整盘故障。helper 明确禁止 `.rustfs.sys`、format、IAM、任意shell或设备操作。用例需要系统记录/特殊格式时走下面人工白名单后端。 + + + + + +### 格式/系统记录故障（人工白名单后端） + +先 `PROCESS stop`，从候选源码/真实task定位**本例**精确记录，生成 `record-plan.json`：绝对路径、run/task/bucket/incarnation、盘UUID、原sha/size、变更offset或替换源sha、允许新增/删除字段、备份和undo。第二人/批准的操作人逐项审核；禁止递归 `.rustfs.sys` 修改。 + +可操作的配方： + +1. **near-tail/长度不符**：复制目标part到本例quarantine并校验sha；在该目标副本上 `truncate -s "$NEW_SIZE" "$TARGET"`，其中 NEW_SIZE=原size−1或指定长度。核对新size/hash和原metadata未变。停止服务期间执行，目标路径来自已审核清单。 +2. **外来format/set**：同run的第二个隔离小集群正常启动产生format，停止两服务。仅把其format安装到本例空替换根，不改健康盘；记录原/外来id、this、sets。启动后期待安全拒绝，原外来format不能被改写。 +3. **未来format版本**：在本例替换根的format JSON副本，用 `jq` 只更改已核实的版本字段为不支持值；原件与变更diff归档。未来对象/checkpoint schema 按候选编解码结构更改对应版本头，不能随机翻位后称“未来schema”。如果存在校验，必须重算后证明结构可解析且schema未知；否则只计坏校验实例。 +4. **冲突代次/quorum**：串行合法写V1/V2或两个incarnation，分别保存全套本例副本；停服务后按预注册盘分配表安装各自不足R的集合。重启前用FORMAT逐盘核对，证明没有任一同版本quorum。 +5. **checkpoint/MRF坏记录**：只读保存 `.rustfs.sys/root-heal-.json` 与候选实际journal路径；确认属于本run。截断、替换为从本run早期阶段保留的陈旧代次、或单独更改校验字段。不得凭目录名删除整个queue。写失败通过IO_FAULT构造，不改权限后未经见证就称失败。 +6. **bucket metadata**：只列入对应本例桶的记录；建立同run健康备份，分别缺失/截断后重启/heal。原策略、tags、版本/lock配置通过S3核对，其他系统文件摘要不变。 +7. **inline偏移**：用匹配候选的codec在证据副本中定位目标version的inline map/value；保存解析offset/长度及反向映射，之后只修改该value。不会解析时不能任意翻xl.meta头冒充inline损坏；改用已明确定义的“坏metadata”实例，原inline实例仍未覆盖。 +8. **orphan/dangling**：通过正常完成/覆盖/删除形成历史目录，FORMAT证明所有活version/MPU引用后再测试remove。宽限期按真实时间/本例服务配置等待，不改主机时间。共享引用控制必须保留。 + +上述人工操作每一步均需精确原件备份、变更前后hash、操作返回值与复查；不能直接 `rm` 记录。用 `mv -n` 同盘移到隔离路径可保留原件，但必须核实源消失/目标摘要相同，因为 `mv -n` 跳过未必报错。恢复时先保留失败后新记录，若有新ACK/代次，禁止把旧metadata拷回去。 + + + +### 逻辑盘不可达、只读、权限 + +仅在已设置 PrivateMounts 的**本例服务 mount namespace**使用；先证明目标PID仍属于本例unit、`/proc/$PID/ns/mnt` 与host不同，且目标挂载非shared。不能进入host namespace卸载 `/data/rustfsN`。原理参考 [Linux nsenter](https://www.man7.org/linux/man-pages/man1/nsenter.1.html)。 + +只读配方（在目标节点，DATA为本例完整数据根）： + +```bash +sudo nsenter -t "$HEAL_PID" -m -- mount --bind "$DATA" "$DATA" +sudo nsenter -t "$HEAL_PID" -m -- mount -o remount,bind,ro "$DATA" +sudo nsenter -t "$HEAL_PID" -m -- findmnt -T "$DATA" -o TARGET,SOURCE,OPTIONS +# 以服务实际UID在该namespace做本例临时文件写探针，必须观察EROFS。 +# 解除仅此额外的bind层： +sudo nsenter -t "$HEAL_PID" -m -- umount "$DATA" +``` + +不可达用经批准的该服务 mount namespace 故障层覆盖此数据根（返回EIO的专用device-mapper设备，见IO_FAULT）；不能把空目录覆盖当作掉盘，因为可能触发新盘初始化。权限实例可使用本例非root UID +精确目录ACL拒绝，在相同UID实测EACCES；root服务单纯chmod不算有效注入。恢复后重新验证mount/磁盘身份和读写可达；在 namespace 内 helper 的跨mount拒绝是正确保护，不应关闭它。 + + + +### 新逻辑盘替换 + +停止本例unit及写入，校验 ROOT/data 与owner/run/mount；将**精确 data 根**同盘移至本例 quarantine 的唯一 old-disk 目录，新建同位置空data根。其他盘不动。重启后读取format，确认新disk UUID与slot；格式初始化失败要如实记录，不手动补出一个UUID。 + +```bash +# 仅已审核的本例 ROOT；先验证备份目标不存在，不递归删除任何目录。 +sudo mv -n -- "$ROOT/data" "$ROOT/quarantine/$FAULT_ID-old-disk" +sudo install -d -m 0700 -- "$ROOT/data" +``` + +检查旧根确实移走、摘要清单完整、服务无法访问quarantine；再正常启动。rebuild期间不得从old-disk复制part。失败救援时先停止unit并把新失败data根移至另一个保留目录，再恢复旧根；该恢复明确为rescue，不是heal结果。真实设备detach/attach仅属于另行批准的H6，不由此命令模拟。 + + + + +### 进程停止、崩溃、重启、升级 + +```bash +sudo systemctl show "$HEAL_UNIT" -p MainPID -p ExecMainStartTimestampMonotonic -p FragmentPath -p ActiveState +sudo systemctl stop "$HEAL_UNIT" +sudo systemctl start "$HEAL_UNIT" +# 独立 crash 实例，必须先 WINDOW、记录被杀的实际MainPID： +sudo systemctl kill --kill-whom=main --signal=SIGKILL "$HEAL_UNIT" +sudo systemctl show "$HEAL_UNIT" -p ExecMainCode -p ExecMainStatus -p Result +sudo systemctl reset-failed "$HEAL_UNIT" +sudo systemctl start "$HEAL_UNIT" +``` + +每次操作前核对 MainPID/exe 摘要/starttime，拒绝0或已替换PID，不 `pkill rustfs`。保存 `/proc/sys/kernel/random/boot_id`，SIGKILL前后boot_id相同，不能说断电。 + +UPGRADE 对本例节点逐一stop，用已经验证的第二个optimized release二进制替换该unit的exec路径（保留旧二进制），原env/endpoint/data不变，start后核对每个peer实际版本。旧新两份构建清单、包sha、库格式兼容表都须预注册；未知格式禁止直接回滚，改测安全拒绝。精确升级期间责任按CHECKPOINT验证，不依赖版本字符串判断兼容。 + + + +### 测试端口分区 / 依赖故障 + +优先预分配专用net namespace，证明它与host不同；需要业务节点连接的路由已配置，SSH仍走host。只对本例API/RPC端口或本例KMS/tier endpoint加规则。先保存规则/路由，设置独立超时恢复任务。例： + +```bash +sudo nsenter -t "$HEAL_PID" -n -- iptables -w -I OUTPUT 1 \ + -d "$TEST_PEER_IP" -p tcp --dport "$TEST_SERVICE_PORT" \ + -m comment --comment "$FAULT_ID" -j DROP +sudo nsenter -t "$HEAL_PID" -n -- iptables -w -nvL OUTPUT --line-numbers +# 相同namespace、完整相同规则删除，不flush任何链： +sudo nsenter -t "$HEAL_PID" -n -- iptables -w -D OUTPUT \ + -d "$TEST_PEER_IP" -p tcp --dport "$TEST_SERVICE_PORT" \ + -m comment --comment "$FAULT_ID" -j DROP +``` + +双向分区分别在绑定双方namespace应用，不能扩大到22、管理平面或host OUTPUT。记录丢包计数与双端探针。无独立net namespace或专项host规则批准时该实例BLOCKED，不能全局断网凑结果。 + + + + + +### 慢IO/EIO/容量与锁窗口 + +此组**只在预分配独占故障盘/有硬上限的专用loop设备**操作，绝不对 `/dev/nvme*` 共享生产设备运行。操作前记录backing file、loop/device-mapper名字、major:minor、mount UUID、上限和唯一FAULT_ID。设备创建/format需要本批单独的资源批准，不是case默认授权。 + +已分配专用 `HEAL_DM_NAME` / `HEAL_BACKING_DEV` 的device-mapper配方： + +```bash +sudo dmsetup table "$HEAL_DM_NAME" +sudo dmsetup suspend "$HEAL_DM_NAME" +sudo dmsetup reload "$HEAL_DM_NAME" --table "0 $HEAL_SECTORS delay $HEAL_BACKING_DEV 0 200" +sudo dmsetup resume "$HEAL_DM_NAME" +# EIO实例用完全返回error的表；只用于专用故障层： +sudo dmsetup suspend "$HEAL_DM_NAME" +sudo dmsetup reload "$HEAL_DM_NAME" --table "0 $HEAL_SECTORS error" +sudo dmsetup resume "$HEAL_DM_NAME" +``` + +恢复reload预先保存的精确原table，不猜linear偏移；在同服务namespace做定长读写见证（记录耗时/errno），并验证源设备可用。块设备EIO不等同应用层短读，短读实例使用FAULT truncate和真实读错误证据；若需“无内容变化的短读”必须有已分配的faultfs后端，另记能力条件。 + +ENOSPC仅写本例**硬上限卷**中的占位文件，先 `findmnt`/`df -B1` 和 inode计数确认不会耗尽外层XFS；按上限分块填充至返回ENOSPC并记录实际服务写失败。inode实例用创建空文件耗尽该卷的inode预算，不能在共享根创建无限文件。释放只删除已列举的本例占位文件或回收专用卷，不使用宽目录递归删除。 + +桶配额使用冻结rc的 `quota --help`，对本例桶 `quota set`/`quota info`；前后查询证明限额真实生效，结束恢复原配置。禁止把客户端本地限速当服务器quota。 + +```bash +"$RC_BIN" --json --no-color quota info "$RC_ALIAS/$B" +"$RC_BIN" --json --no-color quota set "$RC_ALIAS/$B" 64M +# 只在原配置没有quota且本例结束时清除；否则恢复原限额： +"$RC_BIN" --json --no-color quota clear "$RC_ALIAS/$B" +``` + +LOCK_WINDOW：慢IO层作用于一个实际持有对象写锁的请求，另发同key heal/PUT。用日志/原生runtime fence证明锁获取、超时、调用方退出、在途提交退出顺序；没有可观察锁窗口则黑盒实例为NOT_OBSERVED。客户端断开可由有界 `timeout` 结束**本次请求进程**，不杀服务；之后继续采集服务端在途状态。精确所有权证明依赖NATIVE，不能从一次超时推导锁已释放。 + + + +### 强制停机 + +只对H6已批准的精确Azure资源ID执行，先核验该VM独占、没有其他服务、磁盘持久化策略及带外恢复；记录当前boot_id和平台状态： + +```bash +az vm get-instance-view --ids "$APPROVED_EXCLUSIVE_VM_ID" +az vm stop --ids "$APPROVED_EXCLUSIVE_VM_ID" --skip-shutdown +az vm start --ids "$APPROVED_EXCLUSIVE_VM_ID" +az monitor activity-log list --resource-id "$APPROVED_EXCLUSIVE_VM_ID" \ + --start-time "$RUN_STARTED_AT" --end-time "$RUN_FINISHED_AT" +``` + +这是真实VM强制停止，不证明物理控制器掉电。物理主机power-cycle必须使用该独占机器已批准的带外控制台，记录BMC事件和电源周期；不能给未知机型编造通用命令。无授权不执行，也不把SIGKILL替代成通过。 + +## 6. 并发、Pool、ILM、复制及原生证据 + + + + + + +### 有界负载与观测 + +`trace.json` 是按序的S3操作数组，例如： + +```json +[ + {"operation":"put_object","params":{"Bucket":"rv-heal-20260910-a01","Key":"race-${worker}"},"size":5243003,"expectHttp":200}, + {"operation":"head_object","params":{"Bucket":"rv-heal-20260910-a01","Key":"race-${worker}"},"expectHttp":200} +] +``` + +```bash +hw writer run --trace trace.json --rounds 100 --workers 2 --interval 0.1 --duration 300 +``` + +同worker串行；不同worker用不同key可明确先后，测试真正同key竞争时人工按ACK/不确定结果建立期望。`${round}`、`${worker}` 只做JSON字符串替换，不eval。上传内容由run/worker/round/step确定，大小/摘要写入ACK。每次ACK立即独占写文件；意外错误停止后续入场并保留在途结果。不因服务返回错误而无限重试。 + +长稳分支增加已批准 `--duration` 至86400/259200、足够rounds及限速，并核对实际elapsed，提前耗尽rounds/字节预算不算跑满窗口；每批最多1,000,000请求/16worker/64GiB写入预算，可按预注册多个分段运行并保留连续时序。不可通过事后拼接零散短跑冒充连续24h。 + +Admin队列压力使用同样的限量并发方式调用hp（例如shell后台最多4个固定任务），先冻结请求集合，独立label保存每个返回；不能把S3负载当Admin admission覆盖。 + +```bash +# 在目标节点按固定频率采样，时间窗口与本例一致；不要采集全环境凭据。 +sudo journalctl -u "$HEAL_UNIT" --since "$RUN_STARTED_AT" --until "$RUN_FINISHED_AT" --no-pager +sudo systemctl show "$HEAL_UNIT" -p MemoryCurrent -p TasksCurrent -p CPUUsageNSec +pidstat -dru -p "$HEAL_PID" 1 60 +iostat -dx 1 60 +``` + +同时记录FD数量、目标IO、队列/持久记录及临时空间；工具未安装先预检阻断所需性能证据，不跑完才省略。性能ABBA每组四轮、至少三组：warmup、并发、内容、缓存/初始队列、负载窗口和阈值固定。分别统计GET/PUT P95/P99、错误率（所有失败/超时都在总分母）、修复吞吐、CPU/IO。没有预注册门限只能出测量报告，不能记性能通过。 + + + +### Pool交互 + +使用**本例H5服务**的rc alias，已有13项Pool回归结果仅为前置控制，不直接调用其会reset旧fleet的部署脚本。扩容/移除拓扑复用已审阅的endpoint分组方案，但根目录替换为本例heal命名空间。所有poolId从实际status和format映射获得。 + +```bash +"$RC_BIN" --json --no-color admin rebalance start "$RC_ALIAS" +"$RC_BIN" --json --no-color admin rebalance status "$RC_ALIAS" +"$RC_BIN" --json --no-color admin rebalance stop "$RC_ALIAS" +"$RC_BIN" --json --no-color admin decommission start "$RC_ALIAS" "$POOL_ID" --by-id +"$RC_BIN" --json --no-color admin decommission status "$RC_ALIAS" +"$RC_BIN" --json --no-color admin decommission cancel "$RC_ALIAS" "$POOL_ID" --by-id +"$RC_BIN" --json --no-color admin decommission clear "$RC_ALIAS" "$POOL_ID" --by-id +``` + +不是把以上所有操作依次跑完：按H01–H06操作卡在相应窗口执行。每次保存rc stdout/stderr/exitCode、相关Admin状态及前后manifest。退役写入限制沿用既定产品契约；仅survivor验证时源必须真实停机且配置移除。#2422/#2423的临界点是否实际触发单独登记。 + + + + +### ILM、远端tier、复制 + +使用本仓库已实现lane作为**控制实验**核对候选功能，并在本例独立服务重新建立专用桶/远端租户。配置JSON只含本例资源，API操作如下： + +```bash +hp ilm-put s3 --operation put_bucket_lifecycle_configuration --params-file lifecycle.json +hp ilm-get s3 --operation get_bucket_lifecycle_configuration --params-file bucket.json +hp restore s3 --operation restore_object --params-file restore-version.json --expect-http 202 +hp replication-put s3 --operation put_bucket_replication --params-file replication.json +hp replication-get s3 --operation get_bucket_replication --params-file bucket.json +``` + +`lifecycle.json` 使用已验证候选的真实期限；规则ID含run，Filter仅本例prefix，分别配置Expiration/NoncurrentVersionExpiration/Transition。不要用未合并的分钟级测试hook冒充release。远端tier的初始化依照当前rc `ilm tier add --help` 和已有 `scripts/validation_ilm/ilm_behavior.py` 的实际调用，填写本例endpoint/bucket/region与受控凭据；原始凭据不归档。 + +复制配置含本例role/destination ARN、Filter和DeleteMarkerReplication；先证明对端版本映射/删除语义。resync使用本批rc `replicate resync --help` 所支持的实际命令，保存启动ID和两端状态。观察真实重叠窗口；没有tier/第二端，相关分支BLOCKED而非删除该项。结束通过API删除本例lifecycle/replication配置，远端本例数据按版本清单清理，不改共享服务规则。 + +已核对参考rc的resync操作（仍须新批help核验）： + +```bash +"$RC_BIN" --json --no-color replicate resync start "$RC_ALIAS/$B" \ + --target-arn "$TEST_TARGET_ARN" --reset-id "$RUN" --yes +"$RC_BIN" --json --no-color replicate resync status "$RC_ALIAS/$B" --target-arn "$TEST_TARGET_ARN" +``` + + + +### 精确屏障与原生回归补证 + +F9类窗口不能靠普通release黑盒保证命中。使用同源码SHA的**独立native回归二进制**，绝不换掉被测optimized release服务。先列出真实测试并固定完整测试名、crate、源码位置和预期屏障事件： + +```bash +cargo test -p rustfs-heal -- --list +cargo test -p rustfs-ecstore -- --list +cargo test -p e2e_test -- --list +# NATIVE_TEST必须逐字来自上面列表，日志必须显示恰好1项且屏障命中： +cargo test -p "$NATIVE_CRATE" "$NATIVE_TEST" -- --exact --nocapture +``` + +已有replacement回归函数包含 `test_cluster_root_heal_recovers_remote_shards_after_background_target_restart`、`..._crash`、EC8+4和multi-set版本，完整限定名从该commit --list取得。它们默认创建本地E2E临时目录，**不是远端真实独立盘部署器**；不通过环境变量把这些会清理临时目录的测试指到现有fleet。原生锁/代次/checkpoint测试按所选操作卡在同commit源码中找到精确回归和屏障标记后运行，不存在对应屏障的分支要保留“需源侧增加确定性测试”的能力缺口，不能给计划生成器喂手写JSON声称measured。 + +归档 candidate SHA、native测试源码SHA、完整命令、测试数量、exitCode、运行时间及屏障事件。native与fleet分别计分；缺失精确native证据不影响已经完成的黑盒分支真实性，但不能宣布该case所有分支齐全。 + +## 7. 恢复、报告和安全交还 + + + +先保存出错时的状态、日志和PHYSICAL，停止本例负载/任务/进程；再按fault receipt撤销规则/只读层/替换根。对已失效lease允许受限恢复，不允许继续注入。单文件恢复： + +```bash +hf recover --fault-id C02-data-middle --service-quiescent +``` + +只恢复同run的备份，原件摘要不符则拒绝；目标如果已经是原始正确字节不重写；如果变成既不是原始也不是注入后字节（可能有新ACK或合法metadata重写），拒绝盲目覆盖，交人工保留现场。恢复工具输出 `physicalRepairPassAllowed=false`；任何 `restoredFromBackup=true` 都不能作为产品补片证据。 + +For an externally cancelled Functional Chain, `validation_cleanup_run_id` accepts only a completed run on the same workflow branch. F07 recovery supports the bounded H2/DV stale-generation boundary after four missing-part injections and before Admin admission: all 18 earlier branches must have complete cleanup receipts, and all 282 fixture requests must have matching successful SDK acknowledgements. Any additional action, unknown write, lost root or physical drift fails closed. Recovery stops the owned services, restores only the four original part backups, and verifies the complete original business files and disk formats. It then retains every data root and private state directory by the existing non-overwriting, same-filesystem archive protocol, with a durable inventory bound to each node's original business-scene digest. Partial restoration and partial archive moves are retryable. `ISOLATED` releases the active namespace while preserving backups; it never rewrites the original cancelled/RUNNING result, restores old system records, or claims canonical PASS. + +强制故障设置第二条恢复通道/超时撤销；本地 finally 不是网络或进程被杀时可靠的唯一恢复办法。退出前枚举本例网络规则、bind层、dm表、测试unit、临时IAM和未完成MPU，逐条撤销并复核。保留失败数据/backup，retention未到期则保留专用资源；记录剩余路径/到期安排。未恢复为 DIRTY/QUARANTINED，不继续下一用例。 + +每个**实际参数实例**都填写人工结论和证据索引，不只填一个基础case总绿： + +```text +preflight/ 基础设施/包/rc/harness/lease/拓扑/预注册参数 +baseline/ S3版本清单、物理基线、ACK、桶配置 +fault/ 精确plan、intent、原件摘要、故障有效性见证 +trigger/ 原始Admin/GET/后台来源及活动窗口 +control/ token/epoch/coverage/counter/负向错误/游标原始响应 +physical/ after/restart分片、metadata映射、独立重建对照 +s3/ 所有应存/应删版本与ACK解析、完整内容和配置 +recovery/ 故障解除、停止服务、retained路径与安全交还 +logs/ 全部相关节点的有界日志/资源/平台或native事件 +``` + +人工结论为 MANUAL_PASS/FAIL/INFRA_ERROR/BLOCKED/NOT_OBSERVED，列评审人、每条断言及引用。`evidence.json`包含runId、instanceId、九类evidence的数组，每个引用为相对于该实例目录的`path`和`sha256`。全部分支完成才能评审基础case；缺资源/窗口未命中/证据不全都不是PASS。 + +```bash +python scripts/validation_heal/matrix.py audit --plan reports/validation-heal-84-plan.json \ + --evidence-dir reports/heal-84 --out reports/heal-84-evidence-audit.json +``` + +audit检查完整性/路径/摘要，输出`EVIDENCE_COMPLETE_REVIEW_REQUIRED`仍不是产品通过报告。原始失败批次永久保留，修复后新批次另记。本文提供全84项的可操作方案，但实际机器、专用依赖和native屏障的可用性必须单独核验，不能把“方案就绪”改写成“环境就绪”或“已实测”。 + + +自动正向夹具固定使用独立分片承诺配置；`integrity-fleet-attestation` 记录本例 endpoint/节点/候选摘要，启动前逐节点 reserve 校验候选，启动后实际进程环境读回两项开关。此配置只作用于 run-owned 服务。默认 legacy 配置的失败批次独立保留；保护模式不代表旧版本混部或默认环境能力。D07 的条件请求以服务器 HEAD 的 LastModified 和基线 ETag 冻结预期，不改主机时间；304/412/416 不能包含成功 payload 或未知传输结果。 + + +J06 的自动入口为 `heal.performance-abba`。执行器在部署前冻结三组 ABBA、每阶段 30 秒、2 并发、4 请求/秒、GET/PUT 各半与完整阈值；每阶段重启同一服务并按固定次数预热。每个 B 阶段的真实部分修复窗口必须包围 GET 和 PUT，否则整例不能 PASS。A 阶段持续检查完整 idle 与物理不变。所有排定请求（包括失败、未知和错过 admission 的请求）均计入分母；GET/PUT 独立报告 P95/P99，三组配对差异、实际修复分片吞吐与服务 CPU/IO/RSS/FD 分别留证。阈值保存在 `scripts/validation_heal/performance.py` 的 `PROTOCOL`，修改阈值必须产生新的 harness commit 和新验证记录；此入口不缩短 J05 的 24h/72h 要求。 + + +A02 的自动入口为 `heal.admin-scope-isolation`。H2 的专用 `scope-isolation` runtime 使用当前四节点/十六盘，按盘序交织节点并设置 `RUSTFS_ERASURE_SET_DRIVE_COUNT=8`，要求真实两个 set 各跨全部四个节点。普通执行器仍只接受原单 set。两个桶的完整 ACK/实际物理 set 清单冻结后，root/bucket/prefix/object/pool-set 各自新部署、注入选中与控制故障。完整物理和 streaming 证明必须先于 GET 和控制备份恢复。H1 保留原四盘单 set，明确记录不存在未选 set 的适用边界；不能拿同盘目录冒充第二个 set。未限制选择器的原始 receipt 在该固定候选中允许 null，单独保存实际物理映射，显式 pool/set 则必须严格匹配原始字段。 + +### I04 durable responsibility during queue pressure + +The live `queue-pressure` permission fault targets only `-durable` on the first owned drive. The other four queue buckets and `.rustfs.sys` retain their original permissions. The live arm derives this fixed suffix from the root-owned reservation; callers cannot supply another bucket or path. Flapping keeps its separate whole-drive fault and F07 keeps its original dedicated bucket scope. + +Before the three degraded-write ACKs, the same original service/mount-namespace identity must observe business read/write `EACCES` and successful system metadata read/write controls with DAC bypass removed. A whole-drive refusal or unavailable system control is rejected before those writes. Full, Merged, Dropped, all three committed MRF responsibilities, exact source-task retirement, complete six-bucket snapshots and original `769/129/129/129` fixtures remain required. Blocking the MRF journal cannot isolate queue-Full retention. A corrected fault does not establish canonical PASS until a separate complete live run is independently audited; original failed results remain retained. + +Complete the owned permission probes before admitting the finite pressure workload, so SSH setup does not consume its queue window. The later actual MRF Full event after the SDK ACKs remains mandatory; scheduler pauses, deadline increases and a prior Admin Full response cannot replace that event. Terminal object identities use the existing source-reviewed mapping between Admin's all-zero UUID and S3's `null` version. Real version UUIDs, incarnation/set boundaries, complete counters and independent physical/S3 proofs remain exact; untyped version identities are rejected. + +Owned Heal services request a per-unit journal budget of 1,000,000 messages per second, then read back both effective properties and record the systemd version before source attribution. Missing, zero or different settings are rejected; merely passing launch arguments does not establish effective configuration. Host-wide journal settings remain unchanged. Cursor continuity alone does not establish complete source attribution: the reader retains journald's original dropped-message notice and marks that page incomplete, including notices identified by the stable journal message ID. Source observers reject the page before accepting any task events. Existing page/event limits, resource ceilings, workload counts and deadlines remain enforced; absent source receipts in a rate-limited original run cannot establish either successful repair or a product responsibility-loss diagnosis. + +### Exact-source native race subsets + +`race_native.py` builds and runs the reviewed tests in `race_native_contracts.json` on a hosted tools runner. The candidate checkout must match the exact commit, tree, lock and reviewed Rust files; the producer and registry must match the exact auto-testing checkout. The build uses a fresh target directory and rejects compiler wrappers or injected Rust flags. Each selected regression runs in a separate process with one exact nonignored test. Original logs, executable digest and ordered barrier milestones are retained. These artifacts are separate from `native_regressions.py`; existing F07/C11 artifact contracts remain unchanged. + +The first registered source is `6afdec9fe1b0b88007f2e5b445c19bf73d51d3ca`. G01 includes real Heal during committed overwrite cleanup in four disk/parity/mode scenarios and the existing stale read-repair final-rename test. The G02 subset covers the version-specific shared Heal owner and queued successor PUT. The I05 subset reuses the actual cancelled physical unlink owner and stale read-repair fences. These legacy subsets do not complete G02 or I05; I05 also needs its TCP disconnect, timeout and file-descriptor matrix. + +Source `bd0671ba781d8ebf509f4f03533228dfb44496f1` adds complete G02 and G03 native groups. The versioned PUT group requires both the historical lock owner and an actual new-version commit while historical reconstruction is paused. The deletion group requires exact historical and current version deletion before releasing stale reconstruction. Both use real `4/2` and `16/4` local geometries, physical write-tail drain, exact surviving encoded shards and complete version bodies. Its G01 fixture pauses every shuffled cleanup slot and checks every new encoded shard again after cleanup and before GET. These source contracts require fresh same-source hosted artifacts; local tests or an older source's artifacts cannot supply them. + +Source `8380466f3a27c3c3ef4115084178a0ab99a78f32` additionally requires the complete G04 marker/batch-delete group. In each geometry, creating a marker, removing an exact marker, deleting two exact data versions with separate item receipts and deleting an unversioned object all commit while older reconstruction is paused. The foreground physical tail drains before release. Every disk must retain the exact data/marker version set without duplicates, every surviving encoded shard and complete exact/current bodies; removed markers and data directories cannot reappear. Its 24 ordered milestones supplement the existing G01/G02/G03 groups. + +`race_native_evidence.py` admits the downloaded artifact without executing it. It checks the successful hosted workflow attempt, exact candidate/driver/helper identity, selected source contract, native executable, complete unique regression set and every original log. Each ordered-barrier regression must emit exactly the complete source-registered milestone sequence. Regressions marked as in-test assertions retain that narrower witness type; assertion-only evidence cannot invent emitted barrier events. The result states `requiredNativeSubsetPassed`; whole-case, fleet and canonical PASS remain false. Missing source groups, old-source artifacts, ignored or zero-test output and partial evidence are rejected. Updating another candidate's registry entry does not invalidate an unchanged selected contract. Changing the producer or the selected contract requires a fresh tools artifact. + +Runtime sources opt in to specific cases separately from the native registry. The `6af` source enables only G01; `bd0671b` enables G01/G02/G03; `8380466` additionally enables G04 with its complete, distinct native group. Source `757b751` enables A09 through `nolock.py`, G06 through `race.py`, and G05 through the dedicated `mpu_race_runner.py` controller. It does not substitute a different source into G01-G04. The workflow verifies the bundle and invokes the pinned helper's read-only runtime preflight before exporting it to canonical execution. A collector-only helper without the corresponding runtime adapter fails before fleet mutation. The executor repeats admission before package installation or service changes. + +All original H1/H2 black-box branches, SDK ACKs, manifests, overlap witnesses, exact Admin settings and physical/S3 checks remain required. Native logical disk shapes do not certify distributed VM execution. The original `072b57b37559fbe626aeda8abf61617fdcbdd926` route retains its blocked native rows. A09, G05/G06 and I05 still require their own complete native groups and executor evidence. G04 also remains unaccepted until its same-source hosted evidence and every original live branch pass independent audit. No source or case gains canonical acceptance from a registry entry or local native test; a successful exact-source hosted bundle, every live branch and independent original-evidence audit are required. + +The A09/G01-G04/G06 public race adapters retain the frozen fourth-physical-drive fault selection, independent of shuffled data/parity role. H2 selects that local drive on its first original host. After faults and restart, `race_io.py` applies the source-defined `65536` bytes/s write cap only to the run-owned unit on actual fault hosts. Exact process birth/candidate, all four original mount identities, systemd cgroup and `/proc` membership are checked; the actual kernel `io.max` must show the original fourth device and rate. Existing foreign limits, changed identity, wrong devices and unbound release reject. Intent is recorded before each single control request. After the foreground sequence, the cap is removed and kernel readback is verified before waiting for the same original task. This adds no sleep to SDK timing and changes no payload, overlap oracle or task timeout. An interrupted or uncertain control still uses existing owned-service stop and retained-scene rules. G05 refuses this cap and retains its source-defined unrestricted process I/O. + +### A09/G06 native groups and frozen G06 public fixtures + +Source `757b751482bb970bc321e92059690a7698554358` adds two collector-only groups. `default-heal-exclusive-owner` observes actual PUT and exact historical DELETE lock futures becoming pending while default Deep Heal exclusively owns a real staged external repair. Physical reconstruction precedes foreground publication; all acknowledged successor or retained-current shards and full bodies are checked in 4/2 and 16/4. Its sixteen events prove the storage fence, while Admin `nolock=true` normalization and public request timing still require separate evidence. `copy-and-metadata-final-rename` covers real streamed cross-key COPY source/destination, self metadata replacement and tags. Twenty-four events bind actual ACK and physical-tail completion before releasing shared reconstruction, exact stale refusal or unchanged-source repair, final source/target/control payloads, metadata and tags. The registry binds all five storage fixture/barrier source files. This candidate admits A09, G05 and G06 only with their distinct complete case-specific native group and matching hosted helper/source artifact. Both workflow and executor run read-only preflight before bootstrap. A09 always sends wire `nolock=true` and requires effective `nolock=false`, plus both original H1/H2 public branches and the separate storage-native proof. G05 uses the separate same-key runtime described below. + +The G06 public adapter uses the existing DS workload: all 257 original keys, including Chinese, spaces, literal `%2F`, actual slashes and parent-like prefixes. The observed missing key is `中文/文件/0252.bin`; its companion is `literal%2Fslash/0254.bin`. `copy-source` copies from the observed key into the companion; `copy-destination` copies in the reverse direction. Default COPY expectations inherit the source payload, content properties, metadata and tags. Both directions verify the unchanged source, exact destination coding-index shard bytes and a distinct destination generation before GET, rejecting retained old destination directories. Registry, fixture and local transition tests do not establish public overlap or complete canonical PASS. + +G06 reuses `KeepAliveStatusProbe` only while observing the original token and +foreground operation. Existing readiness polls warm the same direct connection; +the controller still starts each foreground observation from the real SDK send +notification. Every connection attempt, actual socket identity, response and +phase remains in `G06-status-transport`. A transport error poisons the session +without an implicit retry; leaving the session closes its connection. The strict +same-clock nested interval, complete ACK ledger and finished-task refusal remain +unchanged. No delay is injected into the SDK call. + +Before race allocation and each explicit service restart, the runner and every +participating node resolve the existing `rustfs-node1` alias (H1) or all four +aliases (H2). `endpoint-resolution` retains bounded, single-attempt `getent` +results and refuses a failed precondition. It never edits hosts files or DNS, +substitutes IP addresses, extends the readiness budget or certifies node identity. +Later topology and physical checks remain required; successful preflight cannot +guarantee that a resolver will remain available during startup. + +G01/G02 retain every serial PUT ACK but follow the frozen source's overlap rule: +at least one strictly nested running status sample per required workload role. +G01's read-repair branch additionally journals every actual GET, verifies its +complete body and joint ETag/length/type/metadata against one original or +ACK-known foreground generation, and requires a reader overlap. Zero reads, +unknown results and mixed generations cannot pass. G02 requires all six +ACK-known historical versions and a separate reader overlap in its read branch. +Sampling continues until an in-flight reader drains after the writer. These +conditions preserve the source workload; they do not supply live acceptance. + +### G03 serial deletion authority + +G03's frozen `VersionLedger.history` determines the new current version after an explicit DELETE. The public S3 inventory is UUID-sorted and cannot supply that order. Each original foreground plan now carries the three seed ACK version IDs in order; the final oracle rejects missing, duplicate, foreign or differently selected identities, removes exactly the acknowledged version and promotes the last retained seed ACK. The physical oracle permits only that `isLatest` change while preserving every other retained descriptor, coding-index byte and data directory. Every original drive must contain exactly the retained version/latest population, with no extra marker or duplicate version. Focused tests use nonmonotonic UUIDs, both delete branches and corrupted physical populations. This source adaptation adds no native or live acceptance. + +### Frozen version and multipart race controllers + +G04 and G05 use the owned `frozen-race-legacy` runtime profile. It explicitly sets `RUSTFS_SHARD_INTEGRITY_WRITE=false`, `RUSTFS_SHARD_INTEGRITY_FLEET_CONFIRMED=false`, `RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE=false`, and `RUSTFS_HEAL_MAX_CONCURRENT_HEALS=1`, matching the frozen source's legacy shard population and serial scheduling. The profile retains disabled scanner, automatic Heal and MRF. The regular `manual` profile continues to enable protected shard writes. Original physical file-population assertions remain strict; an unexpected integrity sidecar is a failure. Owned service configuration is attested through the existing runtime checks. G04 alone may use the frozen fourth-drive write cap under this profile; G05 only observes kernel limits. Retention supports both historical `manual` evidence and the new profile, with the same fixed bucket/key scope and stopped original scene binding. An older helper's tools remain bound to that helper; they cannot certify a newer controller. + + +`version_race_fixture.py` preserves the frozen `heal_version_race_fixture.py` model at `524eb5b32c3236497fdd8a9271df0494fa3d9974` (source SHA256 `755f476ab567dc413e9798bb2bcac15252c3e8308e4b15b15a9eb3083d1df485`), with local import adaptation only. Its eight G02–G04 plans derive versions from serial ACK order. G04 uses flat keys, seeds a latest marker for both marker removal and batch deletion, and deletes exactly the oldest target version, newest target version, marker and control `objects-0002.bin` version. It preserves the middle target generation. A partially acknowledged batch retains each known effect and permanently blocks final acceptance and further mutations. Historical reads address exact versions; negative reads require the original deleted-version 404 and marker 405/current-marker 404 contracts. + +`version_race_physical.py` preserves the frozen physical oracle, replacing the source transport's fixed inspector digest with a mandatory caller-supplied immutable tool digest. `version_race_adapter.py` maps every owned drive to its real format set slot; H1 retains EC2+2 and H2 explicitly requires the deployed EC12+4 geometry. Every drive must match the ACK-derived version/marker population and coding indices. Deleted generations' unreferenced bytes remain separately reported; they are neither live redundancy nor a garbage-collection PASS. + +`version_race_runner.py` replaces the superseded generic G04 fixture and shard-fault path. Its 257 faults remove the first host's fourth-drive `xl.meta` files. The original 120-second partial window requires nonzero original-token progress, a physically reconstructed control, at least 32 pending objects and the selected target metadata still absent. One original SDK mutation runs with the source 90-second observation budget and 25ms sample pause; the owned fourth-drive 65536-byte/s write cap is released while the original call drains. Final checks prove the 253 other controls, exact version/marker/coding populations, full live part streams, every peer's deleted-version/marker negatives and complete inventory, post-GET physical immutability, ACK-bounded completion and original root retirement. Final assertion failures preserve independent diagnostics; evidence I/O failures stop execution. Seed or foreground uncertainty retains the stopped scene and original journals as DIRTY. Recovery can isolate that exact scene without restoration, retry, deletion or a PASS rewrite. Runtime entry requires source `8380466` and the complete marker/batch native group before bootstrap; the legacy source cannot enter this controller. These implementation/component checks add no canonical PASS; fresh exact-helper native tools and complete H1/H2 live acceptance remain required. + +`mpu_race_fixture.py` migrates the frozen validation repository's `heal_mpu_race_fixture.py` model at `524eb5b32c3236497fdd8a9271df0494fa3d9974`. It uses the existing auto-testing `Recorder` and probe directly. Imports are local; no source validation checkout is needed at runtime. Part sizes follow the frozen common DM recipe: `5MiB+17`, `5MiB+93`, and `1MiB+47`. + +The caller supplies a fresh versioned owned bucket. One completed three-part object and two distinct pending uploads share `objects-0256.bin`: A initially has parts 1/2, B has parts 1/2/3. The model binds each mutation to its actual UploadId/part ETags and durable request/ACK files. A continues with part 3 and completes; B aborts. Unknown or malformed mutation results poison the model and prevent later sibling changes. Bounded paginated listings, exact terminal `404/NoSuchUpload`, all acknowledged historical/current versions and complete GET payloads verify sibling isolation. The model does not clean up uploads with uncertain outcomes. + +Immutable native source `09fb813a003e7964bd1b893efb7446a3c41bd3bf` adds `multipart-continue-complete-abort-final-rename`: ten ordered milestones across 4/2 and 16/4 logical geometries, using real three-part commits, same-key A/B pending parts and a paused shared Deep Heal. Completion tails drain before stale Heal release; exact retry disposition, old/sibling preservation, new committed bytes and upload retirement are asserted. The native test's unversioned logical scope is supplemental to the full versioned SDK/fleet case. + +The historical `09fb813` entry remains collector-only. Runtime admission binds source `757b751482bb970bc321e92059690a7698554358`, its complete multipart native group and the exact hosted helper artifact. The old `072b57b` source is rejected before bootstrap. Component tests and native bundles do not add canonical PASS. + +`mpu_race_runner.py` replaces the generic unversioned MPU path with the frozen same-key fixture and schedule for all three public branches in H1/H2. It records 256 versioned flat-key controls, one committed three-part version and A/B pending uploads through the existing durable `Recorder`. The 257 faults target the fourth physical drive on the first host. All 259 original and 262 final part cohorts receive full streaming verification. Every companion ACK has an immediate independent staging checkpoint; only the selected first SDK operation requires a contained original running-token sample. The status transport preserves the frozen warmup, 1024-sample/90-second budget and 5ms poll pause, restricts requests to the owned bucket and original token, and records request/response persistence phases. Kernel I/O sampling reads the owned cgroup and its ancestors without setting or releasing a cap; it makes no continuous no-limit claim. + +Final committed/staging/control proofs precede all-peer S3 verification. Assertion failures retain the frozen source's remaining independent diagnostics while evidence I/O errors stop execution. Repeated terminal responsibility, ACK bounds, post-GET physical stability and exact original-root retirement must pass before cleanup authority exists. Any issued seed request without that authority retains the stopped raw MPU prefix, committed scene and original request/ACK files as DIRTY, with no retry, abort, restore or namespace cleanup. A missing overlap remains `NOT_OBSERVED` even after safe cleanup. Full hosted native acceptance and all six public live instances still require independent evidence before G05 can count as canonical PASS. + +The additive `mpu-race-scene` node action in `mpu_race_physical.py` adapts the frozen `heal_h1_mpu_physical.py` upload-ID and byte-continuity oracle to the current owned four-drive hosts. Each request binds the owned race runtime, bucket, candidate binary and all three ACK-selected uploads. It inventories the complete fixed key prefix, hashes only selected staging files under bounded entry/depth/byte limits, rejects unknown uploads and links, and preserves original process identity during a live capture. Stopped captures additionally verify the owned service remains stopped. Across H1/H2, pending part/sidecar populations and existing sibling bytes remain exact; a completed upload must lose its staging directory and retain all three previously witnessed encoded parts in committed storage. These are staging/continuity witnesses; independent inspector coding/bitrot checks, product Heal proof, real overlap and full SDK inventories remain required. + +### I05 caller-only TCP disconnect component + +`caller_abort.py` sends one signed, exact historical-version S3 GET to a bound endpoint and closes only that caller TCP socket. Invoke it with `python3 -B scripts/validation_heal/caller_abort.py --binding --bucket --key --version-id --output `. Credentials use the existing `HEAL_ACCESS_KEY`, `HEAL_SECRET_KEY` and optional `HEAL_SESSION_TOKEN` environment variables. The request intent is persisted before connection; a completed write checkpoint precedes shutdown, and the final receipt binds the original socket addresses, descriptor, request digest and monotonic clock. The evidence directory cannot be reused. There is no request retry, redirect, proxy or response-derived success. A persistence or transport error closes the socket and retains its uncertain result. Signed headers and credential values are not stored. + +The component always reports `resultUnknown=true`, `acknowledged=false` and `serverOwnershipProven=false`. Successful local command execution means the client transport completed its disconnect sequence. It does not prove that the server admitted the call, acquired a lock, continued after disconnect or released its owner correctly. I05 remains outside canonical dispatch until the controller binds actual server admission and owner events to this exact call, proves the registered timeout and stale-generation fences, and verifies final object generations, drained locks and file descriptors across every required H1/H2 branch. Existing exact-source native owner tests remain separate evidence. + +### I05 service admission and caller drain observer + +`lock_observer.py` observes an existing run-owned H1/H2 service using the explicit `lock-timeout` runtime profile and reviewed source `757b751482bb970bc321e92059690a7698554358`. The profile enables only the existing object-lock diagnostic target at TRACE, sets the actual object-lock acquisition budget to one second, disables early PUT rename ACKs, and preserves the standard disabled scanner/automatic-Heal/MRF settings. It does not modify the service binary. Run `python3 -B scripts/validation_heal/lock_observer.py --binding --nodes vm000 [vm001 vm002 vm003] --bucket --entry --output `. The entry contains the original `key`, `versionId`, `deleteMarker=false`, and complete ACK-bound `payload.size`/`payload.sha256`. This component neither deploys nor cleans a namespace; the enclosing case owns those steps and recovery. + +After persisting and sending the original single GET, the observer waits up to ten seconds for its actual server-generated request ID and `get_object` lock attempt. A unique User-Agent marker connects the exact version URI and method in the `http-request` ancestor span to that lock's owner/namespace/attempt. The marker is correlation data, not an identity asserted by the client. Its HTTP span must carry one exact historical version selector; an optional `requested_version_id` diagnostic field must agree with that selector. Free-text messages, nested native-test capture rows, client-supplied request IDs, repeated lifecycle events and foreign endpoint/owner/version records cannot prove admission. Cursor-bounded raw journald pages remain in evidence, and source message loss prevents completion. + +Before disconnect, two `/proc` samples must connect the exact client/server TCP tuple to an actual established socket descriptor of the original candidate process. A network-namespace table entry alone is insufficient: its inode must also appear in that process's FD links. The read-only adapter accepts no arbitrary PID, path, command or service port. Process birth, binary, environment and mount namespace must match, and its network namespace cannot change during sampling. A second journal capture must still show acquired without a diagnostic drop. An unavailable/already-ended window records incomplete evidence and closes the original caller socket without retry. + +After the caller closes, the observer follows the same request/attempt/owner until its `guard_dropped` event, requires two complete samples without the original owned socket, and waits up to thirty seconds for each peer's FD count to return to its baseline. It then reads and verifies the exact historical version's full content and samples resources again. Every transport result remains UNKNOWN. Diagnostic guard drop does not prove distributed release, snapshots do not prove ownership across the disconnect instant, and aggregate FD counts do not prove absence of every leak. Those claims stay explicitly false. The timeout/stale-generation native fences, queued-successor lock closure, complete DL/DV physical and ACK verification, every H1/H2 branch and independent live acceptance remain required; I05 stays outside canonical dispatch. + +For the I05 runtime, process resource baselines and later samples also retain a bounded `descriptorCensus`: FD numbers, descriptor kinds, hashed link targets and TCP rows whose socket inodes belong to that exact process. The caller socket's two samples include the same census. Raw descriptor paths and descriptor contents are excluded. Collection rechecks the original process birth and network namespace, counts descriptors that vanished during the read, and marks every census as a snapshot without a no-leak claim. These records help diagnose a count that does not return to baseline; they do not exclude any descriptor from the original FD threshold or extend the drain budget. + +### I05 object-lock timeout diagnostic controller + +`lock_timeout.py` extends the caller observer with one real contender PUT while the exact historical GET still owns its read lock. Run it only against the already assigned exclusive `lock-timeout` H1/H2 service and the current auto-testing nodes: `python3 -B scripts/validation_heal/lock_timeout.py --binding --nodes vm000 [vm001 vm002 vm003] --bucket --entry --output `. The original historical payload must be at least 32 MiB so the controller can apply and record a bounded 64 KiB client receive buffer, then observe the same established service-owned TCP socket with a non-empty send queue after the contender response. + +The contender body, exact PUT request, endpoint, structured 503/`ServiceUnavailable`/`Retry-After: 1` response and durable ACK are bound together, including the recorded operation, parameters, payload digest and ordered timestamps in the collector's clock domain. The pinned candidate's `rustfs/src/error.rs` maps typed `StorageError::Lock(LockError::Timeout)` to this response and directly tests its header. Cursor-complete server logs must associate that response's distinct server request ID and write-attempt UUID with the same bucket/key namespace and the exact one-second `put_object_commit` timeout. The same-node GET and PUT share the set's owner label; that label is not a per-request identity. The original GET attempt must still be acquired after the timeout and socket samples. Failure, UNKNOWN, a missing journal event, an empty send queue, a small fixture, or binding drift is retained as incomplete or `NOT_OBSERVED`, never retried or passed. The controller checks the original physical scene before any S3 read, compares complete versioned inventory before and after, and waits for process FD counts to return to the original baseline. + +This establishes a live timeout diagnostic subset only: it does not certify ownership at the precise disconnect edge, stale read-repair fencing, queued successor behavior, H1/H2 DL/DV source branches, or canonical acceptance. The native abort/timeout/stale fences and independent live audit remain required. + +`lock_timeout_runner.py` provides the enclosing diagnostic deployment through +the existing Functional Chain workflow. After auditing prior cleanup and +ownership, explicitly select `validation_case=heal.lock-timeout-and-caller-abort`, +the exact `auto_testing_sha`, candidate build run/attempt and successful +inspector build run/attempt. The existing shared-fleet concurrency group and +auto-testing bootstrap install the immutable package. Local source, helper, +workflow identity, vm000–vm003 binding and inspector checks run before +bootstrap; Fleet verifies the installed binary digest before starting each +owned service. H1 runs on vm000, followed by H2 on vm000–vm003 only after +complete H1 diagnostic and namespace cleanup. A fresh artifact tree is required; +the runner accepts the enclosing script's setup/test logs but refuses any old +topology or summary before deployment. + +Each topology uses the existing Fleet, Recorder and DV seeder: two keys have +three original versions each and one current marker; the selected historical +version is 32 MiB to maintain bounded TCP backpressure. This small fixture is a +diagnostic subset, not the complete source DL/DV matrix. After timeout and +caller-drain checks, every endpoint must preserve the entire original ACK +manifest, payloads, metadata and tags. Physical comparison brackets those +reads, and every original process's FD count must return to its baseline before +cleanup. Failed/unknown seeding, contention, observation or peer verification +stops the owned services, saves the original physical/log scene and retains +the namespace and original receipts. A failed diagnostic or cleanup prevents +the next topology. SIGTERM follows the same failure capture and retention +path. The summary records diagnostic completion while keeping whole-case and +canonical claims false. The manual entry returns canonical `BLOCKED` after a +successful diagnostic with clean finalization, preserves diagnostic failures, +and reports `INFRA_ERROR` for an incomplete summary or failed finalization +after a zero exit. The workflow gate therefore remains non-green even when +the diagnostic succeeds. I05 remains excluded from the automatic matrix; +this manual route does not certify its native barriers or complete source case. + +### J02 original opaque metadata probe + +The `unknown-negative` diagnostic includes the source procedure before the supplemental legacy-object diagnostic. A separate acknowledged object first receives an independently healthy Deep receipt. With the owned services stopped, the existing descriptor-relative fault backend flips byte zero of every original `xl.meta` copy: four copies in H1 and sixteen in H2. Each mutation has its own original backup, receipt and complete physical delta; the candidate-bound decoder must refuse every exact damaged copy. No other metadata, part or format may change. + +A separate protected prefix is checked during the fault. The original key-level task must retain complete truthful counters and an unresolved disposition, with no invented healthy, repaired, absent, version or incarnation claim. Unknown must be counted within skipped and carry the legacy warning. If the candidate reports another truthful unresolved disposition, record `NOT_OBSERVED`; the legacy diagnostic cannot promote it to PASS. Archive the original terminal result and unchanged complete fault scene before restoring original metadata bytes. Verify the full original physical and SDK inventories and the unchanged original token after restoration. Fixture rescue is never product repair. J02 remains outside canonical dispatch until the separate version-specific grace Deferred/new-token procedure and complete live evidence are available. + + +### I05 retained-scene isolation + +The reviewed H2 startup failure `39764496504/1` has a separate preservation +plan pinned to helper `31b787c51de23587da1f9a2ac0f910400ddf66b3` and the +SHA-256 inventory of all 102 official artifact files. H1 completed its timeout +diagnostic and actual root cleanup; H2 reserved all four peers, then failed +before fixture seeding because vm002's bpftool wrapper could not find its +kernel-specific executable. All four stopped original scenes contain no +business files and retain the same complete sixteen-disk format mapping. +The planner requires those exact original owners, candidate, inspector, +cleanup receipts and failure boundary before allowing the existing +audit/isolate/post-audit protocol. It issues no workload request and changes +neither the original FAIL nor canonical acceptance. + +Isolation [39766457981/1](https://github.com/rustfs/rustfs/actions/runs/39766457981) +completed `ISOLATED` with helper `fa0b659`. Independent audit byte-matched all +102 original files and all four preflight/isolation/post-audit baselines. +All twenty data/state archive parents are root-owned mode 0700 and on their +original devices; original tree, owner and inode identities are unchanged. +No S3 request, backup restoration, namespace deletion or original verdict +rewrite occurred. + +`environment.inspect` now also records the selected bpftool path, installed +`/usr/lib/linux-tools/*/bpftool` executables, resolved root-owned executable +digests, kernel release, version exit status and a read-only root-cgroup JSON +query. A working installed executable is distinct from a broken PATH wrapper. +This inventory does not install tools, attach programs, change PATH or certify +the I05 service filter; the actual owned-service attachment checks remain +mandatory before another diagnostic. + +Read-only inventory +[39766570688/1](https://github.com/rustfs/rustfs/actions/runs/39766570688) +found broken PATH wrappers on both vm002 and vm003, which run kernel +`6.17.0-1022-azure`. All four peers already have +`/usr/lib/linux-tools/7.0.0-1017-azure/bpftool` with executable SHA-256 +`aabc337b58169d62c42b3c8f1fccf605e5fc1e445a23feb0156638e9d9606991`. +The executable reports bpftool 7.7.0/libbpf 1.7 and its root-cgroup query exits +zero on each peer. That empty root query alone does not verify service BPF +support. I05 now uses this exact installed executable with protected root-owned +path, digest and version checks, records its identity beside the actual +service attachments, and checks every peer before reserving any namespace. +It changes no installed tool, global PATH or kernel. Service-bound ingress and +egress program identity checks remain mandatory on the running kernel. + +Diagnostic [39767336741/1](https://github.com/rustfs/rustfs/actions/runs/39767336741) +completed both H1 and H2 with helper `8cbdd51` on the unchanged candidate +`757b751482bb970bc321e92059690a7698554358`, package `39364893106/1` and +inspector `39566504743/1`. Independent audit of all 211 official artifact +files verified nine seed request/ACK pairs and seven versions/markers per +topology, candidate/helper bindings, actual process-bound ingress/egress BPF +programs on all peers, complete journal cursor chains and exact server +request/lock attempt identities. Each contender returned a known non-retried +503 after a 1001 ms write-lock timeout while the original GET remained +backpressured. The caller connection disappeared and its diagnostic read +guard dropped; original physical files remained identical before public +verification and all one/four peer inventories agreed with seed ACKs. + +Final FD counts were H1 vm000 `22 -> 22`; H2 vm000 `106 -> 94`, vm001 +`108 -> 104`, vm002 `105 -> 101` and vm003 `101 -> 97`, with unchanged process +birth and network namespace. These are bounded snapshots, not a global +absence-of-leaks proof. All five node cleanup receipts show stopped services +and removed owned roots. The original result is `BLOCKED/cleanup PASS`: +the workflow fails its deliberately retained canonical gate despite successful +diagnostic execution. Exact disconnect-edge ownership, distributed release, +complete native timeout/stale fences, the stale-generation controller +and full source H1/H2 DL/DV coverage remain unaccepted. The separately audited +queued-successor diagnostic is recorded below. No new +canonical case is counted. + +`validation_cleanup_run_id=` selects the narrow I05 preservation +route with the original package build and an immutable helper containing +`timeout_retention.py`. The driver must remain on the original Functional Chain +branch. First check the shared current auto-testing fleet for other work. The +planner accepts the original helper 704bd60 failure before contender +submission and the reviewed helper d100abeb failure after one exact known 503 +contender refusal. It also accepts helper 1b88cad when H1 completed and cleaned +successfully before H2 reached that same exact contender failure. The H1 path +requires nine exact successful seed request/ACK pairs, complete +version/marker/payload/public metadata baseline, a closed read-only historical +GET, original owner/candidate/inspector and an unchanged stopped four-drive +scene. The contender boundary also verifies its request/ACK digest, server +request ID, full bucket-incarnation read-lock lifecycle and exact failed target +write-lock lifecycle. The H2 path additionally requires four reservations, +four inspector uploads, per-node lock checkpoints and balanced four-peer +journal rounds before disconnect, after admission, and during contender +observation, four stopped physical scenes and four service logs. +Unknown or successful writes, missing peers, another action, an unclean H1 or +physical drift refuse recovery. + +The recovery reuses retained-scene audit/isolate/post-audit, retaining all four +roots and state under private per-run parents without S3 calls, backup +restoration or deletion. Verify all original archived files byte-for-byte and +all five final private parents independently before recording active namespace +isolation. `ISOLATED` never changes the original FAIL or supplies canonical +acceptance. H2 recovery requires the completed H1 diagnostic and its successful +cleanup; it isolates all four H2 peers together when the H2 contender boundary +fails. In run 39733064688, the server refused the contender after a 1001 +ms target write-lock timeout while the exact GET held its read lock for 4.148 +s; the validator failed because it combined the same PUT's bucket-incarnation +read-lock events with the target write-lock events. Read-only inspection +[39738071753/1](https://github.com/rustfs/rustfs/actions/runs/39738071753) +observed vm000 with no RustFS service, validation unit or port 9000 listener. +Recovery [39738426191/1](https://github.com/rustfs/rustfs/actions/runs/39738426191) +completed `ISOLATED` for vm000: all 65 failed-artifact files match +byte-for-byte, preflight/move/post-audit physical digests match, and four data +roots plus private state are retained in five root-owned mode-0700 same-device +archive parents. No backup restoration, namespace deletion, S3 request or +original FAIL rewrite occurred. Inspection also observed old failed F07 units +on vm001-vm003; the I05 plan did not touch those nodes. + +The first live run's GET already dropped its lock before observation. The +version-field correction correlates those original events but cannot reopen +that ended window. The exact candidate source and runtime log differ on the +optional version field; the cause remains unresolved. Any subsequent I05 +controller must establish a real streaming/IO ownership window with evidence; +repeating the same warmed full-read premise does not provide that proof. + +Source review narrows the next experiment without changing the oracle. The +candidate defaults the object-data cache off, and an explicit `versionId` +excludes the direct-memory GET path. The 32 MiB fixture is non-inline, while +codec streaming rollout is off by default, so this run used the legacy duplex +reader. `ObjectLockDiagGuard` carries the real namespace guard; the +`SetDiskLockGuardedReader` empty-read path can release that wrapper before the +HTTP response completes. The live journal shows `guard_dropped` before the +seven-millisecond HTTP completion and a later `get_object_stream_body` +`dropped_incomplete` event. This is a source-supported hypothesis for a +controlled streaming/IO reproduction, not a confirmed cause or a canonical +acceptance result. + +I05 descriptor run `39754893467/1` pins helper +`dd7e54e6672cab76b5a6245ab3b4fbd051ddd39d` and the unchanged package from +`39364893106/1`. H1 retains its original FAIL/cleanup FAIL at the unchanged +30-second drain budget; H2 did not start. The baseline owns 22 descriptors. +Every drain resource snapshot owns 23, with all baseline fingerprints intact +and exactly one additional TCP descriptor from remote `168.63.129.16` to local +port 9000. The caller's exact GET socket is absent, and its read-lock journal +contains `guard_dropped`. These snapshots establish external connection +interference; they do not prove the platform probe configuration, continuous +socket absence, no leak, or complete canonical acceptance. + +Preservation recognizes only this additional exact helper and its original +known 503 refusal, closed read-only GET, complete journal lifecycle, stopped +physical scene and bounded request/ACK set. The original 101-file artifact is +retained in `testdata/validation/i05-descriptor-drain-39754893467.zip`. +Descriptor counts, unique FD identities, socket-link fingerprints, owned TCP +joins, snapshot flags, process birth and network namespace must agree. +Missing census, unknown/successful mutation, extra request and physical drift +refuse preservation. Drain journal windows follow the numbered first-drain +label rather than a fixed count of preceding caller windows. This change +permits preservation only; it performs no network change and grants no PASS. + +Isolation run [39757904633/1](https://github.com/rustfs/rustfs/actions/runs/39757904633) +completed with helper `f43151595648d83b184ab95fd3e908e7611d0dae`. +An independent artifact audit confirms all 101 failed-run files are byte exact, +preflight/move/post-audit baselines are identical, and all five archive parents +are root/root 0700 on their original devices. Original business-scene digest +`f921b69ea51622f89e9ad4a78de8c890caf0b7ffefa0b401457d6f9796fc7d31` +is unchanged. Only the owned active paths were isolated; no namespace was +deleted, no backup was restored and the original FAIL/cleanup FAIL remains. +The inventory now reports existing `bpftool` availability before selecting a +service-scoped network filter; this report installs no tool or filter. +An earlier dispatch `39757736435/1` was rejected before candidate resolution +because the submitted helper was abbreviated; no fleet action ran there. + +Read-only inventory [39758842602/1](https://github.com/rustfs/rustfs/actions/runs/39758842602) +with helper `c5d1890467599e27aeffeb5dbf1ed4b37141fd06` confirms that all four +current fleet nodes already provide `bpftool` and cgroup v2, and none listens +on port 9000. This observation does not certify global idleness or historical +scene cleanup. Existing failed F07 units remain outside this operation. + +The I05 `lock-timeout` runtime now starts its private transient systemd unit +with `IPAddressDeny=168.63.129.16/32` and no allow override. Startup and every +resource sample require exact effective properties, the owned MainPID's +membership in that unit's cgroup, both actual `sd_fw_ingress`/`sd_fw_egress` +attachments, and root-owned `cgroup_skb` program identities/tags from bpftool. +Missing support, ignored settings, foreign programs, aliased cgroups and +process changes fail closed. This policy isolates the observed external +source from this test service in both directions. Other units and the host +firewall are untouched; the policy has the private service cgroup's lifetime. +It does not identify the platform's probe configuration. Resource evidence +retains the effective properties and kernel program IDs/tags as snapshots. +The total FD threshold, caller/request identity and 30-second budget remain +unchanged; complete I05 canonical acceptance is still required separately. + +Filter startup run [39761856012/1](https://github.com/rustfs/rustfs/actions/runs/39761856012) +used helper `a9725c88acd4977545b278eebe3653c916f907f8` and stopped at the +kernel attachment check, before seeding or issuing the diagnostic requests. +Its stopped H1 scene has four original format files and no object files; all +22 original artifact files and FAIL/cleanup FAIL remain retained. The parser +had accepted only the older bpftool `ingress`/`egress` attach type strings. +Newer bpftool uses libbpf's `cgroup_inet_ingress`/`cgroup_inet_egress` names for +the same kernel types. Both exact spellings now map to a checked direction, +while evidence preserves the original attach type. Unknown, duplicate or +missing directions and non-string flags still fail closed. + +The startup preservation route requires that exact run/helper and all 22 +original file hashes, original candidate/package/binary, stopped owner and +empty object scene. It reuses the existing same-device, private-parent +isolation backend and its live owner/physical/tree checks. No workload is +replayed and no namespace is deleted. A successful isolation or corrected +startup check cannot certify the I05 source case. + +### I05 queued-successor diagnostic + +Select `I05_DIAGNOSTIC_BRANCH=queued-successor` for the explicit I05 route. +`run_case.sh` passes this branch to both preflight and the runtime controller. +The Functional Chain input `i05_diagnostic_branch=queued-successor` downloads +the pinned native artifact and verifies its original hosted run, candidate, +helper, binary, exact regression logs and required I05 labels before fleet +mutation. `VALIDATION_HEAL_NATIVE_DIRECTORY` supplies that bundle locally. +Native success remains a prerequisite and never certifies the live case. + +`lock_successor.py` uses a separate `lock-successor` runtime: the GET lock +budget remains one second, while the successor PUT commit acquisition budget +is sixty seconds. It starts exactly one marked PUT while the original +historical GET is admitted, observes the PUT acquiring the same namespace, +then closes only the original caller socket. It requires the original GET +guard-drop event before successor acquisition, an unretried server-request-ID +bound PUT ACK, exact ETag/version/payload, preserved historical generations, +a distinct new physical generation, all-peer S3 agreement and bounded FD +drain. Observation windows remain ten seconds for queue admission and thirty +seconds for completion. The fixture is a diagnostic subset of the source DV +workload; it does not complete the source DL/DV matrix. + +Failure stops the run-owned service and retains its original requests, ACKs, +logs and physical scene. `successor-retention.json` binds all diagnostic files +and stopped per-node physical digests to the original owner. Recovery accepts +only an unchanged sealed archive and the same stopped live scene, then uses +the existing same-device private-parent isolation backend. An unknown PUT +outcome is preserved without retry, deletion or old-generation restoration. +Incomplete capture, missing reservations, changed evidence or scene drift +refuse automated isolation. + +The exact-helper native/tools build +[39783997956/1](https://github.com/rustfs/rustfs/actions/runs/39783997956) +passed independent archive, source, executable and ten-regression audits. +The first successor live run +[39788321908/1](https://github.com/rustfs/rustfs/actions/runs/39788321908) +retains FAIL/cleanup FAIL: the same PUT read its bucket's internal +`.metadata.bin` after target write-lock acquisition; the controller +incorrectly classified that read as a target write. The request returned +one unretried 200 ACK, which does not complete the failed diagnostic. + +[39789732916/1](https://github.com/rustfs/rustfs/actions/runs/39789732916) +isolated the unchanged stopped H1 scene without deleting it or restoring +old generations. The parser recognizes only that bucket's exact +`.metadata.bin` and `.bucket-incarnation` read paths, checks their original +HTTP identity, namespace, one-second budget and ordered per-attempt +lifecycle, and excludes them from the unique target write attempt. The +original fifteen-event trace is a regression fixture; foreign paths, +write-mode substitutions and reused or malformed lifecycles still refuse. +A new helper requires a newly bound native/tools build before another live +attempt. Fresh exact-helper native/tools +[39791663588/1](https://github.com/rustfs/rustfs/actions/runs/39791663588) +passed independent verification of all three official archives, 22 original +members, ten exact tests and 104 ordered milestones. The next live run +[39798325977/1](https://github.com/rustfs/rustfs/actions/runs/39798325977) +passed the metadata-read classifier and acknowledged one new PUT version, +then retained FAIL/cleanup FAIL because the controller invoked stopped-only +streaming shard verification while the service was running. + +[39799657074/1](https://github.com/rustfs/rustfs/actions/runs/39799657074) +isolated that stopped H1 scene. Independent official archive verification +byte-matched all 71 original files and all five pre/move/post tree, device +and inode baselines. The new shard's streaming checksums now run only after +all online peer reads and FD observations, under the unchanged stopped-only +verifier. Stopped physical snapshots must match the ACK-bound pre-read scene +both before and after the checksum pass. Drift or checksum failure retains +the original scene and prevents cleanup. Exact helper `6fbc3c4` tools +[39803149039/1](https://github.com/rustfs/rustfs/actions/runs/39803149039) +passed independent audits of all three official ZIPs, 22 original members, +ten exact native tests, 104 ordered milestones and inspector contracts. +Live [39810035277/1](https://github.com/rustfs/rustfs/actions/runs/39810035277) +completed the new H1 stopped-integrity sequence and cleanup, then retained +H2 because a slow-acquire WARN sharing the diagnostic target was parsed as +a lifecycle event. The WARN has no event/state/attempt/namespace fields; +it reports a 1214 ms acquisition against a 500 ms warning threshold. It is +not a timeout or a separate write attempt. The selector recognizes this +exact auxiliary shape only after the same request's acquired target write, +retaining strict HTTP identity, owner, numeric timing and lifecycle checks. +Unknown warnings and lifecycle substitutions still refuse; the auxiliary +warning cannot establish acquisition or guard drop. + +Isolation [39811143272/1](https://github.com/rustfs/rustfs/actions/runs/39811143272) +preserved all 180 original failed files without modification. Independent +archive audit verified all four nodes' pre/move/post tree, device and inode +baselines and twenty root-owned 0700 archive parents on their original +devices. No S3 replay, namespace deletion, old-generation restoration or +original verdict rewrite occurred. + +Exact helper `3012a65` tools +[39812960916/1](https://github.com/rustfs/rustfs/actions/runs/39812960916) +passed independent verification of three official ZIPs, 22 original members, +ten exact native tests, 104 ordered milestones, actual hosted checkouts, +source/tree/Cargo.lock, optimized ELF and inspector quorum/size contracts. +Fresh live +[39817606025/1](https://github.com/rustfs/rustfs/actions/runs/39817606025) +completed H1 and H2 with five passing node cleanup receipts. Independent +audit byte-matched all 246 official files and verified nine seed request/ACK +pairs, eight final versions/markers per topology, one unretried successor +PUT ACK, retained historical physical generations, actual lock lifecycle +events and every peer's complete inventory. Stopped physical snapshots match +the ACK-bound scene before and after all four H1 and sixteen H2 new-shard +streaming receipts. Final FD counts were H1 vm000 `22 -> 22`; H2 vm000 +`108 -> 96`, vm001 `117 -> 113`, vm002 `105 -> 101`, vm003 `108 -> 104`. +Both diagnostic branches are independently accepted, with the original +`BLOCKED/cleanup PASS` unchanged. Precise disconnect-edge ownership, +distributed release and global absence of leaks are explicitly unproven; +the complete source DL/DV fixtures were not covered. +Every successful diagnostic +remains `BLOCKED/cleanup PASS` with `canonicalCasePassClaimed=false`. Exact +disconnect-edge ownership, complete native timeout/stale fences, the +stale-generation controller and full H1/H2 source DL/DV ACK/physical/lock/FD +closure remain required. Migration acceptance stays 108/158. diff --git a/docs/validation-migration-implementation-review.md b/docs/validation-migration-implementation-review.md new file mode 100644 index 00000000..5c62603c --- /dev/null +++ b/docs/validation-migration-implementation-review.md @@ -0,0 +1,123 @@ +# 用例迁移实现审查 + +审查日期：2026-10-28。冻结源 catalog 提交为 +`524eb5b32c3236497fdd8a9271df0494fa3d9974`，SHA-256 为 +`463fdd02dee43e42fef4f3aacf7506709ce87af535e1a53d91c521bc2e361cad`。 +本次交付范围是 auto-testing 中的测试流程、原断言和证据采集。产品执行结果独立保留。 + +## 实现和执行结果 + +| 测试实现状态 | canonical 用例数 | 含义 | +| --- | ---: | --- | +| 已实现 | 145 | 各源分支已有执行路径、原断言及证据采集；其中 115 项已有完整 live 审计，30 项运行失败或尚未完整验收。 | +| 环境条件限制，默认关闭 | 13 | 操作流程、能力检查、原断言和未满足原因已保留；执行需要明确的资源及经过审核的资源适配器。 | +| 实现部分完成 | 0 | 已补齐 F07、I05 的测试接线和源流程总控。 | +| 合计 | 158 | 保留全部源 canonical ID。 | + +独立 live 状态仍为 115 项 accepted、22 项 retained-failure、4 项 pending、 +4 项 blocked、13 项 disabled。共 43 项尚无完整 live 验收，其中 13 项按约定默认关闭。 +这些结果跨多个不可变候选和历史批次，不构成某个候选通过全部 158 项的产品认证。 +22 项失败需要逐例归因，不能统一宣称为产品缺陷，也不能直接据此认定测试没有迁移。 + +`validation-migration-review.json` 列出全部 158 个 ID、实现状态、代码/测试引用、 +源分支、缺口及独立 live 状态。`reviewed_files` 固定引用文件摘要，变化后须重新审查。 +文件存在和本地测试通过不代替源语义审查或 live 审计。本文件不覆盖原始结果。 + +```bash +python3 -B scripts/validation_migration_review.py --require-complete +``` + +此门禁仅因测试实现缺口失败，产品 FAIL 不影响实现计数。13 项环境流程另行输出所需 +适配器清单。产品门禁 `validation_report.py` 仍要求源断言、清理、来源和完整 live PASS。 + +## F07：持久化临界点 + +冻结源 `docs/heal-cases.md#F07` / `scenarios/heal-recipes.json` 要求精确本 run 的 +checkpoint/MRF 故障、持久化责任保留、损坏记录保留，以及独立 native fsync/commit +窗口补证。入口 `persistence.py` 组合以下已存在的能力： + +| 能力 | 证明范围 | +| --- | --- | +| `persistence.py` / `persistence_mrf.py` | 五种 live 字节损坏、写失败及重扫分支，保留原责任、失败和恢复证据。 | +| manager checkpoint-page-order / checkpoint-stale-manager | 正常发布和 reload 次序；旧 manager 不能覆盖新 cursor。正常 reload 不冒称提交中断。 | +| manager mrf-migration-boundaries | 实际 AfterPayload、BeforeManifest、AfterManifest、AfterReadback 四个中断窗口及保留/重试断言。 | +| manager root-uncertain-publication | 实际终态写入后注入故障，保留 pending owner，重启隔离旧任务。 | +| `persistence_native.py` | 六个 ECStore CAS/MRF 目录 fsync 故障回归，覆盖更新、首次创建、删除和恢复锚点。 | + +checkpoint 通过 `EcstoreDiskAPI::compare_and_update_file` 落盘。manager 的发布次序、 +中断/重启责任与 ECStore CAS 落盘失败分别补证；没有为每个 fsync 发明额外测试要求。 +审查源码为允许的不可变候选 `072b57b37559fbe626aeda8abf61617fdcbdd926` 和 +`1aa9be87d18cf941e5020f19e8622c26ba4b10ac`；完整目录 fsync 组合只允许后者。 + +`collect_native_persistence` 在执行前核对 native manifest 与 fleet 候选，执行六项回归 +后独立验证并再次核对候选；`native_branch_result` 再核对 manager/native 同候选。 +成功和失败分别写入 `independent-native-persistence.json` 与 +`independent-native-persistence-failure.json`，原始 native 日志保留在 `native-persistence/`。 +缺少 artifact 时只允许原诊断子集，不能宣称完整 native coverage。 + +使用当前 auto-testing 环境时，由现有 workflow 传入以下目录： + +```text +VALIDATION_HEAL_MANAGER_DIRECTORY/ + tool-run.json + manager-tools/ + manager-evidence/ +VALIDATION_HEAL_NATIVE_DIRECTORY/ + tool-run.json + native-tools/ + native-evidence/ +``` + +伴随 workflow 在精确目录 fsync 候选上下载已有 native artifact，并将目录传给入口。 +同一候选的原生结果仍是独立 prerequisite；不代表 H1/H2 fleet 命中了内部窗口，不声称 +物理断电或每个 file fsync hook 已执行。F07 的完整独立 live 审计待完成，canonical 门禁保留。 + +## I05：源流程总控和代次证明 + +`lock_source.py` 复用 `lock_timeout_runner.py`、`lock_timeout.py`、`lock_successor.py` +及 `lock_observer.py`。默认 `source-plan` 顺序执行 timeout/H1、timeout/H2、 +queued-successor/H1、queued-successor/H2，每次使用独立 run ID、bucket 和证据目录。 +任何分支、证据链或清理失败立即停止后续部署，保留原请求、ACK、日志及停机现场。 + +部署前只读验证同一成功 hosted artifact 中的三个精确源码 native 子集： + +| 子集 | 证明范围 | +| --- | --- | +| I05 owner/unlink/stale | 历史 read-repair owner、取消 physical unlink owner 和原有 stale final rename 回归。旧 stale 测试只检查 ETag。 | +| G02 explicit-version append | 真实缺片重建在 final rename 屏障暂停；新版本 ACK 和物理 tail drain 后释放；检查类型化结果、全部存活 body、版本、dataDir 和逐盘 shard。 | +| G03 explicit-version delete | historical/current 删除与最终提交竞态，保留上述完整内容和物理代次断言。 | + +G02/G03 的较强证明来自已有原生测试，不给旧 ETag 测试补写能力。两组覆盖 4/2 与 +16/4，分别保留 6 与 12 条有序屏障里程碑。append 可能安全修复原选定版本并保留新版本， +不把所有类型化结果统一描述为 stale 拒绝。候选固定为 +`757b751482bb970bc321e92059690a7698554358`；三个 proof 的 package/source tree/ +Cargo.lock/二进制/工作流 attempt/helper/contract 身份必须一致。 + +原生前提可单独检查，命令不执行下载的二进制： + +```bash +python3 -B scripts/validation_heal/lock_source.py \ + --directory "$VALIDATION_HEAL_NATIVE_DIRECTORY" \ + --candidate candidate.json --testing-commit "$AUTO_TESTING_SHA" +``` + +fleet 控制器保留 DV 历史版本和 DL 单次并发请求/ACK ledger，继续核对所有 peer 的 +完整对象/版本、停机物理分片、锁事件和有界 FD 收尾。DL 是工作负载语义，不是新存储格式。 +新证据链关联请求写出、超时/排队、TCP 断开、原 guard drop、successor 获取和观察结束； +对原始 bytes 逐一固定 SHA-256，核对请求/lock owner/binding 和同一协调者时钟顺序。 +不会比较不同主机 monotonic 时钟，也不把本地 guard drop 当成分布式 owner 释放。 + +`source-plan-summary.json` 记录四个源分支的 fleet/native 证明范围； +`sourceProceduresComplete` 只有 H1/H2 四个实例完整且清理通过才为 true。单独选择 timeout +或 queued-successor 仍是诊断子集。全部控制器完成也不自动增加 canonical PASS；完整 +source-plan 的独立 live 审计仍待执行。 + +## 失败归因和后续验收 + +C11 的旧 metadata、checkpoint、MRF replay 和 native incarnation 均有执行路径； +J02 的混合 outcome、dry-run、deferred 和 unknown-negative 均有控制器。其历史产品 +结果和证据限制保留，尤其不能修改 J02 的 Failed/Unknown oracle 来获取全绿。 + +G01/G02 的失败 PUT 不重试，未知 mutation 不算成功。B03/B04 仍缺精确 mount/切换 +适配器，在部署前阻断。今后按产品缺陷、环境前提、证据缺失和测试实现问题逐项归因； +有明确测试实现问题才重开实现缺口。实现交付完成不代替候选包的后续回归与产品验收。 diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md new file mode 100644 index 00000000..e92ecad9 --- /dev/null +++ b/docs/validation-migration-map.md @@ -0,0 +1,2861 @@ +# Release validation case migration + +Source: `rustfs-release-validation/scenarios/catalog.yaml`. The generated +`validation-cases.json` preserves all 158 canonical IDs, source statuses, +assertions, probes, capabilities, and target environments. Its source commit +and SHA-256 bind the snapshot to the source catalog. `reuse_candidates` lists code to inspect; +it is not a coverage claim. A case's run result remains `unverified` until a +live run proves each source assertion on the required topology. + +## Current implementation scope (2026-10-28) + +The migration delivers test procedures and original oracles in auto-testing. +Candidate live failures are recorded independently from missing test code. +The explicit [implementation review](validation-migration-implementation-review.md) +and `validation-migration-review.json` record **145 implemented cases, +13 default-disabled environment procedures and zero partial implementations**. +The 13 procedures require reviewed resource adapters to execute. F07 composes +same-candidate manager interruption and ECStore directory-fsync evidence. I05 +runs timeout and queued-successor on H1/H2 in one source plan, with separate +same-artifact native owner and full-content/physical generation checks. All +30 implemented cases without full live acceptance retain their candidate +failures or pending evidence independently. Source-path review and local +contract tests do not certify every live branch. + +Run `python3 -B scripts/validation_migration_review.py --require-complete` for +the independent counts and implementation gate. It exits zero when the source +implementation has no recorded gaps; retained product FAIL does not change +that decision. `validation_report.py` continues to enforce original assertions, +cleanup and provenance for product acceptance. Unknown failure causes remain +unclassified. Detailed execution prerequisites and evidence scopes are in the +[implementation review](validation-migration-implementation-review.md). + +## Current live acceptance and retained evidence + +Count reconciliation (2026-10-23; updated 2026-10-28): the append-only ledger +`canonical-migration-ledger-20261023-b08-39915022106.json` follows the +B01 ledger and attaches the complete B08 live acceptance plus the latest B02 +failure evidence. The B02 complete acceptance below is a later independent +record; it does not replace that retained failure or replay all 158 cases on +one candidate. + +All 158 canonical IDs are retained. Independent complete live acceptance is +**115/158**. The current review retains 22 live failures, four pending +native/live race acceptances, four orchestration/evidence blockers and 13 +environment cases disabled by default. G01 and G02's actual failed PUTs are +included in retained failures; historical ledger bytes are unchanged. These +30 unaccepted active cases are a live acceptance backlog, not 30 missing test +implementations. Counts span immutable candidates and historical audits; +diagnostic, native, isolation and partial results do not increment PASS. + +| Reconciled case | Complete live run | Required coverage | Original result SHA-256 | +| --- | --- | --- | --- | +| C02 `heal.data-shard-bitrot` | [39281951263/1](https://github.com/rustfs/rustfs/actions/runs/39281951263) | Eight H1/H2 branch instances, 3,334 archive files | `009c96a4bbd1b8c711009c0223519f6f12d8762890d597db749f9e415ca23f1a` | +| C03 `heal.parity-shard-bitrot` | [39286997859/1](https://github.com/rustfs/rustfs/actions/runs/39286997859) | Six H1/H2 branch instances, 2,904 archive files | `da2df2e15aa973b6a04a0cb40491413db17426b63d6367f11682ef579b84235a` | +| C05 `heal.missing-or-corrupt-xlmeta` | [39322217398/1](https://github.com/rustfs/rustfs/actions/runs/39322217398) | Ten H1/H2 DB/DV instances, 3,132 archive files | `489b8179ee90a7412b223e669db8fd4f6520b9e69051cbf927b6f33299de423b` | +| I04 `heal.queue-pressure-and-admission` | [39354651483/1](https://github.com/rustfs/rustfs/actions/runs/39354651483) | Four branches per topology, six exact durable tasks, 5,565 archive files | `4dcde2f34bb7df52f03f5a3ec1e87663145a7134d3ef1a513fda2da52d1709a2` | +| B05 `heal.multiple-shard-loss` | [39843970314/1](https://github.com/rustfs/rustfs/actions/runs/39843970314) | All 38 DB/DV and H1/H2 instances, 9,059 archive files, ten node cleanups and four package purges | `2624f43fc8581eb1b533a477b1a869f9c72208861d7a268ea35ae8f36153fd7e` | +| E04 `heal.scanner-cold-object-repair` | [39863505276/1](https://github.com/rustfs/rustfs/actions/runs/39863505276) | All six H1/H2 instances, 2,307 archive files, five node cleanups and four package purges | `8bfcfb03bfa5cc88649e98c4f9a6fa7a27d64f9fd0cdbe8d851f57fc00bc0702` | +| A05 `heal.admin-overlap-and-force-start` | [39892106641/1](https://github.com/rustfs/rustfs/actions/runs/39892106641) | All eight H1/H2 overlap and force-start branches, 20 cleanup receipts | `4a8073b6fe129a5bf696ffbc74b0024f1b23e66232606df13c7f606e0511b036` | +| B01 `heal.disk-loss-and-repair` | [39899825839/1](https://github.com/rustfs/rustfs/actions/runs/39899825839) | H1/H2 original-drive-return and offline-ACK repair, 272 fixture ACK/request pairs, five cleanups | `fe89fbb0e4e2d93aa5c4408b9218c66d6caa9120f067f3aca3560f4ca360c4e5` | +| B08 `heal.below-write-quorum` | [39915022106/1](https://github.com/rustfs/rustfs/actions/runs/39915022106) | H1/H2 write/lock quorum loss, H1 lock-loss not applicable, 3,463 archive members, ten cleanups | `b5fbc87b2a9e7dab2b07301c8703f0d1733ee44f11ef984b3bce55b65b5df87e` | +| B02 `heal.disk-readonly-and-permission` | [39927051614/1](https://github.com/rustfs/auto-testing/actions/runs/39927051614) | H1/H2 read-only and permission-denied, same-process gate release, exact shard restoration and five cleanups | `ba2f2569f3799a2aed014c9b89d37a80075774e76dfbfd1b3a06c04c38cb533e` | + +| Acceptance state | Cases | Remaining requirement | +| --- | ---: | --- | +| Independently accepted complete live PASS | 115 | Preserve original provenance and complete results. | +| Retained live failures with existing executors | 22 | Preserve original results and attribute the cause; product failures do not imply missing test implementation. | +| Native and live race acceptance pending | 4 | A09 and G03–G05 await complete live acceptance; existing native/runtime paths retain their candidate gates. | +| Orchestration or evidence blocked | 4 | C11/F07/I05/J02 have source execution paths awaiting independent complete live acceptance; no implementation gaps remain in the current review. | +| Environment gated, disabled by default | 13 | Control flows, oracles and unmet reasons are implemented; dedicated resource adapters and real execution remain conditional on the required resources. | + +The current delivery keeps the 13 environment cases disabled with explicit reasons. The other 30 unaccepted cases retain their original live states independently of their completed implementations. C11/F07/I05/J02 have explicit execution routes, while canonical admission stays blocked until independent complete live acceptance. J02's real four-disk regression reproduced scheduler-level whole-batch retries overwriting an earlier repair receipt. RustFS commit `7951bb629821246868ee3761b321dd07c06db45c` retains exhausted Admin batch results; its 210 manager tests and five process-isolated receipt integrations pass. + +### Native prerequisites and runtime admission (2026-10-26) + +Hosted-only tools [40086026324/1](https://github.com/rustfs/rustfs/actions/runs/40086026324) completed successfully on candidate `757b751482bb970bc321e92059690a7698554358`, auto-testing `002473b51f5ceeccadedf7325a1b1691e4c71dcd` and driver `b1f7fb276941d7128b5ddbbda9c3e6536931c89d`. Its immutable package is [40081956660/1](https://github.com/rustfs/rustfs/actions/runs/40081956660), SHA-256 `1a52387a8cbdef586be2616b02298f351aec4ac2d5c0f7a76d094409c89c0b3c`. Independent read-only verification checked the candidate source tree, registered source-file digests, Cargo.lock, executable digest, ten exact nonignored passing test invocations, all 104 ordered milestones and the inspector identity. The original artifacts and audit are retained in the migration evidence store. No downloaded native executable was run during this audit. + +The registry exposes eight native prerequisite groups: A09, G01–G06 and part of I05. This is separate from runtime admission. The pinned helper admits A09, G05 and G06 on this candidate; it intentionally rejects G01–G04 on the same source before bootstrap because their runtime candidate contracts remain pinned to other reviewed sources. G01–G03 may use their reviewed version candidate `bd0671ba781d8ebf509f4f03533228dfb44496f1`, and G04 uses marker candidate `8380466f3a27c3c3ef4115084178a0ab99a78f32`, each with its own exact package and matching helper/tools. Alternatively, a separate source review and matching tools build must precede extending runtime admission. Do not infer admission from registry membership or mix artifacts from different candidates or helpers. + +All ten native tests pass. G06 subsequently passed all eight H1/H2 public branches in [40427815826/1](https://github.com/rustfs/rustfs/actions/runs/40427815826); an independent audit verified its ACK, physical, peer S3, overlap and 20 cleanup receipts, adding one canonical PASS. A09 and G01/G03–G05 still need their complete source branches and clean finalization. G02's native/runtime prerequisites were admitted, but its first live run failed before completing the required matrix and is recorded below. I05 now composes the timeout/disconnect controllers and separate full-content/physical native generation checks through `lock_source.py`; complete independent source-plan live acceptance remains pending. + +### G02 retained live failure (2026-10-27) + +G02 `heal.race-versioned-put` run [40444207551/1](https://github.com/rustfs/rustfs/actions/runs/40444207551) +used RustFS candidate `bd0671ba781d8ebf509f4f03533228dfb44496f1`, +auto-testing `7094c16a5a5cf3b0f818c7be463f3bc0f8fd7645`, and the audited +candidate-bound native tools from [40311010126/1](https://github.com/rustfs/rustfs/actions/runs/40311010126). +The H1 `new-version-vs-historical-heal` branch failed at foreground ACK 000003: +PUT `objects/0256.bin` returned HTTP 503 `ServiceUnavailable` after 5.026 s, +with zero retries. The service log binds request +`2b5289c8-0faa-472d-b5c6-fe9db521659d` to `storage_error_code=Some(Lock)` and +records a contemporaneous `rename_data` operation pending for over 5 s on +`/data/rustfs4`. This supports a Heal/PUT lock-contention diagnosis; the log +does not correlate that rename operation to the request ID, so it is not a +complete call-level causal trace. This was a real failed PUT, not a recorder +false positive or successful ACK. + +The branch result is `FAIL/cleanup FAIL`; its last-ACK receipt records +`unresolvedMutation=true` and +`lastACKPhysicalAuthorityVerified=false`. Evidence-based recovery run +[40444795627](https://github.com/rustfs/rustfs/actions/runs/40444795627) +isolated the active namespace. It did not restore the scene, delete the +namespace or rewrite the verdict, and it claimed no canonical PASS. The +original failed scene remains retained and must not be replayed in place. + +The Auto Testing PR [#113](https://github.com/rustfs/auto-testing/pull/113) +contains the migrated G02 executor and remains Draft. RustFS PR +[#8455](https://github.com/rustfs/rustfs/pull/8455) completed its distributed +4-node/4-disk E2E in run [40631115072](https://github.com/rustfs/rustfs/actions/runs/40631115072) +on candidate `4454b029dd462e0f58f1b3acc0f4cd62137c24d6`: 92 tests passed and +873 were skipped. The PR includes Heal lock-wait observation and extensive +Heal regressions, but the E2E result does not bind those changes to the failed +PUT request or establish that the G02 lock contention is fixed. It is +supporting candidate evidence, not an immutable G02 package or case result. +Retest only after reviewing a concrete source correction, building and +auditing a package pinned to its exact candidate SHA with matching +tools/helper, and confirming the current Auto Testing fleet is safe for a +fresh run. A complete H1/H2 matrix, physical and peer S3 verification, and +clean run-owned finalization are required before G02 can add a canonical PASS. +Until then the 115/158 total is unchanged. + +### Hosted runner inspection (2026-10-26) + +Read-only [40256930895/1](https://github.com/rustfs/rustfs/actions/runs/40256930895) completed on a GitHub-hosted runner with driver `31c9d85645c1a8051562983f18631098b9bc34e7`. At `2026-10-26T02:00:24Z`, GitHub reported the only matching `smoke-testing` runner, `rustfs-smoke-testing` (ID 139653), as **offline**, with `busy=false`. The report retained two queued fleet jobs, including the Functional Chain summary and the health collector, plus pending workflow runs without jobs. This establishes the runner registration state at observation time; it does not diagnose the host's service, disk, network or power state. + +The new `validation_runner_inspect_only=true` dispatch path uses a separate workflow concurrency group, reads only GitHub API metadata and uploads an incomplete report if API access fails. It rejects simultaneous candidate, build, fleet, tools or recovery inputs; all fleet jobs were skipped in the real inspection. No queued workflow was cancelled or replaced, no test-node connection was made, and `mutated`, `cleanStateCertified`, `sharedWorkflowIdleCertified` and `canonicalCasePassClaimed` remain false. Live resumption requires a working maintenance entry to restore this runner, followed by a new resource inspection and the existing retained-scene and shared-workflow admission checks. + +### Interrupted G06 evidence and recovery boundary (2026-10-26) + +The re-registered `rustfs-smoke-testing` runner (ID 139936) was online in hosted inspections [40277791903](https://github.com/rustfs/rustfs/actions/runs/40277791903) and [40282750869](https://github.com/rustfs/rustfs/actions/runs/40282750869). Resource inspection [40278133051/1](https://github.com/rustfs/rustfs/actions/runs/40278133051) observed the current auto-testing fleet's 16 independent endpoints. Neither inspection certifies global workflow idleness or restores an interrupted scene. + +G06 [40281440070/1](https://github.com/rustfs/rustfs/actions/runs/40281440070) was cancelled during its selected-case step. The only annotation is `The operation was canceled.`; its cause is unconfirmed. The workflow uploaded no original artifact. Read-only collection [40284040202/1](https://github.com/rustfs/rustfs/actions/runs/40284040202) subsequently retained 1,777 original files (38,765,135 bytes), independently checked against every collection-receipt size and SHA-256. Collection provenance remains distinct from the original run; its unfinished result stays `RUNNING/PENDING`, unaccepted. + +H1 `copy-source` finalized as `NOT_OBSERVED` with cleanup PASS. H1 `copy-destination` retained all 258 seed request/ACK pairs and its one COPY pair, complete last-ACK physical authority and 1,028 streamed part proofs. It restarted before final peer S3 inspection, then was interrupted. Its recorded public-call overlap is false. No H2 or later source branch was completed, so this evidence adds no canonical PASS. + +`interrupted_race.py` admits only this bounded H1 COPY boundary: one unchanged unfinished case, one finalized predecessor, complete deterministic SDK journals, recomputed ACK-derived final state and physical/streaming proofs, and no later operation. Recovery checks all five original owner markers before stopping the exact owned unit, then requires the current stopped business scene to match the archived last-ACK scene. Existing same-filesystem private isolation preserves the original bytes, backup files, inodes and credentials without reading credential contents. Drift refuses isolation; no old-generation restoration or SDK replay is permitted. A new recovery receipt never rewrites the original verdict or supplies missing canonical assertions. + +The driver may explicitly select `validation_cleanup_retained_evidence=true` together with an exact failed cleanup run. It verifies that run/attempt and same-runner storage ownership, copies through no-follow file descriptors, checks every original and copied digest and records a new collection identity. This is not an original-run artifact fallback. The ordinary artifact-based recovery path remains the default. + +Isolation [40286516720/1](https://github.com/rustfs/rustfs/actions/runs/40286516720) completed with helper `8efc57d7ed89531a8f63cc4f98d06dafa2361499` and driver `3e337bad97a0263fa663f7909b33f247d6f76f17`. Independent audit verified official archive SHA-256 `11f691947bcbf6d707f0ed04ff7bfd625eeb4994221ad0f690bcc8c63cf43645`, all 1,777 original file bytes, the unchanged unfinished-case digest, stopped owned service and identical pre/move/post baselines for all five trees. All archive parents are root/root 0700 on their original filesystems. The disposition is ISOLATED; no namespace was deleted, no backup restored and no canonical verdict rewritten. Follow-up inspection [40287577171/1](https://github.com/rustfs/rustfs/actions/runs/40287577171) observed 16 eligible endpoints, inactive standard services, no port 9000 listeners and no active validation service. The original failed unit remains recorded; inspection does not certify global idleness. + +The original COPY SDK calls lasted approximately 23 ms and 17 ms. Their nearest running samples began before those calls or finished after them, so the frozen strict interval oracle correctly refused overlap. G06 schedules observation from botocore's actual `before-send.s3` notification inside the SDK call, after client preparation and durable request intent. The callback only queues a timestamp; it never waits, delays a response, repeats a mutation or alters the SDK call interval. A qualifying original-token running sample must still fit strictly inside one actual acknowledged SDK call. Real local HTTP/SDK tests cover nested running acceptance, finished-task refusal and response completion without any observer. The previous polling scheduler fails the positive regression. + +Complete G06 replay [40427815826/1](https://github.com/rustfs/rustfs/actions/runs/40427815826), pinned to helper `7094c16`, passed all eight H1/H2 public branches. Independent replay with the executing helper recomputed all six seeded branches' deterministic journals, original seed properties/tags, final manifests, physical transitions, complete streaming receipts, full peer S3 inventories, terminal outcomes and public overlap. All 20 cleanup receipts match the reserved hosts. G06 is now canonical PASS; the earlier failed run [40299213418/1](https://github.com/rustfs/rustfs/actions/runs/40299213418) remains retained unchanged. + +The next executor reuses the existing same-token status transport for G06, preserving its strict overlap oracle, and records read-only endpoint resolution before allocation and restarts. G01/G02 separately restore the frozen source's per-role overlap rule while requiring every PUT ACK and complete concurrent reader evidence. New exact-helper tools and complete H1/H2 live evidence remain required for acceptance. + +The source's `heal_h1_concurrency_batch.py::observe_race` at clean commit `524eb5b32c3236497fdd8a9271df0494fa3d9974` requires every planned request to have an ACK and returns `bool(matches)` for its nested running samples. The frozen G06 tag branch has exactly 16 distinct serial updates. The migrated controller incorrectly required all 16 calls to overlap. G06 tag aggregation now restores the source's requirement of at least one qualifying call, while retaining all 16 original requests, all 16 ACKs, exact intervals, final tags and full physical/S3 checks. The evidence retains every per-call overlap flag and explicitly records the required aggregation, so no match still yields NOT_OBSERVED. Other race aggregation is unchanged. Real local SDK tests verify one matching tag call plus fifteen nonmatching calls, all 16 durable ACKs and complete refusal when every sample is finished. The preceding migrated helper fails that positive test; no historical verdict is rewritten. + +### B08 complete live acceptance + +[B08 run 39915022106/1](https://github.com/rustfs/rustfs/actions/runs/39915022106) is independently accepted on candidate `1a3f038c90704f9ad9693eddcf7d861c9530f13c`, package `39898678364/1`, tools `39914732144/1` and helper `a3d3c62a9f6d29667a1a2b57d13f5cb9e166de3f`. The official 3,463-member archive has SHA-256 `59ddf692cdca495abcd37a1affacf25e4cd3da24c931509bb9b4facc386c2f11`; the independent audit has SHA-256 `63f143d278c5e6a43f0068a31c692cae1064fd2612b54060198c448f52b7e640`. H1 read-present/write-absent and H2 read-present/write-absent plus lock-quorum-absent passed; H1 lock-quorum-absent is explicitly NOT_APPLICABLE under the candidate's in-process local manager. The audit verified all peer inventories, unique PUT absence, physical target restoration, streaming shard proofs and ten cleanup receipts. The prior B08 failure remains retained in the append-only ledger. + +### B02 earlier live failure remains retained + +[B02 run 39916358715/1](https://github.com/rustfs/rustfs/actions/runs/39916358715) used the same immutable candidate package `39898678364/1` and helper `a3d3c62a9f6d29667a1a2b57d13f5cb9e166de3f`. The complete official artifact `12485117739` has SHA-256 `435fe989937adbad376c93506dfe7db7eec914f3fafb68ea98068d8c79ea5d7a`; its case result SHA-256 is `a92ab5c1e312f0e5f25b83f7f8802b1de3e43c5fd93bff05dd3651c9d1e1f978`. Read-only passed on H1 and H2, cleanup passed, and both permission-denied branches failed with `faulted version lacks a repaired responsibility`. The independent artifact audit `B02-live-39916358715-independent-failure-artifact-audit.json` (SHA-256 `8c9b2e03e437542f1d3bd6894cb8dab225e46e0e9f65aecf4b16f4a3db5d27eb`) confirms the follow-up queried the expected `objects/0017.bin` and returned `progress.objectsHealed=1`, while terminal responsibility reports `verified_healthy`, `healed=0` and `unchanged=1`. Physical snapshots show the faulted shard absent after the denied attempt and present with its baseline SHA after the writable follow-up sequence. The runner stops and restarts RustFS after releasing the gate and before that follow-up; there is no physical snapshot between service startup and the new Admin token, so the artifact cannot attribute the restoration to that token or establish a terminal receipt classification bug. The strict source assertion remained unsatisfied in that run; this original failure record remains unchanged. + +### B02 complete live acceptance + +Run [39927051614/1](https://github.com/rustfs/auto-testing/actions/runs/39927051614) independently passed both source assertions across H1/H2 read-only and permission-denied branches. It used candidate commit `1a3f038c90704f9ad9693eddcf7d861c9530f13c`, package `39898678364/1` (SHA-256 `33f97e40add31ebe9bc4e923f68bc11d2e8855b4eedf5aad7bc8fce1f07da9d6`), tools `39925186107/1` and helper commit `c98028b697f8f9349a2faca277c14810d9b3e466`. Official artifact `12490700181` has SHA-256 `ba2f2569f3799a2aed014c9b89d37a80075774e76dfbfd1b3a06c04c38cb533e`; the independent complete audit has SHA-256 `6e7a2c9e21ed3bd031d7189da0c2d8cfd607c56626745cb4a797e0099070771b`. + +For each permission-denied branch, the initial request was rejected before task admission and had no client token. After the gate cleared, candidate process birth identity remained unchanged and the pre-token physical snapshot matched the denied-attempt snapshot. A fresh Admin token then repaired the exact planned shard to its baseline size and SHA-256. The read-only branches also restored their exact planned shards. S3 inventories matched, all five H1/H2 cleanup receipts passed, and both source assertions were present as PASS. This adds one canonical acceptance while retaining the prior failed run. + +A05 scoped live [39875125045/1](https://github.com/rustfs/rustfs/actions/runs/39875125045) +was interrupted by the self-hosted runner's exhausted filesystem. The official +check annotation reports `No space left on device` while writing the runner's +worker log. The selected-case step remained in progress in the completed job; +cleanup and artifact upload stayed pending, and the official artifact listing +contains no files. Neither product failure nor clean finalization can be +concluded from that interruption. Its fixed candidate `c95bb396`, package +`39867606969/1` and matching tools `39874908888/1` retain their independent +byte/provenance audits. The J01 chain stopped before dispatch when A05 evidence +collection refused the empty artifact listing. Its released-client fix remains pending a fresh live acceptance; the B01 +explicit disk-error correction is covered by the accepted B01 run above. The administrator has since deleted the interrupted runner temporary evidence; +the user explicitly accepts retaining this attempt as unaccepted. A05 later +passed on a separate complete run 39892106641; that pass does not reconstruct +or change the status of run 39875125045. No product verdict or cleanup PASS is +inferred for the deleted scene. + +### J01 startup window diagnosis (2026-10-23) + +J01 run [39929436131/1](https://github.com/rustfs/rustfs/actions/runs/39929436131) +used the current auto-testing fleet, candidate `1a3f038c90704f9ad9693eddcf7d861c9530f13c`, +and helper `c98028b697f8f9349a2faca277c14810d9b3e466`. Its official artifact +`12491201856` has SHA-256 +`bd703e00ed88297794a6adb72760ede313f7e9177a30978cac45eb60eddbae5d`. +Enabled-idle, disabled, missing-peer, recovery and all four cleanups passed; +the complete case remains FAIL because H1 and H2 did not expose the required +raw/rc `uninitialized` window. The artifact and original verdict remain intact. + +The H1 startup journal narrows the cause. RustFS began listening at +`09:07:12.377958Z`. The background-heal endpoint returned HTTP 503 with +`server context is not ready` at `.400036Z` and `.411255Z`. Heal runtime +initialization followed at `.438696Z`, and `server_ready` at `.438747Z`. The +first successful signed status response in the collector evidence arrived at +`09:07:12.514298Z` and already reported `idle`. Thus the startup listener +rejects status requests until the heal runtime is initialized; the source +required HTTP 200 `uninitialized` state is not externally observable on this +candidate during that transition. Connection refusal or pre-ready 503 remains +non-evidence. Keep this branch `NOT_OBSERVED` and the canonical J01 result +unaccepted unless a product/source change supplies the required truthful +observable window; do not weaken the source oracle or promote this run. + +### B03 current auto-testing result and mount-admission diagnosis (2026-10-23) + +B03 run [39931810677/1](https://github.com/rustfs/rustfs/actions/runs/39931810677) +used candidate `1a3f038c90704f9ad9693eddcf7d861c9530f13c`, package +`39898678364/1` (SHA-256 +`33f97e40add31ebe9bc4e923f68bc11d2e8855b4eedf5aad7bc8fce1f07da9d6`), +and auto-testing commit `dd53393e6ee713f5831254be92a7a90ea40bdf62`. +Official artifact `12491823352` has SHA-256 +`df3770cf8f2b9a8ba22e6cc2e5064f96c030121e5bfe105f5de2a894adacad01`. +All four H1/H2 × DB/DV instances retained FAIL and cleanup PASS. Their +registered 90-second format observations never found the replacement +`format.json`; candidate journals repeatedly report +`Replacement target is not ready: target mount admission failed`. + +I re-downloaded the official artifact and checked its runner storage preflight, +H1 deployment preflight, replacement receipt, post-start format observations, +and candidate journal. The runner had about 258 GB free at preflight, and the +four parent mounts were four distinct NVMe devices; this rules out the old +disk-space concern for this run. The deployment record lists the actual +endpoint roots under each `/data/rustfsN` mount, but does not establish a mount +lease for those run-private endpoint paths. The service journal confirms that +`renew_disk` attached the empty endpoint and queued the replacement request, +which then failed admission. Thus the missing capability is endpoint mount +binding and safe mount switching, rather than capacity or missing replacement +bytes. + +The runner gives RustFS endpoints under +`/data/rustfsN/heal-validation//data`. The parent `/data/rustfsN` +mounts are independent devices, but the candidate's replacement admission +requires a valid mount lease for the exact target endpoint and each local +sibling endpoint, plus target physical-device IDs disjoint from `/` and those +siblings. The run-private endpoint directories do not meet that exact-mount +contract. The service logs show the actual sequence: `renew_disk` attached the +empty endpoint and submitted an automatic replacement request; the task then +started and repeatedly failed with `Replacement target is not ready: target +mount admission failed`. `RUSTFS_HEAL_AUTO_HEAL_ENABLE=false` only disables the +periodic scanner; it does not suppress this replacement request generated by +`renew_disk`. The executor's `wait_format()` therefore expired after 90 seconds +and the explicit manual `run_task()` call was never reached. The source +procedure and strict UUID/set-slot and physical-byte oracles remain unchanged; +do not lengthen the wait, copy from quarantine, or infer a product verdict from +this environment mismatch. + +Before another canonical attempt, add a read-only preflight that checks every +local endpoint's exact mount lease, mount ID, filesystem identity, and physical +device separation from `/` and its local siblings. The replacement adapter +must then demonstrate a candidate-supported way to switch the target endpoint +to a blank mount, let the candidate obtain a new lease and initialize it before +the manual token, while keeping the old archive unreadable to the service. A +bind mount may be used only if the adapter proves this full lease switch and +all candidate device-identity checks; a bind mount by itself is not evidence. +The Functional Chain now runs a read-only `replacement-admission-preflight` +before `stop_existing`, candidate package download/install, the normal +parent-mount preflight, reservation, service start, or fixture seed. It +inspects all H1/H2 endpoints and records each exact mountpoint, mount ID, +filesystem identity, nearest existing ancestor, physical-device IDs and +root/sibling overlap, along with whether a reviewed mount-switch adapter is +registered. If admission is unavailable, the result is `BLOCKED` with +candidate package and client provenance, `binary_sha256: null`, and an +explicit `NOT_REQUIRED_NO_MUTATION` cleanup disposition; the existing RustFS +package and service remain untouched. If admission passes, `Fleet.deploy()` +repeats the read-only check immediately before reservation. The current runner +reports ordinary run directories and no adapter, so B03 stops before bootstrap +and any 90-second format wait. Once an adapter is installed and independently +reviewed, the gate must pass before recording mount identities before and +after replacement and rerunning the full H1/H2 DB/DV matrix. The source oracle +and prior FAIL evidence remain unchanged. + +The new pre-deployment gate was exercised on the current auto-testing +environment in B03 run +[39942624978/1](https://github.com/rustfs/rustfs/actions/runs/39942624978), +using the same candidate package and helper commit +`49443e7559578cba8be285ed723bb68dfbaf8656`. H1 and H2 both returned +`BLOCKED` before deployment: all endpoint paths were absent ordinary +directories beneath the four distinct NVMe parent mounts, with no exact +endpoint mount IDs or filesystem identities and no reviewed mount-switch +adapter. Root and sibling physical-device separation passed, but that does +not satisfy the endpoint lease contract. The case result records +`execution_stage=pre-deployment`, `binary_sha256=null`, +`mutationStarted=false`, deployment and fixture seeding skipped, and +`cleanup=PASS` with `NOT_REQUIRED_NO_MUTATION`. The case result SHA-256 is +`68ed1713fa3eab0e0d2d972369fca4af00ae58bdb4e6a1e64b072cfae6194621`; the +preflight summary SHA-256 is +`fb47d91ee77d21104be7a400bbfda1aa0d73b30e090476c853a9a53b5036ea12`. +This is verified BLOCKED evidence only: it adds no canonical PASS and does +not replace the retained B03 live failures. The next valid B03 attempt still +depends on installing and reviewing the mount-switch adapter. + +B04 `heal.disk-replacement-unformatted` now uses the same replacement +admission contract. Its runner checks H1/H2 exact endpoint mounts before +stopping any service or installing the candidate, and its executor repeats +the read-only admission immediately before reservation. If the current +ordinary-directory layout remains in place, B04 is recorded as `BLOCKED` +with `binary_sha256=null`, `cleanup=PASS`, and +`NOT_REQUIRED_NO_MUTATION`; no blank or foreign-format fixture is created. +If the first gate passes but the reservation-time recheck observes environment +drift, the executor records the same `BLOCKED` verdict while retaining the +installed candidate provenance, skips reservation and fixture seeding, and +finishes cleanup. That post-bootstrap block is distinct from the +pre-deployment `NOT_REQUIRED_NO_MUTATION` result and cannot claim canonical PASS. + +B05 acceptance uses candidate `7ed6c9320db491c84b54e8232152d7b1282c1b3f`, helper `76dbf7029e82c1e3878c9160090804d4afc38078`, package `39827676705/1` and tools `39841369555/1`. The complete official ZIP digest is `4eb3405af1f47583cd7c2d8e5b9ce9fa6910b76665df529d7dfbe4f1399a20d8`. Independent recomputation covers seed ACKs, full versions/markers, the actual 2+2 and 12+4 role/loss matrices, every endpoint's full online membership before each required boundary, sequential faults, one unique START/token per instance, two identical terminal outcome queries, strict integer scope, exact repaired/healthy responsibility, all physical shard and bitrot proofs, complete S3/HEAD/Range evidence before finalization, ten private-node cleanups and four package purges. No backup restoration was used. Original failure `38200772548` remains unchanged; its vm001 offline root cause is not established by the later pass. + +E04 live [39842352595/1](https://github.com/rustfs/rustfs/actions/runs/39842352595) retains FAIL/cleanup PASS. All 1,402 official files, three H1 branches and five node cleanups were independently verified. H2's healthy Scanner task completed on vm002 while the exact node3 token query was forwarded to vm000 and returned notFound. RustFS commit `553be4b4c62ae0c590e82f91817bc018bdfdc660` lets authenticated original local automatic-task tokens return the source/path-bound in-memory snapshot; Admin/Internal/unknown/alias identities retain coordinator routing. Fixed candidate `9d90884c304dcfbf97057c263c6f2ee110230ce7` adds only that reviewed patch to `7ed6c932`, with unchanged Cargo.lock. Package [39851206784/1](https://github.com/rustfs/rustfs/actions/runs/39851206784) and matching tools [39860218641/1](https://github.com/rustfs/rustfs/actions/runs/39860218641) passed independent complete provenance and byte audits. + +The subsequent E04 live [39860501067/1](https://github.com/rustfs/rustfs/actions/runs/39860501067) retains FAIL/cleanup PASS. All 1,795 official files, H1's three branches, H2's cycle-boundary branch, five node cleanup receipts and four package purges were independently verified. H2's missing-shard and bitrot branches failed attribution: real repair tasks ran on remote disk workers while complete Scanner cycles were logged on vm003. The old same-node cycle requirement omitted both exact tasks and queried later healthy tasks. Independent replay binds each missed task to its original remote Scanner RPC request and cycle, and verifies restored physical bytes before rescue; neither missing structured receipt is retroactively accepted. The corrected executor retains the task owner for exact token queries and requires the remote RPC identity, Deep disk scan, matching positive cycle number and one unambiguous complete leader cycle. Invalid or duplicate identities, wrong/failed/ambiguous cycles and healthy substitution remain rejected. The later complete six-branch acceptance of run 39863505276, detailed below, raises canonical acceptance to 110/158 while retaining this original failure. + +I05 progress (2026-10-22): four-node startup scene `39764496504/1` was +independently verified and isolated by +[39766457981/1](https://github.com/rustfs/rustfs/actions/runs/39766457981), +preserving all 102 original artifact files and twenty same-device private +archive parents. Read-only inspection identified broken kernel-selecting +bpftool wrappers on vm002/vm003; the exact already-installed executable is +now digest-bound and checked before reservations. Fresh diagnostic +[39767336741/1](https://github.com/rustfs/rustfs/actions/runs/39767336741) +completed H1 and H2 on helper `8cbdd51`, with all five node cleanups passing. +Independent audit of 211 artifact files verified exact 1001 ms write-lock +timeouts, closed caller sockets, unchanged physical content, full diagnostic +seed ACK/peer inventories and final FD counts at or below baseline. The +original canonical result remains `BLOCKED/cleanup PASS`; source DL/DV, +exact disconnect/native fences and stale-generation coverage remain outstanding. +Independent queued-successor diagnostic acceptance is recorded below. Exact-helper native/tools +[39783997956/1](https://github.com/rustfs/rustfs/actions/runs/39783997956) +passed independent audit. The initial successor live +[39788321908/1](https://github.com/rustfs/rustfs/actions/runs/39788321908) +failed at internal bucket metadata-read classification despite one +unretried 200 PUT ACK. Its stopped H1 scene was preserved by +[39789732916/1](https://github.com/rustfs/rustfs/actions/runs/39789732916). +The parser fix retains exact request/owner/budget/lifecycle checks and has +a regression over the original fifteen events. Fresh helper-bound tools +[39791663588/1](https://github.com/rustfs/rustfs/actions/runs/39791663588) +passed independent archive/source/native audits. Live +[39798325977/1](https://github.com/rustfs/rustfs/actions/runs/39798325977) +then reached the new acknowledged generation but invoked the stopped-only +shard verifier on a running service. Its original FAIL/cleanup FAIL and +71 files were preserved unchanged by isolation +[39799657074/1](https://github.com/rustfs/rustfs/actions/runs/39799657074). +The controller now finishes all online peer/FD observations before stopping +the service and checking streaming shard integrity, with matching physical +snapshots before and after verification. Tools +[39803149039/1](https://github.com/rustfs/rustfs/actions/runs/39803149039), +bound to helper `6fbc3c4`, passed independent official archive/source/native +and inspector audits. Live +[39810035277/1](https://github.com/rustfs/rustfs/actions/runs/39810035277) +completed H1 including stopped integrity and cleanup. H2 started and queued +its PUT, then the parser treated a same-target slow-acquire WARN as a lock +lifecycle row. The original result stays FAIL/cleanup FAIL. Isolation +[39811143272/1](https://github.com/rustfs/rustfs/actions/runs/39811143272) +preserves all 180 original files and twenty same-device private archive +parents; every four-node pre/move/post tree, device and inode baseline +matches. The parser now recognizes only the exact auxiliary warning bound +to an already acquired target write; it cannot replace lifecycle evidence. +The original sixteen-event H2 trace and incremental prefixes are covered by +positive and negative regression tests. Exact helper `3012a65` tools +[39812960916/1](https://github.com/rustfs/rustfs/actions/runs/39812960916) +passed independent archive/source/native and inspector audits. Fresh successor +[39817606025/1](https://github.com/rustfs/rustfs/actions/runs/39817606025) +completed H1/H2 and all five node cleanups. Independent audit byte-matched all +246 official files, nine seed request/ACK pairs and eight final versions/markers +per topology, one successor ACK, historical generations, lock events, every +peer inventory, ACK-bound stopped physical invariance and four/sixteen streaming +shard receipts. FD counts returned to or below each peer's baseline. The original +result remains `BLOCKED/cleanup PASS`; exact disconnect-edge ownership, +distributed release, global no-leak and full source DL/DV coverage are unproven. +That I05 diagnostic added no acceptance. The separate complete B05 audit brought the historical total to 109/158; the subsequent E04 audit brings the current total to 110/158. + +Fresh D03 [39801936817/1](https://github.com/rustfs/rustfs/actions/runs/39801936817) +retains FAIL/cleanup PASS. Independent H1 re-evaluation verified all four +branch physical and strict version-responsibility oracles, sixteen seed +request/ACK pairs, fourteen versions/markers, 48 HEAD observations and 144 +range payloads. H2 failed before seeding when a `pool.bin` write returned +`SystemMetadataRepairRequired` and the pool transaction remained blocked +at `transaction_unknown/prepare_cas`. All 392 original files and five node +cleanup receipts were independently checked. The exact candidate's bounded +owner-repair retries only cover identity metadata, so ordinary init retries +do not discharge this `pool.bin` responsibility. The initial replica/network +cause remains unproven; no D03 historical failure is disproved or promoted. + +[J02 diagnostic 38874900846/1](https://github.com/rustfs/rustfs/actions/runs/38874900846) completed all 20 H1/H2 × DB/DV branch instances on that exact candidate. Sixteen branches report PASS; the four original all-metadata-corruption probes report permanent Failed rather than Unknown and retain NOT_OBSERVED. Independent archive and provenance audits passed. Independent mixed-branch audits verified all four repaired shards against original complete bytes, exact receipts and counters, surviving below-quorum failures, original terminal preservation and post-rescue S3 inventories. All 50 node namespace finalization receipts passed audit. Real legacy writes separately produce Unknown with zero repairs and unchanged physical files, but do not replace the original source procedure. Full J02 canonical acceptance remains blocked; these partial results add no canonical acceptance. + +A10 run `39197088183/1` uses immutable helper `b16e6072679d6a60a044b9d4520b30b4a79b5557`, candidate `072b57b37559fbe626aeda8abf61617fdcbdd926`, package `38043283428/1` and inspector `38715612825/1`. All nine source variants across H1/H2 and DB/DV passed their 68 required execution instances. Independent audit checked original SDK request/ACK hashes and version/marker identities, physical set selection, exact task options/receipts/counter partitions, 27,096 complete shard-streaming proofs, physical closure before target GET or control rescue, complete peer inventories and all 170 node cleanup receipts. Full inventories are checked against ACK manifests; their endpoint iteration is established by the immutable executed helper source, since the inventory documents omit endpoint labels. The remove=true branch replays one complete below-quorum original object after exact DELETE and requires whole-orphan removal with unchanged live controls. Rescue restores only post-DELETE authority. The original failed A10 result remains byte-identical. + +The sections below retain earlier implementation and run history. Historical failed or partial results are not promoted by this summary. + +## E04 complete acceptance and A05 cancellation regression + +Fresh E04 live [39863505276/1](https://github.com/rustfs/rustfs/actions/runs/39863505276) passes complete independent acceptance on helper `cd36287f1abe42662333fab31cf45ac23de91c8e`, candidate `9d90884c304dcfbf97057c263c6f2ee110230ce7`, package `39851206784/1` and tools `39863244404/1`. All 2,307 official archive files were byte-checked against ZIP SHA-256 `ecbd537cb6422cc3af631d4a7d1145040ffd30c20da0008778e442752e0f193e`. All six source branches passed independently recomputed seed ACKs, healthy preflight, exact Scanner source/token/RPC/cycle binding, strict repaired responsibility, complete physical and bitrot proofs before S3 reads, 768 HEADs, 672 deterministic Range payloads, unchanged inventories and finalization without backup restoration. All five node cleanups and four package purges passed. Independent complete audit SHA-256 is `c96ab5fc91b13fecb475a0cee86c85a3f5c983ce125b912b4485f7cd080d3243`. This raises canonical acceptance to 110/158 and preserves every earlier failure. + +A05's cancellation after successful bucket storage I/O is deterministically reproduced at the receipt boundary: the unchanged code rejects the late positive receipt but falls back to Unknown. RustFS commit `78c803e1efd67cc04edc8fc4d08a8ea75af77ddf`, pushed to the existing PR, retains the positive-receipt rejection and routes observed cancellation through the existing typed Cancelled outcome, with partial coverage and zero positive or Unknown responsibility. The exact regression failed before the fix and passes after it; all 153 task tests and Rust formatting pass. This establishes the code defect and local regression coverage, not the exact historical live interleaving or a new A05 canonical result. Fresh immutable candidate and complete A05 live acceptance remain required. + +## Supplemental J02 evidence audit + +The 2026-10-23 supplemental independent audit closes twelve positive branch instances from original J02 run 38874900846: grace Deferred/new-token cleanup, read-only terminal failure/new-token repair and dry-run across H1/H2 and DB/DV. Every original archive member was byte-checked again, all fixture requests/ACK digests and version/marker manifests were rebuilt, and the exact executed helper was pinned. + +Grace uses both original ACK versions, natural 120-second expiry under unchanged candidate processes, the exact typed future retry time, two unchanged original-token terminal queries and every H2 peer, a distinct compensating token, physical post-DELETE authority before GET/rescue and complete peer inventories with old-version refusal. Read-only verifies actual private-namespace EROFS, the protected-control incarnation, unchanged failed and released scenes, a distinct repair token, original full shard/bitrot evidence before GET/rescue and all peer inventories. Dry-run retains every original typed disposition, the actual R-1 fault geometry and unchanged fault bytes. All twelve node cleanup groups bind their original reservations. + +Together with the earlier four mixed-branch audits, sixteen branch instances now have independent acceptance; the four original opaque-metadata Unknown probes still prevent whole-case acceptance. The supplemental report SHA-256 is `4424b3e681931527d244a4ca0211dde534a8145cd95ea0b460f388262ccb8c10`. This J02 audit preserves historical results and adds no canonical acceptance; the current total is 110/158 after the separate E04 audit. + +## Original reuse assessment + +| Source module | Cases | Existing auto-testing code to inspect | Current gap | +| --- | ---: | --- | --- | +| Pool lifecycle | 13 | `rustfs_pool_expand.sh` | Current steps combine operations; each validation ID needs its own assertions, object manifest, and result. | +| ILM/scanner | 15 | `rustfs-tier-test.sh`, `rustfs-s3-compat-test.sh` | Existing tier/lifecycle CRUD tests do not prove the accelerated scanner deadlines and all 15 expiration outcomes. | +| Bucket replication | 18 | `rustfs-replication-test.sh` REP-101..109 | Existing cases cover some operations; source filters, exact topology, sync behavior, failure recovery, and result evidence need individual checks. | +| Site replication | 20 | `rustfs-replication-test.sh` SITE-101..106 | Existing cases cover part of setup, IAM, objects, policy, status, and removal; canonical case boundaries and missing assertions remain. | +| Heal | 84 | `rustfs_heal_test.sh`, `rustfs-fault-tolerance-test.sh` | Existing fault flows are useful helpers, but each A-J procedure requires its own fixture, fault, public assertion, and cleanup evidence. | +| Planned | 8 | KMS, Tier/Event/Audit, Security, S3, Storage suites | Existing tests may implement the behavior; compare each source assertion before promoting it. | + +`s3.basic-object-round-trip` now has a dedicated executor on `single-1disk`. +Its Bash function combines bucket creation/removal, object PUT/HEAD/GET/LIST/ +DELETE, payload SHA-256, metadata, ETag, and `NoSuchKey` checks on one fixture. +It has local mock coverage and live PASS evidence in RustFS Actions run +`38042083422`. + +`scanner.usage-convergence` has a dedicated executor on `single-4disk`. It +uses the storage suite's deployment and SigV4 Admin client, accelerates the +scanner cycle, records the 20-object fixture, and checks a successful leader +cycle after upload plus a newer complete usage snapshot with exact object and +logical-byte counts. The old STOR-134 threshold remains a separate regression +test. Its live candidate run passed in RustFS Actions run `38042200106`. + +`scanner.status-readiness` has a dedicated executor on two of the existing +auto-testing nodes, each with four disks. It checks both nodes' accelerated +scanner runtime settings and requires exactly one leader to finish a new +successful cycle within 120 seconds. Its live candidate run passed in RustFS +Actions run `38044125099`. + +The 14 remaining ILM IDs now dispatch to the source ILM black-box behavior +checks under `scripts/validation_ilm`. The Storage suite deploys the same +two-node/four-disk source topology on current auto-testing nodes with the +source's accelerated scanner/ILM timing, and supplies a cluster restart hook +for `ilm.restart-persistence`. `ilm.transition-and-restore` uses the third +auto-testing node as an independent warm-tier target. Each selected case must +pass its own harness result, evidence, and bucket-cleanup verification before +the canonical result can become PASS. `ilm.configuration-roundtrip`, +`ilm.invalid-configuration`, `ilm.current-expiration`, +`ilm.tag-filter-expiration`, `ilm.disabled-rule`, and `ilm.rule-removal` have +live PASS evidence in RustFS Actions runs `38044956285`, `38045047518`, +`38045154019`, `38045412209`, `38045496042`, and `38045714246` respectively. +`ilm.delete-all-versions`, `ilm.delete-marker-expiration`, +`ilm.multipart-expiration`, and `ilm.newer-noncurrent-retention` also passed +live in runs `38049740183`, `38049817172`, `38049890130`, and `38049965929`. +`ilm.noncurrent-expiration`, `ilm.restart-persistence`, +`ilm.transition-and-restore`, and `ilm.versioned-current-expiration` passed in +runs `38050040606`, `38050434607`, `38050720293`, and `38051148291`. +All 16 ILM/scanner cases now have live PASS evidence. + +All 13 Pool lifecycle IDs now dispatch to the source black-box pool checks. +The driver reuses the existing auto-testing Pool suite's two-node seed and +expansion operations, supplies marker-guarded reset/topology/restart/physical +census hooks, and requires source case evidence plus physical cleanup before +canonical PASS. `pool.status` passed live in RustFS Actions run `38047001285`; +`pool.decommission` and `pool.decommission-cancel-clear-resume` also passed in runs `38051227313` and `38052304396`. `pool.decommission-live-io`, `pool.decommission-restart-recovery`, `pool.decommission-versioned-integrity`, `pool.expand-and-rebalance`, and `pool.expansion-restart-persistence` passed in runs `38053444515`, `38053814812`, `38054128863`, `38054208114`, and `38054287896`. `pool.decommission-multipart-integrity` failed in run `38053715867`: no fresh owned-bucket MPU drain failure evidence was found; cleanup passed. `pool.operation-guards` and `pool.rebalance-live-io` passed in runs `38054540360` and `38054696080`. `pool.rebalance-stop-resume` and `pool.retired-config-removal` subsequently passed in independently verified runs `38110959836` and `38111198632`. The multipart failure remains visible. + +The 18 bucket-replication and 20 site-replication IDs now dispatch to the +source black-box checks. The driver uses the existing auto-testing four-node +main cluster and second RustFS instance on port 19000, including the source's +site-group fixture and site outage hooks. Logical bucket isolation and +physical teardown are required for PASS. +`bucket-replication.source-versioning-required` passed live in run +`38049256196`; the other 37 replication cases still require live evidence. + +`kms.sse-kms-round-trip` has a dedicated executor using the existing local +KMS single-disk deployment. It checks the same object's encryption key, +metadata, and payload, then requires an invalid-key PUT to fail without +exposing plaintext or key material. Its live candidate run passed in RustFS +Actions run `38042359938`. + +`tiering.transition-and-restore` has a dedicated executor using the Tier +suite's existing `single-single` source and warm-tier nodes. It verifies the +same object's transition state, readable bytes after transition, restore +state, and SHA-256 after restore. The case is recorded under the source +`lifecycle-b` environment profile. Its live candidate run passed in RustFS +Actions run `38042759074`. + +`events.delivery` and `audit.request-correlation` now have dedicated executors +on the Tier suite's `single-single` topology. Event delivery is counted for a +unique object after a quiet window; audit checks matching PUT/GET records, +distinct request IDs, successful status, and absence of known secrets. +`events.delivery` passed live in RustFS Actions run `38046470999`. +`audit.request-correlation` passed in run `38050138583` with the candidate's +`Records` envelope and exact S3 operation names. The audit fixture uses a +distinct random secret and retains the secret-leak, request-ID, +object-identity and successful-status assertions. + +`oidc.login-and-session` uses the Security suite's current single-node +deployment and a run-owned Keycloak container with a pinned image. It performs +password login, verifies issuer/audience/subject, exchanges the identity token +for STS credentials, and checks the readonly identity policy plus a narrower +session policy on existing objects. Missing/invalid session tokens, tampered +identity tokens, and wrong audiences must fail. Fixtures, containers, and the +RustFS deployment must all be cleaned before PASS. Runtime credentials are +excluded from evidence. This executor does not require a RustFS checkout. +Its complete live candidate run passed in RustFS Actions run `38051425083`. +Policy denials require `AccessDenied`; absent or mismatched temporary session +tokens require the candidate authentication contract's `InvalidRequest`. + +`iam.policy-enforcement` has a dedicated executor on the Security suite's +`single-single` deployment. It uses one principal with a policy that allows +PUT/GET and explicitly denies DELETE, then checks the denied DELETE did not +remove the object. Its live candidate run passed in RustFS Actions run +`38043548625`. + +"Existing validation environment" means the fleet used by `auto-testing` +and the RustFS `smoke-testing` workflows, with `RUSTFS_NODES` and +`RUSTFS_SSH_USER` supplied there. For Heal, reuse the nodes deployed by +`rustfs_heal_test.sh`; the source H1-H6 names describe required topology and +fault capabilities, not a separate SSH fleet. The ten `heal-extended` cases +have no source target-environment field, so their runtime binding must record +the actual auto-testing Heal nodes and topology before any result is scored. +The source's standalone ZIP, port 9100/9200, and dedicated systemd profiles +are reference implementations for assertions, not deployment targets here. + +The RustFS Functional Chain workflow has a manual validation lane on the same +`smoke-testing` runner and `rustfs-shared-functional-tests-v2` concurrency +group. Set `build_run_id` and `build_run_attempt` to one successful nightly +build with its retained candidate artifact, `auto_testing_sha` to an exact +commit, and `validation_case` to one migrated ID. The lane resolves the +immutable package URL and SHA-256 from that build attempt, then calls +`scripts/run_validation_case.sh`. The dispatcher rejects an ID without an +executor before touching the fleet and runs the result report with +`--gate-case`, so a missing, failed, or incomplete selected case fails the +workflow. Each run uploads its candidate identity, canonical result, and +case evidence. The 154 currently dispatchable IDs include all 74 non-Heal cases, 67 automatic Heal executors, and 13 disabled-by-default environment gates. The gates preserve the operator procedures and emit explicit BLOCKED results; they are not automatic live executors or PASS claims. The preceding census confirmed 101 unique complete canonical live PASS cases: 85 on source `072b57b37559fbe626aeda8abf61617fdcbdd926`, 12 on the older immutable source, two on source `16cf19817e0c81e24d0d787a916f734a34517ec5`, and two on source `e224eb48e5311cff022b04fd78c97a8572d5c270`. Its latest additions were C08 metadata/payload mismatch `38235978152` and E03 read-repair attribution `38237270640`. The independently verified Pool tail runs `38110959836` and `38111198632`, A03 run `38198122041`, C04 run `38203454300`, the completed multipart run `38204339623`, D07 run `38205235080`, D09 run `38205854024`, D10 run `38207381816`, and manual-normal attribution run `38207956101` are included. Native, read-only, recovery and duplicate runs are excluded; B07 run `38727279347` subsequently raises complete acceptance to 102, leaving 56 cases without complete canonical live PASS. + +All 84 Heal recipes, 302 operator branches and 534 applicable topology branch +templates are preserved under `scripts/validation_heal`, with the public +contract, operation book, signed probe, ACK-logged workload, guarded file-fault +tool and source-bound metadata inspector source. `matrix.py plan` binds current +auto-testing SSH nodes and required resources; its audit checks every branch +and nine evidence stages without manufacturing PASS. +`heal.admin-healthy-noop` now has an automatic H1/H2 executor using the existing +Heal suite's immutable deb download, installed binary digest, independent disk +mounts, run-owned services/data roots, exact normal/deep responsibility +receipts, repeated status, physical hashes and complete S3 inventories. It +failed live in run `38052673980`: the candidate returned an `unknown` receipt for a fresh zero-byte object with no bucket-incarnation identity, violating the source healthy-object contract. Cleanup passed; the oracle remains strict. +`heal.admin-invalid-request` passed all six branches on both H1 and H2 in run `38052868969`, with complete peer coverage and unchanged physical/S3 fixtures. +The external-shard executor also dispatches C01–C04, D02, E01 and E02 with all 24 source branches on H1/H2. It freezes exact version/dataDir/part, actual erasure role/set slot/disk UUID and full original shard hashes, stops all run-owned services before fault injection, witnesses the same fault immediately before the manual token, and checks complete physical repair plus streaming HighwayHash before any target GET. Full S3 inventories, cross-part Range, HEAD and pending MPU controls retain source semantics. Backups are used only for failed-case rescue or finalization; an unexpected new generation keeps the namespace DIRTY. Protected-mode C01 passed all ten H1/H2 branches in run `38057373420`; the other six executors still require live evidence. A03 also has both H2 branches: a non-coordinator shard fault, exact token queries through coordinator/peer/fixed current-fleet entry, a 30-second convergence budget for task start and full terminal outcome, physical proof and S3 inventories through all four entries. +A04 dispatches all six dry-run branches with separate DB/DV fixtures on H1/H2. It freezes version and marker identities, requires complete dry_run_observed receipts and zero repaired counters, proves byte-for-byte no mutation, then rescues the injected fault before full S3 verification. It does not require a dry-run incarnation or claim backup restoration as product repair. The first C01/C02 runs (`38055041726`, `38055254447`) stopped before injection because the remote script had an invalid future-import placement; their FAIL records and clean finalization are retained. The script compiler regression is fixed, and fresh runs use the corrected executor. +`heal.range-head-and-conditions` preserves all seven D07 branches, with real baseline/post-repair HEAD, cross-part Range, exact ETag conditions, and server-time-derived conditions including 304/412/416 negatives. Completed MPU target-side stale `part.N.meta` cleanup is allowed only on the repaired drive/dataDir and for parts still represented by the unchanged authoritative xl.meta; controls and payload shards remain exact. +`heal.historical-versions` now preserves all four D03 branches: historical and +current part faults on distinct physical roles/drives, a recursive bucket fault +with missing historical part and another drive's complete xl.meta, and multiple +versions faulted together. All acknowledged versions and markers must have +exact, complete responsibility receipts and unchanged S3/physical references. +`heal.repeat-idempotency` preserves all four J04 branches. It freezes the first +repair's physical/S3 result, repeats normal/deep with independent tokens, and +restarts the same run-owned units for the process branch. The versioned branch +removes a whole xl.meta with three data versions and a marker; first repair may +attribute all identities or a mixture of repaired and positive health, while +repeat permits zero repairs and mode-specific health only. Counters must agree +with the complete receipt set. D05 `heal.versioning-suspended-null` now uses real +S3 versioning suspension, consecutive null overwrites and acknowledged null +marker creation/deletion. It faults a live null data/parity part or an entire +xl.meta containing the current null marker and historical versions, verifies +every exact identity, then re-enables versioning and checks a new non-null +version plus the unchanged original version set. These three executors still +require live evidence. +B07 `heal.below-read-quorum` now retains exactly R-1 same-version authorities +and isolates all other part files, separately for never-versioned and historical +fixtures. The exact candidate tool build freezes the recognized metadata read +quorum rule, S3 HTTP/error mapping and source digests. Unrecognized or stale +contracts cannot dispatch the negative case. GET must return the frozen refusal +without payload or an unknown transport outcome; manual deep Heal must retain +the exact failed version and complete truthful counters. All good parts and +metadata must remain unchanged. Only after that failure proof may backups be +restored and a new token verify recovered integrity; recovery never proves +product repair during the negative window. This executor still requires live +evidence and a fresh tool build containing the quorum contract. +C05 `heal.missing-or-corrupt-xlmeta` also has all five automatic branches on +H1/H2: missing/truncated/header-corrupt xl.meta, a flip inside the unique decoded +inline value, and whole metadata loss with three historical data versions and a +marker. The tool now freezes each inline value's physical offset/length/SHA-256 +and verifies its streaming HighwayHash framing. Complete original inline shard +digests, version/dataDir/part/role references and all S3 versions must be restored. +A decoder error can be captured only when its file length/hash exactly matches +an injected fault receipt; it never becomes valid metadata or a repair proof. +Legal metadata re-encoding may move an inline value while its exact length/hash +stay fixed. The extended helper passed the 24/12 external shard oracle and one +frozen released-process inline fixture with three corruption negatives locally; +this executor still needs a new candidate-bound tool build and live evidence. +D06 `heal.object-metadata-and-tags` preserves all three H1/H2 branches with +three historical payloads per Unicode/space/literal-percent key, actual gzip +content, encoded Chinese user metadata and two distinct tag groups. It checks +the initial normalized metadata/Content-Type/Encoding/Disposition/Cache-Control +and precise versioned tags, then faults an external data part, an entire xl.meta +header, or a parity part. Exact physical/version references, complete S3 +inventories, HEAD and ranges must agree after repair. This executor still +requires live evidence. +Four Heal cases retain a canonical acceptance gate: C11, F07, I05 and J02. All have dispatch routes. All four have explicit diagnostic routes that preserve their blocking canonical verdicts; I05's H1/H2 timeout and queued-successor diagnostics have independently audited live results, exact-source native prerequisites and stopped-scene isolation. I05 now composes both H1/H2 controllers with separate same-artifact native owner and full-content/physical generation checks. The native proof does not establish that the fleet request held a distributed lock at disconnect, and the composed source plan still requires independent complete live audit. The 13 environment-gated cases have disabled-by-default control flows and oracles in `environment.py`; actual execution requires an assigned and reviewed auto-testing resource adapter, and canonical results remain review-gated. See [the environment gate reference](validation-heal-environment-gates.md) for each missing resource, capability and native barrier. +H4 needs at least 24 independent disk endpoints; H6 and native-fence branches +retain their platform and source-bound evidence requirements. + +The RustFS tool-build lane produced the initial candidate-bound metadata inspector in run `38052536366`; A07 also passed with that tool in run `38053214984`. The extended bitrot helper was successfully built against the same candidate in run `38054812014`, including the released-shard oracle checks. Set `validation_tools_only=true` with the pinned build and auto-testing SHA to build the current helper. Heal live runs consume `validation_tools_run_id`/`validation_tools_run_attempt`; the workflow verifies the successful build attempt, candidate/helper identities and binary/lock digests. The node adapter installs the verified tool under its run-owned root. Plain metadata decoding makes no payload-integrity claim; the new explicit part-verification mode validates current and legacy streaming HighwayHash. The build checks 24 healthy shards captured from external released processes and rejects 12 controlled corruptions. Ordinary live execution needs no RustFS source checkout. + +The eight planned cases have these concrete reuse candidates: + +| Validation ID | Candidate IDs | Specific item to verify | +| --- | --- | --- | +| `scanner.usage-convergence` | STOR-134 | Dedicated executor and live PASS evidence. | +| `kms.sse-kms-round-trip` | KMS-104, KMS-301, KMS-214 | Dedicated executor and live PASS evidence. | +| `tiering.transition-and-restore` | TIER-105, TIER-107 | Dedicated executor and live PASS evidence. | +| `events.delivery` | EVT-103, EVT-104 | Dedicated executor and live PASS evidence. | +| `audit.request-correlation` | AUD-102, AUD-104, AUD-106 | Dedicated executor and live PASS evidence. | +| `iam.policy-enforcement` | IAM-105, IAM-109 | Dedicated executor and live PASS evidence. | +| `oidc.login-and-session` | OIDC-103 | Dedicated executor using real Keycloak login and STS session boundaries; live PASS evidence. | +| `s3.basic-object-round-trip` | S3C-101, 201, 203, 205, 206, 207 | Dedicated executor and live PASS evidence. | + +Migration work on an ID is complete only when its result contains the exact +source assertion text in order, evidence paths for each assertion, the +candidate/source/auto-testing/rc provenance, and a clean finalizer result. +The report rejects a PASS that lacks any of those fields. Existing suite +results cannot be renamed or counted automatically. + +The runner should use the catalog's target environment as a profile, keep +one unique artifact directory per run, write `RUNNING` before invoking a case, +and retain the primary failure when cleanup also fails. Tests that mutate the +shared fleet need the RustFS workflow concurrency group and an environment +lease. Special Heal topologies and source-bound tools are provided by +separate RustFS build/environment jobs; ordinary case execution consumes +their immutable artifacts. + + +The corrected default-mode C01 run `38055597354` executed every H1/H2 fault branch with clean finalization. Single-part bytes and HighwayHash were restored, but the candidate returned unknown responsibility with no incarnation. The candidate explicitly defaults independent-shard writes off; current positive fixtures now enable both shard-integrity write and fleet-confirmation knobs only after every private peer is bound to the same candidate. Their values are read back from the actual processes and archived. The default-mode failure remains separate and never becomes a positive proof. New protected-mode runs are required before any new PASS claim. + +Protected-mode A01 run `38056636793` passed all 512 exact H1/H2 normal/deep +object responsibility receipts and clean finalization. This proves only the +explicit, uniformly enabled shard-integrity fixture configuration; the default +deployment failure remains retained. A04 run `38056884022` passed DB branches +but failed DV receipt completeness with cleanup PASS. The executor had used a +nonrecursive object request, which gives latest-object responsibility rather +than all historical versions. DV requests now explicitly set recursive=true +for the exact object prefix; the original FAIL remains and a new run is required. +The corrected recursive DV request passed all DB/DV H1/H2 branches in run +`38057908339` with clean finalization, including complete dry_run_observed +version/marker receipts and byte-for-byte no-mutation proof. + +D03 run `38058420193` passed historical/current/multiple-version branches on +both H1/H2, but its bucket-recursive branch used an extra trailing slash and +received HTTP 501 before admission. Cleanup passed. The task URL now uses the +registered exact bucket route; this harness failure remains retained and D03 +needs a fresh full run before a canonical PASS claim. + +J04 run `38058699913` retained FAIL with clean finalization. Its DB checks hit +an executor Range bug on exact/near-1MiB controls; positive ranges now use +actual bounded intervals and have boundary regression coverage. The DV H1 +branch passed first repair and both normal/deep repeats. DV H2 physically +restored all original bytes, decoded version references and S3 manifests, but +the candidate attributed the current delete marker as unknown while three data +versions were repaired. That genuine receipt failure remains strict and visible. +Tool build `38058949922` successfully produced the current decoder plus the +recognized exact-candidate quorum/error contract for B07. + +The corrected D03 run `38059278988` admitted every source branch, passed H1 +completely and passed H2 historical/current/recursive branches. H2's simultaneous +multiple-version fault left one original current-version parity part missing. +The exact physical oracle retained FAIL and clean backup-assisted finalization; +that restoration does not count as product repair. + +Fresh corrected J04 run `38059567206` passed every DB/DV H1/H2 source branch, +normal/deep independent-token repeats and same-unit process restarts, with clean +finalization. The earlier same-candidate H2 unknown-marker failure in run +`38058699913` is still retained; the new PASS does not rewrite that evidence or +claim a product fix. Tool build `38060217084` successfully built the extended +inline verifier and checked the 24 healthy/12 corrupt external shards plus the +one healthy/three corrupt released-process inline samples. + +D05 run `38060400818` exercised all H1/H2 suspended/null branches and cleaned +up successfully, but its receipt oracle compared the Admin typed nil UUID with +the S3 `null` spelling literally. The candidate source defines these as the same +null identity. The oracle now normalizes only those representations and still +rejects every other version mismatch. The original FAIL remains; a fresh full +run is required. Auto-testing CI passed all six lanes at commit `f9c153f`. + +B05 `heal.multiple-shard-loss` now expands 1..M losses from the actual decoded +EC geometry, including every possible data/parity role count. Each parameter +instance uses a distinct acknowledged object; DB and DV both run on H1/H2. +All drives remain online for the write/lock condition while selected same-version +parts are isolated. Exact full original shard hashes, streaming checksums, +complete version/marker responsibility and S3 controls must all pass. B05 has +not yet been validated live. + +B07 run `38060686737` retained FAIL with cleanup PASS. At R-1 shards H1 +returned HTTP 500/InternalError instead of the recognized candidate read-quorum +refusal. H2 returned 503/SlowDownRead, but Heal recorded a grace-window deferred +receipt and completed execution rather than the source's required failed +responsibility. Good-shard preservation and the raw task outcome remain +archived; neither branch is promoted to PASS. + +D10 `heal.special-keys-and-pagination` now creates 265 unique objects, including +Unicode, spaces, literal percent/encoded slash, real slash, nested paths and +adjacent `a/`/`ab/` controls. Both version and current LIST explicitly request +MaxKeys=17 and retain observed page sizes; the complete ACK manifest must match +without missing or duplicate keys. Exact logical-key physical mappings precede +part/xl.meta faults and signed object requests. Every control remains exact. +D10 has not yet been validated live. + +C05 run `38061075879` exercised all ten DB/DV H1/H2 branches and retained FAIL +with cleanup PASS. Nine branches passed. H1's truncated xl.meta remained at the +exact injected 1866-byte length instead of its 3732-byte baseline and could not +decode; the physical oracle rejected it before target GET. + +D06 run `38061589839` passed all H1/H2 semantic branches with cleanup PASS: +three distinct versions for each Unicode/space/percent key retained exact public +metadata, actual gzip bytes/headers and versioned tags through data, metadata +and parity faults. Physical original-shard/bitrot and full S3/HEAD/Range checks +passed before receipt promotion. + +D01 `heal.object-size-boundaries` now has a complete automatic parameter matrix +for zero/one, 4KiB, actual inline and EC stripe boundaries plus a multi-block +external object. A tool-build contract freezes recognized candidate policy and +source/extractor digests; live process environment and actual EC metadata derive +the limits. H1/H2 need different inline limits under the recognized current +policy (the runtime derives these rather than guessing 256KiB). Each size tests +both actual data/parity roles with missing and bitflip faults. Zero objects use +metadata loss/header faults and never invent a payload part. Exact byte hashes, +complete responsibility, HEAD/LIST and actual first/tail/stripe Range checks +are required; zero Range is a strict S3 416/InvalidRange negative. Tool build +`38062137032` successfully produced the recognized size and quorum contracts +for the fixed candidate, with current extractor/source and binary identities. + +Corrected D05 run `38061843290` passed every H1/H2 suspended/null branch and +clean finalization. Real enabled-period history, null overwrites and markers +retain precise S3 identities; the typed Admin nil UUID is mapped only to that +acknowledged null version. The candidate then re-enabled versioning and created +a distinct non-null version without changing old payloads or version/marker +identities. The original literal-identity harness failure is still retained. + +A08 `heal.admin-authorization` now dispatches every H1/H2 source principal +branch. Anonymous, deliberately bad, correctly signed but expired, and denied +IAM requests each test start/status/stop with exact HTTP/error refusals and +recursive JSON/XML task-disclosure rejection. A temporary principal with only +`admin:Heal` admits, queries and stops its independent token while S3 reads stay +denied. Temporary users/policy/tasks are finalized and credential revocation is +verified; original physical/S3 fixtures must remain unchanged. Evidence redacts +root and temporary credentials without altering oracle inputs. Nine focused +tests cover independent SigV4 verification, denial leakage, lost IAM admission, +cleanup failures and revocation. A08 still requires full live evidence. + +D01 run `38062610870` retained FAIL after 35 H1 parameter attempts. Three +physical comparisons rejected changed target-side `xl.meta.bkp` files; archived +per-drive comparison shows each new backup is the exact pre-repair authoritative +`xl.meta`, while payload shards and decoded logical identities agree. The +original verdict remains retained pending a new complete run with a bounded +target-backup oracle. A fourth attempt was rejected before injection because +its expanded fault ID exceeded the tool limit; retrying that invalid ID during +finalization caused cleanup FAIL. Fault IDs now use branch/index digests within +the allowed length. Recovery run `38063798238` independently proved stopped +owned service, no injection intent, exact archived pre-fault physical scene and +removed run roots. It records cleanup PASS, `originalVerdictRewritten=false` +and no canonical/product-repair PASS claim. Serial deployment resumed only +after inspecting that recovery evidence. + +B09 `heal.node-outage-rejoin` now has both source H2 branches with independent +DB and DV deployments on the existing four nodes. The candidate tool freezes +the recognized same-set read/write quorum policy; actual metadata maps the +four lost drives before admitting degraded I/O. Exact unit exit and refused +endpoint connections witness the outage. PUT/overwrite/marker/historical-delete +ACKs define the new manifest, while the stopped node's complete physical scene +stays unchanged. Only the explicit run-owned `offline-replay` profile enables +MRF; scanner and automatic disk-rebuild remain disabled. After the same node +and roots rejoin, physical polling precedes any target S3 reads or manual task. +Every drive must contain the exact acknowledged version/marker set; surviving +shard bytes, committed streaming checksums and all-node S3 manifests must agree. +Deleted identities receive exact S3 absence checks. No copied backup contributes +to repair. Six focused tests cover profile boundaries, source-quorum recognition, +ACK ambiguity, per-drive omissions/resurrection and deleted-read refusals. +B09 requires the fresh write-quorum tool artifact and full live evidence. + +D01 run `38064187494` completed all 96 H1/H2 parameters: 76 passed and 20 +failed the then-strict existing-backup oracle; cleanup passed. Independent +forensic comparison of all 20 scenes found only target `xl.meta.bkp` updates +to the exact pre-repair authoritative metadata. The corrected oracle now +bounds both new and existing backup updates to that drive/key/dataDir and +exact size/hash. The original FAIL is retained; a fresh full run is required. + +A08 runs `38066188001` and `38066437827` failed policy creation because bare +`Resource:["*"]` is not an ARN. The fixed candidate accepts `admin:Heal` +without Resource. Corrected run `38066970195` reached every branch: bad and +expired signatures, denied user and least-privilege user passed on H1/H2. +Anonymous traffic returned 403/AccessDenied from S3Router::check_access before +the handler, exposing an incorrect 400/InvalidRequest harness expectation. +The route-level expectation is now exact; all original FAILs retain cleanup +PASS and none is promoted without a fresh complete run. + +Tool build `38065628505` succeeded against the same fixed candidate and carries +the recognized `data-plus-one-if-equal-parity` write-quorum policy. Its helper, +extractor, candidate and binary digests were verified locally; it is used by +the serial live queue for D01/B07/B09. Ordinary runs still need no RustFS checkout. + +D09 `heal.object-lock-retention` now dispatches all four H1/H2 source branches +on real Object Lock/versioning buckets. Three versions per key retain exact +Governance/Compliance metadata or ON legal hold; the ten-minute UTC window is +anchored to the server HTTP Date. A run-owned principal can read but explicitly +cannot bypass Governance. Baseline and pre/post-fault DELETE must return exact +403/AccessDenied while retention is active. Data/parity faults and whole xl.meta +loss cover current/historical versions; all affected version shards, public +properties, tags and responsibility receipts remain independently checked. +Finalization releases hold and Governance only through normal APIs, waits up to +twelve minutes for real Compliance expiry and deletion ACK, proves the bucket +empty/absent, and removes IAM resources. Only then may physical roots be removed. +Incomplete finalization retains a stopped namespace with cleanup FAIL. Partial +seed ownership, real server time, same-drive multi-version proof selection and +protected cleanup failure boundaries have focused tests. Full live evidence +is still required. + +C08 `heal.metadata-payload-mismatch` dispatches every H1/H2 branch with three +real versions per distinct key. The donor is another acknowledged version's +same-role/same-slot part: a different physical length for length mismatch, +the same length with different bytes for payload mismatch. Candidate-bound +streaming verification proves both original and donor cohorts before injection. +The descriptor-relative fault layer admits only another owned same-number part +on the same disk, exact donor/target hashes and quiescent service; it rejects +metadata, traversal, symlinks, hardlinks, unowned buckets and identical bytes. +Intent and original bytes are persisted before replacement. Every physical +fault receipt must equal the frozen donor; metadata and controls stay unchanged. +The negative branch leaves exactly R-1 correct same-version shards and requires +exact read refusal, truthful failed responsibility and no physical mutation +before rescue. Restored backups certify cleanup only; a later-generation target +cannot be overwritten. Positive branches require original shard bytes, full +version metadata/contents/tags, streaming checksums and repaired receipts. +This executor has local fault/recovery/authority tests and still needs live proof. + +E03 `heal.read-repair-attribution` dispatches GET-missing and Range-bitrot DB +branches plus an exact historical-version DV read on H1/H2. Its explicit +profile leaves scanner, automatic rebuild and best-effort MRF ingress off, +while enabling candidate JSON debug events; the candidate's durable consumer +stays initialized independently of that ingress switch. Before each single +GET/Range the exact fault remains physically present. Correct bytes, exact +version/ContentRange and a server request ID are required. A new structured +`Read-repair heal request admitted` event must match bucket/key/part/pool/set +and accepted task ID; stale events, text mentions, merges and ambiguous tasks +cannot prove attribution. Only status queries observe that automatic task; +no Admin start/deep verifier repairs the fixture. The task's exact version +receipt, physical original shard hashes, streaming checksums and full public +S3/version controls must all agree. Runtime credentials are redacted from +archived debug journals. Focused tests cover source identity, stale-task +refusal, single-read/no-mutation dispatch, exact content/range/request identity +and runtime profile isolation. Full live proof remains pending. + +Corrected D01 run `38067139720` passed all 96 H1/H2 parameters (48 per +topology), with canonical PASS, complete summary and cleanup PASS. Its exact +candidate/package/binary/harness identities and each parameter's evidence were +downloaded and independently checked by the canonical report gate. It covers +zero/one/4KiB, actual inline and EC-block boundaries, multi-block payloads, +both roles, missing/bitflip faults and exact Range outcomes. The three prior +D01 failures and independent recovery record remain unchanged. + +B02 `heal.disk-readonly-and-permission` dispatches both H1/H2 branches. The +read-only gate uses systemd PrivateMounts/ReadOnlyPaths for the selected +run-owned data root; host mounts are never remounted. The permission gate +records/restores that root's original 0700 mode and drops both DAC override +capabilities from the root service, so chmod cannot be bypassed. A write probe +enters the actual service mount namespace with the same effective UID/GID and +DAC boundary and requires exact EROFS/EACCES. Original shard isolation precedes +the gate. Failed/deferred exact-version responsibility must show zero repaired +counters and the complete physical fault scene must remain unchanged. After +service exit and gate release, a distinct task must actually repair the +original target; full physical, streaming and public checks follow. Gate intent +precedes chmod/start and is recovered before any fault rescue or physical +cleanup. Root drift or release failure retains the namespace. Focused tests +exercise permission restoration, write refusal identity/errno, no root bypass, +truthful unresolved receipts and cleanup retention. Live proof is pending. + +E04 `heal.scanner-cold-object-repair` now dispatches all three H1/H2 branches. +It reads back the actual private process settings for five-second scanner +cycles, every-cycle bitrot, zero deep cooldown and full object selection, while +replacement auto-heal and best-effort MRF stay disabled. Before any fault, each +cold target needs an exact healthy scanner task and unchanged physical files. +Faults are injected with all run-owned processes stopped; the stopped scene +proves damage immediately before restart. Cursor-bounded journal windows bind +the exact object discovery to an accepted Scanner source request, task start, +completed cycle and terminal task. The cycle-boundary branch requires two +complete post-fault cycles. Observation issues status queries only; no target +GET/HEAD or Admin mutation contributes to the repair. Exact repaired version +responsibility, original physical shard bytes, streaming checksum and complete +S3 inventories still gate PASS. Truncated logs, another source, a later healthy +receipt or cycle counters alone cannot certify repair. The executor passed its +local source/timing/cursor negatives; full live evidence remains pending. + +D04 `heal.delete-marker-semantics` now dispatches its four H1/H2 source +branches as six real instances. Each key has three separately acknowledged +data versions and, where applicable, a current or historical marker. Missing +replicas cover both physical roles. Stale creation and deletion isolate one +original drive with a service-identity read/write EACCES gate, perform one +exact marker mutation, and prove the gated replica stayed unchanged while all +other members carry the ACK authority. The expected metadata is derived only +for comparison from that authority and the original target-drive data +references; the harness never writes synthesized metadata. Formats, data +shards, controls, exact version receipts and streaming checksums must agree. +Every peer checks current-marker 404 and explicit-marker 405 with exact marker +identity headers. Explicit current-marker deletion must reveal the precise +previous data version, remove the marker identity and preserve the full +remaining version/public-property manifest. A failed or ambiguous stale ACK +stops the topology and retains its result; gate cleanup precedes physical +finalization. Local ACK/order/authority/header/gate negatives passed; full live +evidence remains pending. + +J01 `heal.status-partial-disabled-unknown` now has a dedicated current-fleet +executor for H1/H2. Fresh manual and disabled deployments retain actual process +configuration, complete physical fixtures and both raw Admin and rc JSON/text +outputs. Peer coverage must balance exactly; absent peers need an explicit +unknown count/reason and cannot report idle. Any raw or CLI sample that erases +uncertainty fails immediately, including during bounded convergence. H2 stops +one original peer, witnesses its process exit and connection refusal, and +queries every survivor before restarting the same member. The source's step 2 +explicitly restricts that branch to H2; H1 records its applicability reason +and instead proves an original-service restart followed by complete idle. +A startup observer begins before any service starts and requires raw +uninitialized status to bracket matching rc/replacement observations. A +transport error or an already initialized service cannot prove that window; +NOT_OBSERVED makes the complete case fail. Original formats/object files and +full S3 manifests must remain unchanged. Temporary rc credentials use a 0600 +configuration outside artifacts and are removed during finalization. Local +coverage/uncertainty/startup/config-cleanup negatives passed; live evidence +remains pending. + +J01 selects the official rc v0.1.36 Linux amd64 musl release in a new private +artifact directory. Both archive and executable SHA-256 values are fixed and +checked before execution; its manifest records the release source, target, +version output and exact bytes. This release includes the existing CLI fix +that preserves incomplete/unknown status in human-readable output. The musl +build avoids depending on the runner's glibc version. Other Heal cases retain +their existing client selection. The original failed J01 result is retained; +the complete raw/CLI startup, disabled, missing-peer and recovery observations +still require live acceptance with the fixed client. + + +J03 `heal.incremental-cursor-and-truncation` now has a current-fleet DX +executor with 1025 pre-registered external objects of 1MiB+73 bytes. The +reference candidate retains 1024 result items, separately from its 128-object +outcome diagnostics; a 257-object task cannot prove the retention-gap branch. +The actual terminal minSeq must move and an old cursor must carry truncation. +Incremental queries treat sinceSeq as exclusive and advance only over returned +items, including when the response payload itself is cropped. Same-task bounds +cannot move backwards or duplicate object/version identities. At least two +useful running windows are mandatory; NOT_OBSERVED prevents a complete PASS. +H2 queries every actual node with the admitted path/token and uninterrupted +deployment epoch; H1 records why a cross-node query is inapplicable. The +candidate renders startTime anew on each response, so A03 also binds its +original query path/token instead of comparing that timestamp. Canonical +counters must cover all 1025 ACKs with the exact 17 real missing-shard repairs, +no failures/unknowns, and complete traversal. Bounded diagnostic receipts remain +explicitly incomplete. Full physical manifests, every original same-role shard +and streaming checksum, and all-peer S3 manifests independently cover every +object before PASS. The case has a fixed 16384-file snapshot budget, 128-part +verification batches, a 45-minute service/lease budget and 2GiB free per drive. +Local cursor/gap/counter/scope/part-omission negatives passed; live evidence +remains pending. + + +A06 `heal.admin-stop-and-stale-token` now uses the same pre-registered 1025 +external-object DX fixture on H1/H2. Cancellation requires actual running, +nonzero partial canonical progress and both repaired and still-missing targets. +The frozen candidate policy requires active stop acknowledgement, retained +aborted/cancelled partial responsibility, and exact HTTP 200/stopped +idempotent acknowledgements for repeat/stale stops of a durable terminal token. +Unknown tokens remain a separate exact 500/InternalError contract. Three complete physical snapshots with +five-second gaps keep the service running after cancellation; stopping the +process cannot substitute for task write quiescence. Actual restored shard +counts must equal the retained cancellation counters, and remaining shards +must still be missing. A distinct new token must be running before and after +old-token STATUS/STOP; unobserved brackets prevent PASS. The new task repairs +only the remaining defects, and its completion must preserve the original +cancelled outcome. Full streaming/physical and all-peer S3 verification happens +before backup rescue. All four source branches remain explicit. Local +cancellation/counter/quiet-physical/token/error negatives passed; full live +proof remains pending. + + +F06 `heal.cancelled-task-restart` extends the same real partial cancellation +and write-quiescence proof with a graceful restart of every original H1/H2 +unit. A read-only node action accepts only a canonical admitted task UUID and +three exact candidate journal names under the private run roots; it records +all four drives on every node, bounds terminal/intent to 64KiB and report to +8MiB, and rejects links, hardlinks, oversized or changing files. The actual +cancelled terminal and report need one matching physical owner, exact task and +prefix scope, no runnable intent anywhere, and the settled partial outcome and +valid sequence window. It never synthesizes a checkpoint or edits system +records. After same-root restart, every actual process must be new while its +unit/candidate/formats/objects remain original, all peers must be idle, old +status must still be cancelled, and the exact original journal bytes must be +preserved. A fresh forceStart task remains independent under old STATUS/STOP +and repairs only the remaining witnessed defects. Full physical/streaming/S3 +proof remains mandatory. A06 also now preserves both its initial cancellation +and allowed monotonic in-flight report refinement before freezing the report +after actual drain; later quiet/restart/new-task checks cannot rewrite it. +Local journal-owner/schema/scope/cursor/read-boundary negatives passed; full +live evidence remains pending. + +A05 `heal.admin-overlap-and-force-start` now dispatches all four source +branches on both H1 and H2, with an independent deployment/ACK ledger/fault +set for each instance. Two real prefix scopes contain 257 external objects +each; five missing shards per selected scope retain positive authority. +The actual process environment freezes `merge`: equivalent starts must return +the same token, parent/child requests must return exact 409/OperationAborted +with `overlapping_paths`, and disjoint requests must receive independent +tokens. Every control request needs a nonzero running bracket; a missed +window remains NOT_OBSERVED. forceStart must cancel the old task with partial +coverage, admit an independent replacement token, and resist old-token STOP. +Initial and settled cancellation reports are retained; exact old/new repair +counters, incarnations and diagnostic identities cannot leak or duplicate +responsibility. All 257 original item sequences are retained while running +and reconciled with the exact terminal outcome and retained suffix. The +completed-report byte budget can evict an early prefix below the 1024-item +count cap; an unobserved prefix still fails completeness. Both disjoint tasks +are sampled concurrently, and replacement drain samples are preserved. Three +live physical samples after actual all-peer drain must remain unchanged, +with the same processes. Complete same-role shard bytes, streaming checksums +and all-peer full S3 manifests remain mandatory before PASS. Local +scope/error/window/duplicate-repair/item-omission/cleanup-stop negatives +passed. Run `38199371461` retained FAIL with clean finalization: H2's first +three branches exposed the early-prefix accounting gap, H1 force-start had a +typed cancelled receipt counted as skipped, and H2 force-start contained a +real unknown receipt. Replaying the original running records verifies all +three 257-item primary ledgers; H1 cancellation now requires its exact typed +receipt count, while H2 unknown remains rejected. This diagnostic replay +does not change the original FAIL or claim a complete new live PASS. + +F01 `heal.root-graceful-coordinator-restart` now dispatches both source +branches on H1/H2 with the same fixed DX physical/S3 oracles. The coordinator +comes from the unique admitted physical root intent, rather than a guessed +node alias. Actual nonzero task progress and both restored/pending target +shards must precede interruption. Linux pidfd binds pause, SIGTERM and resume +to the original process birth; the original run-owned unit retains actual +normal-exit-zero evidence. All peers retain their original process except +that coordinator. The exact schema-3 root intent stays on the same drive, +with original task/scope/options/priority/admission time and a finite +remaining timeout that cannot increase. Pending peer writes must settle +before same-unit/environment/volumes/candidate restart. The original token +must recover, report complete exact responsibility and repair the remaining +physical defects; no new START compensates for lost responsibility. Runnable +intent must retire on every drive, while completed terminal/report remain on +the original owner. A second coordinator restart preserves their bytes, +completed outcome and all user files and remains idle. This proves durable +admission replay, and makes no cursor-resume or record-disk failover claim. +Local owner/budget/process-birth/exit/old-token/journal negatives passed; +full live evidence remains pending. + + +B06 `heal.read-quorum-boundary` now dispatches all three source branches on +H1/H2 with three independent 5MiB+123-byte never-versioned ACK objects. Its +read and write quorums come from the recognized candidate artifact and actual +decoded geometry; the reviewed candidate lock policy counts pool host:port +clients, so four H2 hosts have exclusive quorum three, not nine disk locks. +The exact-R GET archives read-repair interference and never certifies manual +repair. The separate exact-R manual object must reconstruct every missing +original shard and return exact completed responsibility. F1 only isolates +payload files: all drives, metadata and locks remain online, so it cannot +represent insufficient writable-drive or lock quorum. That condition is +explicitly non-applicable here; B08 retains its independent real fault +requirements. The restored-W branch first proves the candidate's exact R-1 +read refusal with no physical changes, then restores only W-(R-1) original +backups. Those sources never count as product repairs. A distinct manual +token must reconstruct all N-W remaining targets, followed by full physical, +streaming checksum and all-peer S3 checks. Live evidence is pending. + +A08 `heal.admin-authorization` passed live in RustFS Actions run +`38069648003`, harness `0789743d439fd91b967c2d6fba85c06e29f844c5`. All five +source branches passed independently on H1 and H2; the canonical gate, +complete summary, cleanup and workflow all passed with the fixed candidate. +The prior A08 failures remain recorded. Thirty-seven Heal executors remain +to be migrated; automatic dispatch and local tests do not claim live PASS. + + +C06 `heal.stale-version-rejoin` now dispatches all three source branches on +H1/H2, with never-versioned overwrite and independent versioned historical +delete/marker fixtures. A private permission gate removes both DAC bypass +capabilities and activates only after the original candidate has started. +Same-process read/write EACCES proves that the unchanged member cannot +participate in the subsequent ACK transition. Every online metadata replica +must agree on the complete acknowledged namespace, the isolated member's +entire physical disk remains exact, and controls retain their original bytes. +The same disk UUID and set slot must rejoin without harness metadata writes +or restoring old-generation backups. An independent deep token must correct +the stale namespace; exact outcome counters, retained same-drive shards, +streaming checksums, deleted-version absence, current marker headers and +all-peer S3 manifests are required. Original payload nonces remain attached +to ACK versions after deletion so Range checks cannot silently renumber +history. The candidate's dangling grace is explicitly zero for this rejoin +profile; C10 retains the independent grace-policy requirements. Complete +cursor-bounded generation-selection logs are archived. Live evidence is pending. + +B02 run `38079787495` retained FAIL and cleanup PASS on both H1/H2. The +readonly window produced `completed_with_errors` with one failed receipt, +zero repaired targets and `summary=stopped`, but omitted bucket incarnation. +The stopped summary is now interpreted only with explicit failed execution, +complete coverage and exact counters; missing incarnation still fails. The +permission setup previously chmod'ed the data root before service startup, +causing the candidate to exit with EACCES before injection could be witnessed. +Permission gates now retain an armed recovery marker during initial startup +and inject chmod only into the same live candidate with DAC bypass already +removed. This fixes the injection window without certifying the old run or +relaxing physical/refusal/responsibility assertions. A fresh run is required. + + +B01 `heal.disk-loss-and-repair` now dispatches both source branches on H1/H2 +with the full 128-object DB manifest, including zero/inline/external objects. +The same-service permission gate must deny reads and writes on one exact +original disk, and signed read-only Admin info must independently identify +that exact formatted UUID/set slot/endpoint as offline while all other +members remain online. The double witness brackets every unique-key PUT ACK +and degraded GET. Original member files, controls and formats remain exact +through the offline window; all N-1 online new-key metadata replicas and +payload shards must form one protected ACK authority. Original-drive-return +admits one new key and offline-ack-repair admits six independent new keys. +The same disk returns without restoring backups; each ACK needs its own +manual task, complete repaired receipt, actual new target metadata/part, +unchanged donor shards, protected streaming checks and full all-peer S3 +manifests. Ready/GET alone never prove repair. Live evidence is pending. + +B06 run `38081025407` retained FAIL and cleanup PASS. Its exact-R GETs on +both H1/H2 returned correct full payloads, but the collector omitted +`resultUnknown=false` on successful responses and the boundary check rejected +them. Successful complete S3 responses now explicitly expose known results; +partial bodies still fail. Both exact-R manual branches and H2 restored-W +passed within that failed run. H1's R-1 read returned HTTP 500/InternalError +instead of the frozen 503/SlowDownRead contract; that failure remains visible. +No failed run is promoted to PASS. + +C06 run `38081269532` retained FAIL and cleanup PASS before injection. Its +original ACK nonce was incorrectly compared as if it were an S3 property. +Inventory checks now validate the nonce as private collector state while +retaining exact version/marker/currentness/length/digest comparisons. A +regression check exercises nonce-bearing ACKs against real public inventory +shape, including wrong content and version negatives. Fresh live evidence +is required for both corrected collectors. + + +B03 `heal.disk-replacement-formatted` now dispatches both source branches on +H1/H2, with full DB and historical/marker DV fixtures. A stopped, owned adapter +moves exactly one complete private data root into same-mount quarantine and +creates a genuinely empty new root. Before/after whole-tree digest and original +inode/device preserve every old file, including system records; no shared +mount is detached. The actual service uses a private namespace with quarantine +masked and both DAC bypass capabilities removed. Same-process EACCES proves +that the original cannot supply rebuild bytes. The candidate must initialize +a distinct new disk UUID in the same original pool/set slot without replacing +another member or changing geometry. Every original target version, marker, +inline value and external part must rebuild through one exact deep task, with +truthful complete receipts/counters, original same-role hashes, protected +streaming checks and full all-peer S3 manifests. Missing/unchanged UUID or early +automatic repair cannot satisfy this manual case. Partial rename/mkdir intent +remains recoverable; archive drift retains the namespace. Live proof is pending. + +B01 run `38081858808` retains its original FAIL and cleanup PASS. Both H1/H2 +had real same-service read/write EACCES. Independent replay of all 46 H1 and +45 H2 original Admin responses finds the exact original slot, endpoint and UUID with +`state="drive access denied"` and cached `runtimeState="online"`. The candidate's +`DiskAccessDenied` error renders that exact state, and its Admin disk-count +logic counts it as offline. The executor omitted this spelling from its +explicit error set, so it rejected the recorded witness and never started +degraded ACKs. The state mapping now recognizes that exact error while still +requiring the original identity and every other member's healthy UUID. Unknown, +initializing, unrecognized error and online-only observations remain refused. +Replaying this offline check does not supply the unexecuted ACK or repair +branches; complete fresh H1/H2 live evidence is still required. + + +B06 corrected run `38082447868` retained FAIL and cleanup PASS. Both exact-R +GET and independent manual branches now pass on H1/H2; all three H2 branches +pass. H1's R-1 GET still returns 500/InternalError rather than the frozen +503/SlowDownRead, so no complete canonical PASS is claimed. + +C06 corrected run `38082227039` retained FAIL and cleanup PASS after proving +ACK authorities: full fleet restart allowed physical changes before the manual +token and the attribution check stopped execution. Rejoin now restores only +the exact active permission gate in the same live process. Two identical live +physical observations freeze the settled ACK authority; every peer's actual +process birth/unit/candidate/environment must remain unchanged, and the fault +must still exist immediately before the independent manual token. Startup +reconciliation cannot contribute to the manual proof. Fresh live evidence is +pending; the original attribution failures remain unchanged. + +E04 journal evidence now reads the first bounded page after each exact cursor +using `journalctl -n +10000` and per-entry journal cursors. Full pages are drained +in order, with six pages per capture and a hard failure on incomplete or +nonadvancing windows. Raw pages remain evidence; only source-attribution targets +are retained in the scanner's in-memory index. The healthy checkpoint starts +after seed, baseline and the initial physical snapshot, so a fresh complete +scanner cycle must actually occur in the registered observation window. Earlier +log-budget failures remain FAIL; this harness change has no live PASS claim. + +F05 `heal.full-cluster-restart` now has an H2 executor for all three source +branches. It seeds DB, historical DV/markers, completed multipart objects, +exact pending UploadIds/parts and a 1,025-object DX observation fixture on the +current four Heal peers. A root task must have physically repaired some but not +all injected shards before the original coordinator is paused. Every original +peer then exits with witnessed SIGTERM/normal code zero in a registered order. +The same task ID, physical intent owner, non-timeout options and remaining +budget must survive all units being stopped. Recovery uses no new root start. +Complete canonical counters, bounded exact diagnostics, durable terminal/report, +full same-role physical manifests, streaming shard checks and all-peer S3 +inventories must agree. All four processes restart again after completion; +terminal records and user files must remain unchanged. Pending MPU listings +remain exact across both restarts and their original parts are then completed +and checked by full payload hash on every peer. Multi-bucket snapshots and +streaming verification bind each run-owned bucket explicitly. F05 has no live +PASS claim until that complete sequence succeeds. + +C06 run `38083035055` retained FAIL and cleanup PASS: all three H1 branches +passed with the same live service rejoin, but H2 failed the older 10,000-line +journal bound. The paginated journal implementation must be validated in a fresh +run. B03 run `38083328049` retained FAIL and cleanup PASS on all H1/H2 DB/DV +instances: the candidate had no replacement `format.json` at the first physical +snapshot. The executor now observes a registered 90-second, read-only format +initialization window before requiring distinct new UUID/set-slot evidence; +service health still cannot certify replacement acceptance. Neither result +increases the complete live PASS count. + +E06 `heal.replacement-auto-rebuild` now dispatches both H1/H2 DB/DV branches +using the same whole-root replacement isolation and complete shard oracles as +B03. Its isolated runtime enables only automatic disk replacement (10-second +interval), with scanner and MRF disabled. The current lab's private directory +roots explicitly use the candidate's directory-backed replacement fallback; +physical-device replacement readiness is not claimed. Every peer must expose +one fresh, stable generation bound to the exact target endpoint and set, with +completed state and a fresh verification timestamp. Missing/partial/unknown +status, a changed generation or divergent terminal records fail. No Admin heal +mutation or target S3 read occurs before full same-role physical reconstruction +and streaming verification. The old archive remains inaccessible to the service +and unchanged. A healthy service or empty queue cannot finish the case. This +executor still requires complete live evidence. + +E04 `38083897141` and C06 `38084060749` retained FAIL/cleanup PASS because the +first paginated collector used journald's default JSON field-size limit, which +can omit a long MESSAGE. The collector now requests `--all`, decodes complete +UTF-8 byte-array messages and rejects missing fields; original failures remain. + +F02/F03 now execute both H2 source branches using automatic replacement. The +unversioned observation fixture has 1,025 external objects; the versioned +fixture has 64 keys, 192 data versions and their delete markers, with a 45-minute +private-service lease. Only node1 enables automatic disk discovery; the other +three nodes retain enabled Heal with discovery disabled. Before interrupting +node4, fresh cursor-bounded logs must bind the original generation to accepted +`auto_heal` admission and exact erasure-set task start on one distinct +coordinator. Physical census must show both committed original same-role +external shards and pending data versions. The target is paused through its +exact MainPID/birth identity and the partial window is checked again. F02 then +requires witnessed SIGTERM/normal exit zero; F03 requires pidfd-bound SIGKILL, +signal 9 and a non-graceful systemd exit, with no platform power event. No other +peer process may change. Restart uses the same unit, environment, roots and new +disk UUID, reproves old-archive refusal and finishes the same automatic +generation. Complete physical/streaming and all-peer S3 proof remain mandatory. +A fast completed task, missing distinct coordinator, changed generation or +missed physical window cannot PASS. External target verification uses bounded +128-part batches to keep the DX run within its registered resource budget. + +B03 bounded run `38084312163` retained FAIL/cleanup PASS in all four instances: +the candidate still did not initialize the replacement format within 90 seconds. +That explicit acceptance failure is not relabeled as a completed reconstruction. + + +E05 `heal.mrf-offline-write-replay` now has a dedicated H2 executor for all +three source branches: unique version PUT, exact historical-version DELETE, +and marker creation while the original fourth peer is stopped and its endpoint +refuses connections. Every ACK must have a fresh active durable responsibility +in the candidate's committed MRF snapshot and matching lifecycle companion. +The read-only observer opens only six exact snapshot/commit/lifecycle filenames +per private root. It validates full journal CRCs, both commit SHA-256 bindings, +owner/sequence, responsibility ID and bucket incarnation; it never enumerates +IAM or credential records, changes journals, or accepts operator risk. + +After the same node and disk identities rejoin, fresh manager admission with +source `mrf`, an exact-version automatic object task, complete successful +responsibility and `repaired`/`verified_healthy`/`authoritatively_absent` must +agree with committed responsibility retirement. Missing/held/unknown results, +queue emptiness, copied backup bytes or a manual verifier cannot certify this +case. All 16 replicas, retained same-role bytes, deleted data directories, +streaming checks and complete all-peer S3 ACK inventories precede PASS. The +record/proof decoder is explicitly bound to candidate source +`072b57b37559fbe626aeda8abf61617fdcbdd926` and reviewed source file digests; +ordinary execution consumes package/tools without a RustFS source checkout. +E05 has no live PASS claim yet. + +F05 run `38084904969` retained FAIL/cleanup PASS before any restart because its +combined DB/DV/DM/DX physical census exceeded the cursor resource profile. It +now registers a distinct bounded 24,576-file census with the original 2 GiB +per-drive free-space requirement and 45-minute service lease. E04 +`38085241447` verified complete journal collection on the corrected `--all` +reader, then failed its healthy task's missing public pool/set responsibility +identity; cleanup passed. The original failed runs remain unchanged. + + +C06 `heal.stale-version-rejoin` passed complete H1/H2 unversioned overwrite, +historical-version deletion and marker creation in run `38085554143` at harness +`8d78ff32188412644a55776be7a794024a4f5614`. All six applicable instances have +PASS and cleanup PASS; downloaded canonical evidence passed the independent +selected-case gate. This raises the full live PASS total to 44. Older incomplete +journal-window runs remain FAIL. + + +B04 `heal.disk-replacement-unformatted` now dispatches both H1/H2 source +branches with independent 128-object ACK fixtures. A second released-process +H1 service generates the actual foreign format; the read-only exporter retains +its exact bytes, digest, candidate binary and cluster/member identities before +complete donor cleanup. The replacement adapter copies only those verified +format bytes into the witnessed empty run-owned root, rejects reused original +identities, and never manufactures a disk UUID or edits the old archive. + +Blank initialization must expose the candidate's new UUID in its exact original +slot, complete the same automatic replacement generation, reconstruct every +original same-role shard, pass streaming verification and preserve all peer +ACK inventories. Refusal instead needs the exact Admin slot to remain explicitly +offline/unformatted/corrupt throughout a registered 90-second window, with all +other UUIDs/formats/user files unchanged and no target user writes. Foreign +format bytes must stay exact. The negative branch then retains the rejected +scene and restores the original archive only for recovery; it explicitly makes +no product-rebuild claim. Complete all-peer S3 and physical checks are still +required. B04 has no live PASS claim yet. + +E06 `38085937937` retained FAIL/cleanup PASS in every H1/H2 DB/DV instance. +Three instances generated a format without a distinct replacement UUID; the +H2 unversioned instance instead exposed partial/unknown/incompatible cluster +replacement status. Neither directory-root fallback nor a format file alone +certifies actual replacement readiness or responsibility. + + +The cursor resource budget now includes the candidate's `part.N.integrity.*` +sidecars: 1,025 external objects on four roots already require at least 12,300 +files. The explicit limit is 16,384 and the registered DX budget reads the same +configuration. F02 `38086220066` and F03 `38086565751` retain FAIL/cleanup PASS: +their DB branches exceeded the old 12,288-file allowance; the DV branches +failed to expose a distinct replacement UUID. Budget correction permits fresh +verification and does not change these original results. + +E05 `38086919036` retained FAIL/cleanup PASS on all three branches. PUT/marker +reached durable ACK binding but the replay caller passed an unsupported label +to `JournalWindow.capture`; the actual shared interface and replay observer +are now covered together by a regression test. Exact historical deletion +failed to expose committed durable responsibility within the registered +30-second persistence deadline. That failure remains; no substitute manual +mutation or inferred queue record is accepted. + +J06 `heal.performance-abba` now dispatches three complete H2 ABBA groups on +the same candidate, rc, four nodes, original roots and disk UUIDs. A frozen +protocol sets 30-second phases, two workers, four requests per second, a 1:1 +GET/PUT ratio and the same 16 logical 64KiB load objects. Every scheduled request +has durable intent/receipt evidence; unknowns, failed samples and missed +admission deadlines remain in both error and latency denominators. Each phase +restarts the same services and uses the same fixed warmup without changing host +caches. Every A phase proves idle throughout and unchanged repair files. Each +B phase injects the same 17 missing data/parity shards in an independent +1,025-object repair bucket and binds one actual task's partial running progress +to both foreground operations. No observed overlap means NOT_OBSERVED and a +complete-case failure. + +Exact owned-process birth, boot and settings bind read-only CPU, logical/storage +IO, RSS and FD samples. Candidate completion, actual reconstructed shard bytes, +streaming verification and full peer S3 manifests must agree. Repair throughput +uses physical target bytes, separately from logical S3 throughput. Frozen gates +require zero errors, GET/PUT P95 <= 5s and P99 <= 10s, paired B/A ratios <= 3, +positive verified repair throughput and bounded resources. All twelve phases +and every paired difference are reported; no phase may be omitted or reordered +and functional success cannot substitute for performance acceptance. The first +candidate run expired its 45-minute fault lease before group 3 phase 3; its +physical failure scene showed no applied fault and the retained namespace was +later independently finalized. The executor now binds a 120-minute lease for +the full evidence window. J06 has no live PASS claim yet and needs a fresh +candidate run. + + +F05 `38087158034` passed the expanded physical census and reached the real +all-peer graceful exit, then retained FAIL/cleanup PASS when the original +process exited between systemd MainPID and `/proc/PID/stat` reads. The node +observer now refreshes the complete service observation up to three times +only for disappearing `/proc/PID` entries. It never infers PID zero from +absence, accepts arbitrary file loss, or relaxes exact normal exit code zero, +service identity and pidfd birth checks. A fresh run is required. + + +B04 `38087657754` retained overall FAIL/cleanup PASS. H1 foreign-format passed +its complete structured refusal window, unchanged original format/controls, +rejected foreign bytes and all 128 ACK/S3 checks, with archive restoration +explicitly excluded from product repair. Both blank branches lacked a distinct +new candidate-generated disk UUID. H2 foreign-format did not make every bound +peer ready; a startup failure alone is not relabeled as structured safe refusal. + +A02 `heal.admin-scope-isolation` now dispatches all five selectors with fresh +H1/H2 fixtures, two owned buckets and exact `a/`, `ab/`, `b/`, nested, Unicode +and literal encoded-slash keys. The candidate-bound H2 runtime preserves the +current four nodes and sixteen independent mounts, explicitly configures two +8-drive sets and interleaves original endpoints so each set spans all four +hosts. Actual format UUIDs, complete set membership, metadata distribution and +same-role shard identities bind every ACK to its physical set. Existing +single-set executors retain their strict default and reject this layout unless +it is explicitly requested. No format, UUID or second control set is invented. +H1 retains its original four-drive set and records the physical applicability +boundary for a nonexistent unselected set. + +Root covers the entire owned service manifest; bucket, exact object and `a/` +prefix selectors preserve other buckets, adjacent keys/prefixes and control +faults; pool/set selectors require real selected-set reconstruction and an +unchanged missing shard in the other actual set. The complete responsibility +set and canonical counters must match all selected ACKs, including healthy +objects. On this exact candidate, unrestricted request selectors may remain +null in raw receipts; independent complete physical mapping resolves their +scope in separate evidence, without rewriting public fields or weakening +shared validators. Explicit selectors require exact integer receipt scope; +partial nulls, wrong sets, mixed incarnations, duplicates and unknowns fail. + +Full physical comparison and selected streaming integrity precede any target +GET or unselected-control backup rescue. Rescue cannot certify product repair. +Both complete S3 manifests through every peer and a final unchanged physical +scene are required. A02 has no live PASS claim yet. Ten scope/topology/receipt/ +control negatives join the focused local checks; 226 checks now pass. + + +D11 now dispatches all four source branches separately on H1/H2 (eight fresh +instances). Its dedicated node backend derives only two owned bucket-config +object paths and their incarnation controls. It never widens the general +user-object fault allowlist, walks IAM records, or prints bucket-policy/inline +payload bytes from physical metadata. The candidate startup persists a non-sensitive +`config/iam/format.json` object even in a fresh root-only service. Its exact +physical files and streaming proofs are read-only controls before and after +execution; mutation never targets that object. The six exact sensitive IAM +subtrees are checked for absence without traversal or payload reads. The reviewed +IAM manager source hash is bound alongside the bucket/XL2 contracts. Exact system object geometry +binds each shard index to the original disk UUID/set slot; current inline or +external bytes require candidate-bound streaming verification. + +The fixture preserves Enabled versioning, bucket-only policy, tags, an Object +Lock default set after fixture writes, and all acknowledged historical business +versions. One config-envelope replica is missing or truncated for positive +recovery; the candidate's startup/config reads and explicit bucket Heal are +recorded as allowed mechanisms. Every original system file must be restored +byte-for-byte before backup rescue or business GET, while incarnation/control +records, business files and formats remain unchanged. The unknown-record +negative changes only outer XL2 major 1 to 2 on the exact primary config +replicas; it claims no inner msgpack schema coverage. It requires structured +public refusal, zero successful repair responsibility, preserved unknown +bytes and healthy control-bucket APIs. A startup/readiness failure alone is +not safe-refusal evidence. Rescue archives remain separate from product +repair, and an unrecognized config rewrite retains the stopped namespace. +D11 still requires live evidence. + +F02 corrected-budget run `38088240693` retained FAIL/cleanup PASS in both H2 +DB/DV branches: neither reached partial-rebuild interruption because the +candidate did not expose a distinct new replacement UUID. F03 corrected-budget +run `38088777413` also retained FAIL/cleanup PASS in both H2 DB/DV branches: +replacement source status was partial, unknown or incompatible. The bounded +file census no longer prevented either run; these product/evidence failures +are not converted into successful restart coverage. + + +The optional native-regression tools job builds the unmodified candidate's +optimized ecstore test executable on hosted Linux, retains Cargo/lock/toolchain +identities, enumerates actual full test names, and runs each selected name with +`--exact` in a fresh process and a fixed 120-second deadline. It currently +collects two real read-repair/namespace barrier regressions and two uncontrolled +overwrite/delete concurrency regressions. Zero-test filters, ignored tests, +foreign source/collector identities and other-name success are rejected. The +four regression verdicts have their own archive and never increment a fleet or +canonical-case PASS count. Existing tests assert captured lock events internally; +the collector does not fabricate emitted barrier logs. Missing critical-point +fixtures, Admin nolock semantics and required fleet overlap remain explicit gaps. +The current implementation has 310 focused Heal/catalog checks, including the +Linux-only ownership/mount/recovery checks. Hosted native compilation and all +four exact regressions completed in run `38096433036`; their independently +verified identities and limits are recorded below. + + +J03 corrected-budget run `38089238767` passed every applicable within-window, +past-retention-gap and terminal-truncation branch on H1/H2. The cross-node +branch is explicitly NOT_APPLICABLE on single-node H1 and passed on H2. It retained complete DX responsibility, real running-status windows, +physical/streaming integrity, all-peer S3 evidence, cleanup PASS and the +independent canonical selected-case gate. This increases complete live PASS +to 45/158; branch-only successes and native regressions remain outside that +count. A06/F06/F01 and A02/E05/F05/J06/D11 queues continue serially on the +current auto-testing fleet. The optional native build uses the existing +Functional Chain tools-only driver on hosted Linux, because the reusable +workflow is still in a draft PR and cannot be dispatched directly from the +default branch. + + +B08 now has an automatic current-fleet executor for both source branches. +The write-negative branch measures every original metadata read and root write +inside the exact candidate process mount namespace. Precisely N-W+1 private +roots return EROFS, including the missing payload target; all N metadata +members and all lock clients remain online. The H2 lock-negative branch +isolates two exact run-owned peer processes, witnesses their exited MainPIDs +and refused endpoints, and keeps R metadata/payload sources and shared-lock +quorum while losing exclusive-lock and object-write quorum. It does not claim +host network partition or physical disk loss. H1 uses an in-process local +manager; its distributed peer-loss branch is explicitly NOT_APPLICABLE with +the reviewed candidate locking policy, while its write-negative branch remains +mandatory. Both original branch IDs and H1/H2 contracts are retained. + +Every negative scene is frozen before removing the fault. A single unique-key +PUT has no automatic retry; all unknown outcomes and success ACKs are queried +on every recovered peer. Unresolved outcomes retain the stopped namespace and +fail cleanup. The negative Heal must preserve complete failed/deferred exact +responsibility with zero repaired counters and no changed physical control. +After gate removal/source return, an independent task must reconstruct the +original missing target before any target GET or backup rescue. Streaming +verification, all-peer complete S3 manifests and clean finalization remain +required. Ten boundary, identity, mount, cleanup and unknown-ACK negatives join +the focused checks. This implementation section predates the complete B08 acceptance above; its earlier no-PASS statement remains historical. + +A06 run `38090151509` retained FAIL/cleanup PASS on H1/H2 after its repeat-stop +assertion. It exposed a harness contract defect: the reviewed manager's +`root_recovery::cancel_pending` treats a retained terminal record as an +idempotent stop, so the public response is HTTP 200/stopped, rather than the +unknown-token HTTP 500 assumed by the harness. The original result remains +unchanged. Fixing the candidate-bound assertion and repeating the complete +canonical run are required; the observed acknowledgement alone is not PASS. + + +The A06/F06 durable-retired-token assertion is corrected against all four +reviewed source hashes, including `root_recovery::cancel_pending`. It now +requires the exact signed old-token request and the candidate's HTTP 200, +stopped acknowledgement shape. Its default acknowledgement settings and new +response timestamp are not mistaken for task identity or original task options. +The old settled canonical outcome must remain byte-equivalent immediately +after both repeat and stale stops, and the new task must stay running across +the stale request. Unknown-token queries and forceStart-retired tokens without +a retained durable terminal keep their independent exact 500/InternalError +checks. Both archived A06 acknowledgement samples satisfy the corrected shape; +that replay is not fresh live acceptance. Complete A06/F06 reruns remain required. + + +C09 `heal.foreign-format-and-unknown-schema` dispatches all four canonical +branches on H1/H2. An isolated second released-process deployment exports a +real foreign format. The disk branch replaces only `xl.this` with its witnessed +donor UUID; the set branch copies the exact donor bytes, including its real +four-member set when the target has sixteen members. This topology difference +is retained explicitly. The future-format branch changes only the top-level +V1 semantic version to `999`. Only one empty, owned replacement root is +modified, while its complete original root is quarantined and inaccessible to +the candidate. Shared formats and IAM are outside the mutation scope. + +A registered 90-second window requires exact Admin slot/endpoint refusal and +stable original peer identities, followed by deep Heal and target GET. Startup +failure alone cannot earn PASS. Manual Heal may verify the surviving object or +report its complete failure, but must never claim reconstruction of the refused +member. All surviving files/formats and the injected bytes stay frozen. + +The object branch changes only the reviewed XL2 envelope's little-endian major +from 1 to 2 on every member of one ACKed target. It does not fabricate a future +inner msgpack schema. Per-disk metadata hashes bind each actual decoder refusal; +all peer GETs must return a structured server refusal, and deep Heal must retain +complete failed responsibility without a missing/healthy/repaired claim. All +payloads, control objects and formats remain exact. Only after the negative +proof may original bytes/identity be rescued. All 128 ACKs, every external shard +with streaming checksums, and full S3 inventories through every peer are checked +again. Backup rescue never earns product-repair credit; failed recovery retains +the namespace and stops subsequent fleet use. C09 still requires fresh live +evidence. Eleven targeted oracle/guard tests are added; its actual Linux +FD/mount guard test runs in CI, not as a simulated macOS syscall claim. + + +F01 `heal.root-graceful-coordinator-restart` run `38091619866` retained +canonical FAIL with cleanup PASS. H1 passed both branches; H2 exceeded the +collector's 2 MiB response limit during original-token recovery. The reviewed +candidate `crates/heal/src/heal/channel.rs` permits an 8 MiB status payload +(source SHA-256 `7f669ca43b9559db9b97fba6b419a6c76a94d76acd8526db132d74b7c534d6b0`). +The collector now uses that explicit 8 MiB bound, records complete byte counts +and still rejects overflow or Content-Length mismatch. A boundary regression +covers a complete 8 MiB JSON response, one-byte overflow and truncated bodies. +No F01 canonical PASS is claimed until a fresh complete rerun. + +A06 run `38092515125` retained FAIL/cleanup PASS on both H1/H2 because the +collector represents signed query pairs as Python tuples in memory and as +JSON arrays on disk. The exact durable-stop validator incorrectly accepted +only the latter. It now normalizes either pair representation while retaining +the exact pair count/order/token, signed POST, empty body, response shape and +old/new responsibility assertions. Tests reject duplicated tokens and mapping +queries. All six CI lanes passed at `62d0ca9`; fresh complete A06/F06 runs are +still required. Earlier FAIL results are preserved. + +D08 `heal.encrypted-objects` dispatches SSE-S3, SSE-KMS, SSE-C and encrypted +historical versions on both H1/H2. It configures one fresh AES-256 static KMS +key across the run-owned stopped services, passes material through SSH stdin +and retains it only in a private 0600 file inside each run namespace. Every +snapshot attests the file/configuration identity and the actual process +settings, including explicit per-key authorization enforcement. Shared KMS +resources are not modified and secret material is excluded from evidence. + +Each branch independently ACKs sixteen external encrypted objects; the +versioned branch ACKs three distinct generations per key. It removes a data +shard and corrupts a parity shard, including historical and current versions, +then requires a manual deep token with complete exact responsibility. Before +any decrypted target GET, the original same-role ciphertext hashes/lengths and +streaming integrity must be reconstructed. An encrypted-only mapper preserves +the distinct plaintext actualSize and ciphertext part size without relaxing +the ordinary plaintext mapper. Every peer then verifies the entire plaintext +ACK/version/tag/property inventory and re-freezes the ciphertext scene. + +SSE-C missing/wrong keys must produce the reviewed 400/InvalidRequest refusal. +SSE-KMS principals have the same working S3 GET rights to an independently +verified plaintext control, but no decrypt grant or only an unrelated key +grant; the encrypted GET must return 403/AccessDenied before and after repair. +SSE-S3 records that its service-managed key has no caller decryption-key +argument. Failed IAM cleanup retains a DIRTY stopped namespace. Nine targeted +geometry, authority, version, redaction and Linux file-ownership tests pass; +D08 still requires fresh complete live evidence and has no live PASS claim. + + +A02 `38094094085` passed every applicable H1/H2 root, bucket, object, prefix, +recursive and actual two-set selector branch. H1 keeps the source's selector +inapplicability explicit; H2 tests the actual independent sets. Its complete +physical/streaming and all-peer ACK manifests, cleanup PASS and independent +canonical gate increase the complete live total to 46/158. F06 `38093199055` +retained FAIL/cleanup PASS on the already queued old harness; its durable-stop +query assertion is fixed at `62d0ca9` and remains queued for a fresh run. +B08 `38093907057` retained FAIL/cleanup PASS: H1 lacks the exact incarnation +receipt, while H2's actual default layout cannot preserve R with the selected +write/lock-loss plan. No layout or missing identity is assumed sufficient. + +F04 `heal.coordinator-process-crash` now dispatches its three separate source +mechanisms on actual H2. It records a candidate-bound persistence support table +and only sends SIGKILL through Linux pidfd after freezing the unique original +coordinator at nonzero and unfinished physical progress. Target and other peer +process birth identities must remain unchanged throughout recovery. No host or +platform restart is issued, and an unobserved window cannot pass. + +The root branch ACKs the full DX fixture, places missing original data shards +outside the coordinator, freezes the exact durable admission owner and requires +the original token to replay with complete responsibility. SIGKILL must leave +the original intent bytes unchanged; it makes no graceful elapsed-time or cursor +checkpoint claim. Repeated terminal restart, original record disk, full physical +hashes/streaming and all-peer S3 inventories remain mandatory. + +The MRF branch ACKs sixty-four independent never-versioned DB writes while one +original node is offline. Its isolated profile enables only MRF, with one worker, +one task per set and a replay batch of one. Actual process settings and the +configuration source digest are retained. Fresh accepted MRF task/start logs, +exact version/null representation and a partial target census bind the paused +coordinator to a still-committed original ACK responsibility. The complete +CRC/commit/lifecycle checkpoint remains byte-identical across the crash. +The lost volatile token must return its explicit unknown-token error; a separate +new automatic task must prove the interrupted ACK, while every ACK's original +responsibility retires with a verified receipt. Automatic rescan success is never +reported as original-token recovery. The final census, all surviving original +shards, streaming checks and S3 inventories gate the branch. + +The replacement branch independently ACKs 1025 external objects and preserves +an inaccessible original drive archive. Its coordinator is distinct from the +live target. An exact-name read-only journal captures the schema-seven sealed +intent for the original generation, endpoint, physical device and filesystem +identity before and after SIGKILL. The original generation must finish on every +peer after the same coordinator unit restarts, with the original replacement +UUID/set slot, full same-role shard hashes, streaming and ACK inventories. +No manual replacement compensation or backup rescue earns repair credit. + +Eleven focused F04 tests cover process continuity/exit, exact root intent, +replacement seals/name/identity and filesystem guards, partial MRF authority, +source attribution, bounded configuration and mandatory three-mechanism scoring. +F04 has no live PASS claim yet. The source-bound MRF log reader also accepts the +reviewed Rust Option debug spelling without merging different version identities; +existing E05 receipts and every original branch remain independently required. + + +B08's read-only plan now measures writable drives independently of readable +sources, including the candidate's actual default H2 R=12/W=12 geometry. Five +owned read-only roots leave fifteen payload sources and all sixteen metadata +authorities readable while only eleven roots remain writable. Numerical R